monitor

commit 3f4faf42e3b4a916b60a4aa39561ad64e4b0022c
parent 7897adeb7a8f531266d371fb305ed33f4e462b84
Author: bain <bain@bain.cz>
Date:   Tue, 28 Dec 2021 01:50:13 +0100

fix: typing; add: retry decoration function
additionaly add retries to ip_requirement

Diffstat:
M
monitor.py
 | 
55
++++++++++++++++++++++++++++++++++++++-----------------

1 file changed, 38 insertions(+), 17 deletions(-)
diff --git a/monitor.py b/monitor.py
@@ -6,6 +6,7 @@ import datetime
 import socket
 import json
 import os
+from typing import Callable
 
 import requests
 import pydig
@@ -19,7 +20,10 @@ logger.setLevel(logging.DEBUG)
 REPO_ROOT = "status-repo"
 
 # last states of services to keep from detecting downtime repeatedly
-last_states = {}
+last_states: dict[str, bool] = {}
+
+RequirementCheck = Callable[..., bool]
+MonitorDict = dict[str, dict[RequirementCheck, dict]]
 
 
 # publish a failed service, no dependents so edit at will
@@ -36,6 +40,7 @@ def fail(service_name: str, failed_requirements: list):
         logger.warning("Failed to pull from origin! Aborting!")
         return
 
+    # noinspection PyShadowingNames
     with open(REPO_ROOT + "/" + filename, 'w+') as f:
         lines = [
             "---\n",
@@ -60,7 +65,7 @@ def fail(service_name: str, failed_requirements: list):
     logger.warning(f"service {service_name} failed {[r.__name__ for r in failed_requirements]}")
 
 
-def self_check():
+def self_check() -> bool:
     try:
         if requests.get("https://google.com/").status_code != 200:
             return False
@@ -69,18 +74,33 @@ def self_check():
     return True
 
 
-def http_requirement(url: str, code: int) -> bool:
-    passed = False
-    for i in range(2):
-        try:
-            resp = requests.head(url)
-        except requests.exceptions.ConnectionError:
+def retry(n: int = 3, sleep: int = 5) -> Callable[[RequirementCheck], RequirementCheck]:
+    """Decorator maker for calling a function multiple times with sleep time between calls."""
+
+    def inner_retry(func: RequirementCheck) -> RequirementCheck:
+        def inner(*args, **kwargs) -> bool:
             passed = False
-        else:
-            passed = resp.status_code == code
-        if passed:
-            break
-    return passed
+            for i in range(n - 1):
+                passed = func(*args, **kwargs)
+                if passed:
+                    break
+                time.sleep(sleep)
+            return passed
+
+        inner.__name__ = func.__name__  # preserve names in log (instead of each requirement being called "inner")
+        return inner
+
+    return inner_retry
+
+
+@retry()
+def http_requirement(url: str, code: int) -> bool:
+    try:
+        resp = requests.head(url)
+    except requests.exceptions.ConnectionError:
+        return False
+    else:
+        return resp.status_code == code
 
 
 def dns_requirement(name: str, ip: str) -> bool:
@@ -91,6 +111,7 @@ def dns_requirement(name: str, ip: str) -> bool:
     return query and (ip == "*" or ip in query)
 
 
+@retry()
 def ip_requirement(ip: str, port: int, prot: str) -> bool:
     protocol = socket.SOCK_STREAM if prot == "tcp" else socket.SOCK_DGRAM
     sock = socket.socket(type=protocol)
@@ -102,7 +123,7 @@ def ip_requirement(ip: str, port: int, prot: str) -> bool:
     return True
 
 
-def check(monitors: dict):
+def check(monitors: MonitorDict):
     for service, requirements in monitors.items():
         logger.debug(f"Checking service {service}")
         failed = []
@@ -122,7 +143,7 @@ def check(monitors: dict):
     logger.debug("check complete")
 
 
-monitors = {
+monitors_: MonitorDict = {
     "f.bain.cz": {
         http_requirement: {"url": "https://f.bain.cz/status", "code": 200},
         # dns_requirement: {"name": "f.bain.cz", "ip": "*"},
@@ -140,13 +161,13 @@ monitors = {
 }
 
 if __name__ == '__main__':
-    # we assume this is gonna be run in a cron job as the gitpython
+    # we assume this is going to be run in a cron job as the gitpython
     # library is slowly leaking memory apparently
     if os.path.exists("last-state"):
         with open("last-state", 'r') as f:
             last_states = json.load(f)
 
-    check(monitors)
+    check(monitors_)
 
     with open("last-state", 'w+') as f:
         json.dump(last_states, f)