openreplay/api/chalicelib/core/health.py

from urllib.parse import urlparse

import redis
import requests
from decouple import config

from chalicelib.utils import pg_client

if config("LOCAL_DEV", cast=bool, default=False):
    HEALTH_ENDPOINTS = {
        "alerts": "http://127.0.0.1:8888/metrics",
        "assets": "http://127.0.0.1:8888/metrics",
        "assist": "http://127.0.0.1:8888/metrics",
        "chalice": "http://127.0.0.1:8888/metrics",
        "db": "http://127.0.0.1:8888/metrics",
        "ender": "http://127.0.0.1:8888/metrics",
        "heuristics": "http://127.0.0.1:8888/metrics",
        "http": "http://127.0.0.1:8888/metrics",
        "ingress-nginx": "http://127.0.0.1:8888/metrics",
        "integrations": "http://127.0.0.1:8888/metrics",
        "peers": "http://127.0.0.1:8888/metrics",
        "quickwit": "http://127.0.0.1:8888/metrics",
        "sink": "http://127.0.0.1:8888/metrics",
        "sourcemapreader": "http://127.0.0.1:8888/metrics",
        "storage": "http://127.0.0.1:8888/metrics",
        "utilities": "http://127.0.0.1:8888/metrics"
    }

else:
    HEALTH_ENDPOINTS = {
        "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/health",
        "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics",
        "assist": "http://assist-openreplay.app.svc.cluster.local:8888/health",
        "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics",
        "db": "http://db-openreplay.app.svc.cluster.local:8888/metrics",
        "ender": "http://ender-openreplay.app.svc.cluster.local:8888/metrics",
        "heuristics": "http://heuristics-openreplay.app.svc.cluster.local:8888/metrics",
        "http": "http://http-openreplay.app.svc.cluster.local:8888/metrics",
        "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics",
        "integrations": "http://integrations-openreplay.app.svc.cluster.local:8888/metrics",
        "peers": "http://peers-openreplay.app.svc.cluster.local:8888/health",
        "sink": "http://sink-openreplay.app.svc.cluster.local:8888/metrics",
        "sourcemapreader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/health",
        "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics",
    }


def __check_database_pg():
    with pg_client.PostgresClient() as cur:
        cur.execute("SHOW server_version;")
        server_version = cur.fetchone()
        cur.execute("SELECT openreplay_version() AS version;")
        schema_version = cur.fetchone()
    return {
        "health": True,
        "details": {
            "version": server_version["server_version"],
            "schema": schema_version["version"]
        }
    }


def __not_supported():
    return {"errors": ["not supported"]}


def __always_healthy():
    return {
        "health": True,
        "details": {}
    }


def __always_healthy_with_version():
    return {
        "health": True,
        "details": {"version": config("version_number", default="unknown")}
    }


def __check_be_service(service_name):
    def fn():
        fail_response = {
            "health": False,
            "details": {
                "errors": ["server health-check failed"]
            }
        }
        try:
            results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2)
            if results.status_code != 200:
                print(f"!! issue with the storage-health code:{results.status_code}")
                print(results.text)
                fail_response["details"]["errors"].append(results.text)
                return fail_response
        except requests.exceptions.Timeout:
            print(f"!! Timeout getting {service_name}-health")
            fail_response["details"]["errors"].append("timeout")
            return fail_response
        except Exception as e:
            print("!! Issue getting storage-health response")
            print(str(e))
            try:
                print(results.text)
                fail_response["details"]["errors"].append(results.text)
            except:
                print("couldn't get response")
                fail_response["details"]["errors"].append(str(e))
            return fail_response
        return {
            "health": True,
            "details": {}
        }

    return fn


def __check_redis():
    fail_response = {
        "health": False,
        "details": {"errors": ["server health-check failed"]}
    }
    if config("REDIS_STRING", default=None) is None:
        fail_response["details"]["errors"].append("REDIS_STRING not defined in env-vars")
        return fail_response

    try:
        u = urlparse(config("REDIS_STRING"))
        r = redis.Redis(host=u.hostname, port=u.port, socket_timeout=2)
        r.ping()
    except Exception as e:
        print("!! Issue getting redis-health response")
        print(str(e))
        fail_response["details"]["errors"].append(str(e))
        return fail_response

    return {
        "health": True,
        "details": {"version": r.execute_command('INFO')['redis_version']}
    }


def get_health():
    health_map = {
        "databases": {
            "postgres": __check_database_pg
        },
        "ingestionPipeline": {
            "redis": __check_redis
        },
        "backendServices": {
            "alerts": __check_be_service("alerts"),
            "assets": __check_be_service("assets"),
            "assist": __check_be_service("assist"),
            "chalice": __always_healthy_with_version,
            "db": __check_be_service("db"),
            "ender": __check_be_service("ender"),
            "frontend": __always_healthy,
            "heuristics": __check_be_service("heuristics"),
            "http": __check_be_service("http"),
            "ingress-nginx": __always_healthy,
            "integrations": __check_be_service("integrations"),
            "peers": __check_be_service("peers"),
            "sink": __check_be_service("sink"),
            "sourcemapreader": __check_be_service("sourcemapreader"),
            "storage": __check_be_service("storage")
        }
    }
    for parent_key in health_map.keys():
        for element_key in health_map[parent_key]:
            health_map[parent_key][element_key] = health_map[parent_key][element_key]()
    return health_map