jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(chalice): health-check test

Browse source
This commit is contained in:
Taha Yassine Kraiem 2023-03-07 18:12:36 +01:00
parent 1d06e651ea
commit 86af6f37c5
6 changed files with 211 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
api/chalicelib/core/health.py
View file

@ -1,3 +1,6 @@
from urllib.parse import urlparse
import redis
import requests
from decouple import config
@ -65,7 +68,21 @@ def __not_supported():
return {"errors": ["not supported"]}
def check_be_service(service_name):
def __always_healthy():
return {
"health": True,
"details": {}
}
def __always_healthy_with_version():
return {
"health": True,
"details": {"version": config("version_number", default="unknown")}
}
def __check_be_service(service_name):
def fn():
fail_response = {
"health": False,
@ -87,7 +104,6 @@ def check_be_service(service_name):
except Exception as e:
print("!! Issue getting storage-health response")
print(str(e))
print("expected JSON, received:")
try:
print(results.text)
fail_response["details"]["errors"].append(results.text)
@ -103,32 +119,61 @@ def check_be_service(service_name):
return fn
def __check_redis():
fail_response = {
"health": False,
"details": {"errors": ["server health-check failed"]}
}
if config("REDIS_STRING", default=None) is None:
fail_response["details"]["errors"].append("REDIS_STRING not defined in env-vars")
return fail_response
try:
u = urlparse(config("REDIS_STRING"))
r = redis.Redis(host=u.hostname, port=u.port, socket_timeout=2)
r.ping()
except Exception as e:
print("!! Issue getting assist-health response")
print(str(e))
fail_response["details"]["errors"].append(str(e))
return fail_response
return {
"health": True,
"details": {"version": r.execute_command('INFO')['redis_version']}
}
def __check_assist():
pass
def get_health():
health_map = {
"databases": {
"postgres": __check_database_pg
},
"ingestionPipeline": {
"redis": __not_supported
"redis": __check_redis
},
"backendServices": {
"alerts": check_be_service("alerts"),
"assets": check_be_service("assets"),
"assist": check_be_service("assist"),
"chalice": check_be_service("chalice"),
"db": check_be_service("db"),
"ender": check_be_service("ender"),
"frontend": check_be_service("frontend"),
"heuristics": check_be_service("heuristics"),
"http": check_be_service("http"),
"ingress-nginx": check_be_service("ingress-nginx"),
"integrations": check_be_service("integrations"),
"peers": check_be_service("peers"),
"quickwit": check_be_service("quickwit"),
"sink": check_be_service("sink"),
"sourcemapreader": check_be_service("sourcemapreader"),
"storage": check_be_service("storage"),
"utilities": check_be_service("utilities")
"alerts": __check_be_service("alerts"),
"assets": __check_be_service("assets"),
"assist": __check_assist,
"chalice": __always_healthy_with_version,
"db": __check_be_service("db"),
"ender": __check_be_service("ender"),
"frontend": __check_be_service("frontend"),
"heuristics": __check_be_service("heuristics"),
"http": __check_be_service("http"),
"ingress-nginx": __always_healthy,
"integrations": __check_be_service("integrations"),
"peers": __check_be_service("peers"),
"quickwit": __check_be_service("quickwit"),
"sink": __check_be_service("sink"),
"sourcemapreader": __check_be_service("sourcemapreader"),
"storage": __check_be_service("storage"),
"utilities": __check_be_service("utilities")
},
# "overall": {
# "health": "na",

2
api/env.default
View file

@ -52,4 +52,4 @@ PRESIGNED_URL_EXPIRATION=3600
ASSIST_JWT_EXPIRATION=144000
ASSIST_JWT_SECRET=
PYTHONUNBUFFERED=1
THUMBNAILS_BUCKET=thumbnails
REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379

2
api/requirements.txt
View file

@ -13,3 +13,5 @@ uvicorn[standard]==0.20.0
python-decouple==3.7
pydantic[email]==1.10.4
apscheduler==3.10.0
redis==4.5.1

223
ee/api/chalicelib/core/health.py
View file

@ -1,3 +1,6 @@
from urllib.parse import urlparse
import redis
import requests
from decouple import config
@ -61,6 +64,137 @@ def __check_database_pg():
}
def __not_supported():
return {"errors": ["not supported"]}
def __always_healthy():
return {
"health": True,
"details": {}
}
def __always_healthy_with_version():
return {
"health": True,
"details": {"version": config("version_number", default="unknown")}
}
def __check_be_service(service_name):
def fn():
fail_response = {
"health": False,
"details": {
"errors": ["server health-check failed"]
}
}
try:
results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2)
if results.status_code != 200:
print(f"!! issue with the storage-health code:{results.status_code}")
print(results.text)
fail_response["details"]["errors"].append(results.text)
return fail_response
except requests.exceptions.Timeout:
print(f"!! Timeout getting {service_name}-health")
fail_response["details"]["errors"].append("timeout")
return fail_response
except Exception as e:
print("!! Issue getting storage-health response")
print(str(e))
try:
print(results.text)
fail_response["details"]["errors"].append(results.text)
except:
print("couldn't get response")
fail_response["details"]["errors"].append(str(e))
return fail_response
return {
"health": True,
"details": {}
}
return fn
def __check_redis():
fail_response = {
"health": False,
"details": {"errors": ["server health-check failed"]}
}
if config("REDIS_STRING", default=None) is None:
fail_response["details"]["errors"].append("REDIS_STRING not defined in env-vars")
return fail_response
try:
u = urlparse(config("REDIS_STRING"))
r = redis.Redis(host=u.hostname, port=u.port, socket_timeout=2)
r.ping()
except Exception as e:
print("!! Issue getting assist-health response")
print(str(e))
fail_response["details"]["errors"].append(str(e))
return fail_response
return {
"health": True,
"details": {"version": r.execute_command('INFO')['redis_version']}
}
def __check_assist():
pass
def get_health():
health_map = {
"databases": {
"postgres": __check_database_pg,
"clickhouse": __check_database_ch
},
"ingestionPipeline": {
"redis": __check_redis,
"kafka": __not_supported
},
"backendServices": {
"alerts": __check_be_service("alerts"),
"assets": __check_be_service("assets"),
"assist": __check_assist,
"chalice": __always_healthy_with_version,
"db": __check_be_service("db"),
"ender": __check_be_service("ender"),
"frontend": __check_be_service("frontend"),
"heuristics": __check_be_service("heuristics"),
"http": __check_be_service("http"),
"ingress-nginx": __always_healthy,
"integrations": __check_be_service("integrations"),
"peers": __check_be_service("peers"),
"quickwit": __check_be_service("quickwit"),
"sink": __check_be_service("sink"),
"sourcemapreader": __check_be_service("sourcemapreader"),
"storage": __check_be_service("storage"),
"utilities": __check_be_service("utilities")
},
# "overall": {
# "health": "na",
# "details": {
# "numberOfEventCaptured": "int",
# "numberOfSessionsCaptured": "int"
# },
# "labels": {
# "parent": "information"
# }
# },
# "ssl": True
}
for parent_key in health_map.keys():
for element_key in health_map[parent_key]:
health_map[parent_key][element_key] = health_map[parent_key][element_key]()
return health_map
def __check_database_ch():
errors = {}
with ch_client.ClickHouseClient() as ch:
@ -84,90 +218,5 @@ def __check_database_ch():
}
def __not_supported():
return {"errors": ["not supported"]}
def check_be_service(service_name):
def fn():
fail_response = {
"health": False,
"details": {
"errors": ["server health-check failed"]
}
}
try:
results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2)
if results.status_code != 200:
print(f"!! issue with the storage-health code:{results.status_code}")
print(results.text)
fail_response["details"]["errors"].append(results.text)
return fail_response
except requests.exceptions.Timeout:
print(f"!! Timeout getting {service_name}-health")
fail_response["details"]["errors"].append("timeout")
return fail_response
except Exception as e:
print("!! Issue getting storage-health response")
print(str(e))
print("expected JSON, received:")
try:
print(results.text)
fail_response["details"]["errors"].append(results.text)
except:
print("couldn't get response")
fail_response["details"]["errors"].append(str(e))
return fail_response
return {
"health": True,
"details": {}
}
return fn
def get_health():
health_map = {
"databases": {
"postgres": __check_database_pg,
"clickhouse": __check_database_ch
},
"ingestionPipeline": {
"redis": __not_supported,
"kafka": __not_supported
},
"backendServices": {
"alerts": check_be_service("alerts"),
"assets": check_be_service("assets"),
"assist": check_be_service("assist"),
"chalice": check_be_service("chalice"),
"db": check_be_service("db"),
"ender": check_be_service("ender"),
"frontend": check_be_service("frontend"),
"heuristics": check_be_service("heuristics"),
"http": check_be_service("http"),
"ingress-nginx": check_be_service("ingress-nginx"),
"integrations": check_be_service("integrations"),
"peers": check_be_service("peers"),
"quickwit": check_be_service("quickwit"),
"sink": check_be_service("sink"),
"sourcemapreader": check_be_service("sourcemapreader"),
"storage": check_be_service("storage"),
"utilities": check_be_service("utilities")
},
# "overall": {
# "health": "na",
# "details": {
# "numberOfEventCaptured": "int",
# "numberOfSessionsCaptured": "int"
# },
# "labels": {
# "parent": "information"
# }
# },
# "ssl": True
}
for parent_key in health_map.keys():
for element_key in health_map[parent_key]:
health_map[parent_key][element_key] = health_map[parent_key][element_key]()
return health_map
def __check_kafka():
pass

5
ee/api/env.default
View file

@ -70,4 +70,7 @@ SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe
DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob
PRESIGNED_URL_EXPIRATION=3600
ASSIST_JWT_EXPIRATION=144000
ASSIST_JWT_SECRET=
ASSIST_JWT_SECRET=
REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379
KAFKA_SERVERS=kafka.db.svc.cluster.local:9092
KAFKA_USE_SSL=false

3
ee/api/requirements.txt
View file

@ -17,3 +17,6 @@ apscheduler==3.10.0
clickhouse-driver==0.2.5
python3-saml==1.15.0
python-multipart==0.0.5
redis==4.5.1
kafka-python==2.0.2