feat(chalice): upgraded startup/shutdown logic

feat(alerts): health-check endpoint

api/app_alerts.py

@@ -1,33 +1,17 @@
 import logging
+from contextlib import asynccontextmanager
 
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from decouple import config
 from fastapi import FastAPI
-from chalicelib.utils import pg_client
 
 from chalicelib.core import alerts_processor
-
+from chalicelib.utils import pg_client
-app = FastAPI(root_path="/alerts", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default=""))
-logging.info("============= ALERTS =============")
 
 
-@app.get("/")
+@asynccontextmanager
-async def root():
+async def lifespan(app: FastAPI):
-    return {"status": "Running"}
+    # Startup
-
-
-app.schedule = AsyncIOScheduler()
-
-loglevel = config("LOGLEVEL", default=logging.INFO)
-print(f">Loglevel set to: {loglevel}")
-logging.basicConfig(level=loglevel)
-ap_logger = logging.getLogger('apscheduler')
-ap_logger.setLevel(loglevel)
-app.schedule = AsyncIOScheduler()
-
-
-@app.on_event("startup")
-async def startup():
     logging.info(">>>>> starting up <<<<<")
     await pg_client.init()
     app.schedule.start()
@@ -39,24 +23,44 @@ async def startup():
     for job in app.schedule.get_jobs():
         ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)})
 
+    # App listening
+    yield
 
-@app.on_event("shutdown")
+    # Shutdown
-async def shutdown():
     logging.info(">>>>> shutting down <<<<<")
     app.schedule.shutdown(wait=False)
     await pg_client.terminate()
 
 
-@app.get('/private/shutdown', tags=["private"])
+app = FastAPI(root_path="/alerts", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default=""),
-async def stop_server():
+              lifespan=lifespan)
-    logging.info("Requested shutdown")
+logging.info("============= ALERTS =============")
-    await shutdown()
-    import os, signal
-    os.kill(1, signal.SIGTERM)
 
 
+@app.get("/")
+async def root():
+    return {"status": "Running"}
+
+
+@app.get("/health")
+async def get_health_status():
+    return {"data": {
+        "health": True,
+        "details": {"version": config("version_number", default="unknown")}
+    }}
+
+
+app.schedule = AsyncIOScheduler()
+
+loglevel = config("LOGLEVEL", default=logging.INFO)
+print(f">Loglevel set to: {loglevel}")
+logging.basicConfig(level=loglevel)
+ap_logger = logging.getLogger('apscheduler')
+ap_logger.setLevel(loglevel)
+app.schedule = AsyncIOScheduler()
+
 if config("LOCAL_DEV", default=False, cast=bool):
-    @app.get('/private/trigger', tags=["private"])
+    @app.get('/trigger', tags=["private"])
     async def trigger_main_cron():
         logging.info("Triggering main cron")
         alerts_processor.process()

api/chalicelib/core/health.py

@@ -29,7 +29,7 @@ if config("LOCAL_DEV", cast=bool, default=False):
 
 else:
     HEALTH_ENDPOINTS = {
-        "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/metrics",
+        "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/health",
         "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics",
         "assist": "http://assist-openreplay.app.svc.cluster.local:8888/health",
         "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics",

api/entrypoint.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload --proxy-headers
+uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --proxy-headers

api/entrypoint_alerts.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 export ASSIST_KEY=ignore
-uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload
+uvicorn app:app --host 0.0.0.0 --port 8888

api/run-alerts-dev.sh

@@ -1,3 +1,3 @@
 #!/bin/zsh
 
-uvicorn app_alerts:app --reload
+uvicorn app_alerts:app --reload --port 8888

ee/api/.gitignore

@@ -265,6 +265,8 @@ Pipfile.lock
 /app_alerts.py
 /build_alerts.sh
 /build_crons.sh
+/run-dev.sh
+/run-alerts-dev.sh
 /routers/subs/health.py
 /routers/subs/v1_api.py
 #exp /chalicelib/core/dashboards.py

ee/api/app.py

@@ -1,5 +1,6 @@
 import logging
 import queue
+from contextlib import asynccontextmanager
 
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from decouple import config
@@ -10,9 +11,9 @@ from starlette import status
 from starlette.responses import StreamingResponse, JSONResponse
 
 from chalicelib.core import traces
+from chalicelib.utils import events_queue
 from chalicelib.utils import helper
 from chalicelib.utils import pg_client
-from chalicelib.utils import events_queue
 from routers import core, core_dynamic, ee, saml
 from routers.crons import core_crons
 from routers.crons import core_dynamic_crons
@@ -20,7 +21,43 @@ from routers.crons import ee_crons
 from routers.subs import insights, metrics, v1_api_ee
 from routers.subs import v1_api, health
 
-app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default=""))
+loglevel = config("LOGLEVEL", default=logging.INFO)
+print(f">Loglevel set to: {loglevel}")
+logging.basicConfig(level=loglevel)
+ap_logger = logging.getLogger('apscheduler')
+ap_logger.setLevel(loglevel)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup
+    logging.info(">>>>> starting up <<<<<")
+    app.schedule = AsyncIOScheduler()
+    app.queue_system = queue.Queue()
+    await pg_client.init()
+    await events_queue.init()
+    app.schedule.start()
+
+    for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs + traces.cron_jobs + ee_crons.ee_cron_jobs:
+        app.schedule.add_job(id=job["func"].__name__, **job)
+
+    ap_logger.info(">Scheduled jobs:")
+    for job in app.schedule.get_jobs():
+        ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)})
+
+    # App listening
+    yield
+
+    # Shutdown
+    logging.info(">>>>> shutting down <<<<<")
+    app.schedule.shutdown(wait=True)
+    await traces.process_traces_queue()
+    await events_queue.terminate()
+    await pg_client.terminate()
+
+
+app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default=""),
+              lifespan=lifespan)
 app.add_middleware(GZipMiddleware, minimum_size=1000)
 
 
@@ -71,43 +108,3 @@ app.include_router(v1_api_ee.app_apikey)
 app.include_router(health.public_app)
 app.include_router(health.app)
 app.include_router(health.app_apikey)
-
-loglevel = config("LOGLEVEL", default=logging.INFO)
-print(f">Loglevel set to: {loglevel}")
-logging.basicConfig(level=loglevel)
-ap_logger = logging.getLogger('apscheduler')
-ap_logger.setLevel(loglevel)
-app.schedule = AsyncIOScheduler()
-app.queue_system = queue.Queue()
-
-
-@app.on_event("startup")
-async def startup():
-    logging.info(">>>>> starting up <<<<<")
-    await pg_client.init()
-    await events_queue.init()
-    app.schedule.start()
-
-    for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs + traces.cron_jobs + ee_crons.ee_cron_jobs:
-        app.schedule.add_job(id=job["func"].__name__, **job)
-
-    ap_logger.info(">Scheduled jobs:")
-    for job in app.schedule.get_jobs():
-        ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)})
-
-
-@app.on_event("shutdown")
-async def shutdown():
-    logging.info(">>>>> shutting down <<<<<")
-    app.schedule.shutdown(wait=True)
-    await traces.process_traces_queue()
-    await events_queue.terminate()
-    await pg_client.terminate()
-
-
-@app.get('/private/shutdown', tags=["private"])
-async def stop_server():
-    logging.info("Requested shutdown")
-    await shutdown()
-    import os, signal
-    os.kill(1, signal.SIGTERM)

ee/api/clean-dev.sh

@@ -86,3 +86,5 @@ rm -rf ./chalicelib/core/performance_event.py
 rm -rf ./chalicelib/core/saved_search.py
 rm -rf ./app_alerts.py
 rm -rf ./build_alerts.sh
+rm -rf ./run-dev.sh
+rm -rf ./run-alerts-dev.sh

ee/api/entrypoint.sh

@@ -2,4 +2,4 @@
 sh env_vars.sh
 source /tmp/.env.override
 
-uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload --proxy-headers
+uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --proxy-headers

ee/api/entrypoint_alerts.sh

@@ -2,4 +2,4 @@
 export ASSIST_KEY=ignore
 sh env_vars.sh
 source /tmp/.env.override
-uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload
+uvicorn app:app --host 0.0.0.0 --port 8888

ee/api/run-dev.sh

@@ -1,3 +0,0 @@
-#!/bin/zsh
-
-uvicorn app:app --reload