From a5bab0a438217ec253be430777b6bf8e36f4f38c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 17 Feb 2023 13:10:34 +0100 Subject: [PATCH] feat(alerts): fixed no-events-join builder --- api/app_alerts.py | 7 +++++++ api/chalicelib/core/alerts_processor.py | 19 ++++++++++--------- api/run-alerts-dev.sh | 3 +++ ee/api/chalicelib/core/alerts_processor.py | 16 +++++++++------- .../chalicelib/core/alerts_processor_exp.py | 10 ++++++++-- 5 files changed, 37 insertions(+), 18 deletions(-) create mode 100755 api/run-alerts-dev.sh diff --git a/api/app_alerts.py b/api/app_alerts.py index 7107423de..111bad2a1 100644 --- a/api/app_alerts.py +++ b/api/app_alerts.py @@ -53,3 +53,10 @@ async def stop_server(): await shutdown() import os, signal os.kill(1, signal.SIGTERM) + + +if config("LOCAL_DEV", default=False, cast=bool): + @app.get('/private/trigger', tags=["private"]) + async def trigger_main_cron(): + logging.info("Triggering main cron") + alerts_processor.process() diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index 8049b2f39..4babe64ce 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -49,10 +49,12 @@ LeftToDb = { schemas.AlertColumn.errors__4xx_5xx__count: { "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(session_id)", "condition": "status/100!=2"}, - schemas.AlertColumn.errors__4xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=4"}, - schemas.AlertColumn.errors__5xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=5"}, + schemas.AlertColumn.errors__4xx__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=4"}, + schemas.AlertColumn.errors__5xx__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=5"}, schemas.AlertColumn.errors__javascript__impacted_sessions__count: { "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(DISTINCT session_id)", "condition": "success= FALSE AND type='script'"}, @@ -95,7 +97,7 @@ def can_check(a) -> bool: a["options"].get("lastNotification") is None or a["options"]["lastNotification"] <= 0 or ((now - a["options"]["lastNotification"]) > a["options"]["renotifyInterval"] * 60 * 1000)) \ - and ((now - a["createdAt"]) % (TimeInterval[repetitionBase] * 60 * 1000)) < 60 * 1000 + and ((now - a["createdAt"]) % (TimeInterval[repetitionBase] * 60 * 1000)) < 60 * 1000 def Build(a): @@ -119,7 +121,7 @@ def Build(a): subQ = f"""SELECT {colDef["formula"]} AS value FROM {colDef["table"]} WHERE project_id = %(project_id)s - {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" + {"AND " + colDef["condition"] if colDef.get("condition") else ""}""" j_s = colDef.get("joinSessions", True) main_table = colDef["table"] is_ss = main_table == "public.sessions" @@ -142,8 +144,7 @@ def Build(a): "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, "timestamp_sub2": TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000} else: - sub1 = f"""{subQ} AND timestamp>=%(startDate)s - AND timestamp<=%(now)s + sub1 = f"""{subQ} {"AND timestamp >= %(startDate)s AND timestamp <= %(now)s" if not is_ss else ""} {"AND start_ts >= %(startDate)s AND start_ts <= %(now)s" if j_s else ""}""" params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 sub2 = f"""{subQ} {"AND timestamp < %(startDate)s AND timestamp >= %(timestamp_sub2)s" if not is_ss else ""} @@ -206,7 +207,7 @@ def process(): cur = cur.recreate(rollback=True) if len(notifications) > 0: cur.execute( - cur.mogrify(f"""UPDATE public.Alerts + cur.mogrify(f"""UPDATE public.alerts SET options = options||'{{"lastNotification":{TimeUTC.now()}}}'::jsonb WHERE alert_id IN %(ids)s;""", {"ids": tuple([n["alertId"] for n in notifications])})) if len(notifications) > 0: diff --git a/api/run-alerts-dev.sh b/api/run-alerts-dev.sh new file mode 100755 index 000000000..54db30171 --- /dev/null +++ b/api/run-alerts-dev.sh @@ -0,0 +1,3 @@ +#!/bin/zsh + +uvicorn app_alerts:app --reload \ No newline at end of file diff --git a/ee/api/chalicelib/core/alerts_processor.py b/ee/api/chalicelib/core/alerts_processor.py index 69a0f7f5f..06663336c 100644 --- a/ee/api/chalicelib/core/alerts_processor.py +++ b/ee/api/chalicelib/core/alerts_processor.py @@ -54,10 +54,12 @@ LeftToDb = { schemas.AlertColumn.errors__4xx_5xx__count: { "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(session_id)", "condition": "status/100!=2"}, - schemas.AlertColumn.errors__4xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=4"}, - schemas.AlertColumn.errors__5xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=5"}, + schemas.AlertColumn.errors__4xx__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=4"}, + schemas.AlertColumn.errors__5xx__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=5"}, schemas.AlertColumn.errors__javascript__impacted_sessions__count: { "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(DISTINCT session_id)", "condition": "success= FALSE AND type='script'"}, @@ -100,7 +102,7 @@ def can_check(a) -> bool: a["options"].get("lastNotification") is None or a["options"]["lastNotification"] <= 0 or ((now - a["options"]["lastNotification"]) > a["options"]["renotifyInterval"] * 60 * 1000)) \ - and ((now - a["createdAt"]) % (TimeInterval[repetitionBase] * 60 * 1000)) < 60 * 1000 + and ((now - a["createdAt"]) % (TimeInterval[repetitionBase] * 60 * 1000)) < 60 * 1000 def Build(a): @@ -124,7 +126,7 @@ def Build(a): subQ = f"""SELECT {colDef["formula"]} AS value FROM {colDef["table"]} WHERE project_id = %(project_id)s - {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" + {"AND " + colDef["condition"] if colDef.get("condition") else ""}""" j_s = colDef.get("joinSessions", True) main_table = colDef["table"] is_ss = main_table == "public.sessions" @@ -211,7 +213,7 @@ def process(): cur = cur.recreate(rollback=True) if len(notifications) > 0: cur.execute( - cur.mogrify(f"""UPDATE public.Alerts + cur.mogrify(f"""UPDATE public.alerts SET options = options||'{{"lastNotification":{TimeUTC.now()}}}'::jsonb WHERE alert_id IN %(ids)s;""", {"ids": tuple([n["alertId"] for n in notifications])})) if len(notifications) > 0: diff --git a/ee/api/chalicelib/core/alerts_processor_exp.py b/ee/api/chalicelib/core/alerts_processor_exp.py index 7a300654c..0d8b7753c 100644 --- a/ee/api/chalicelib/core/alerts_processor_exp.py +++ b/ee/api/chalicelib/core/alerts_processor_exp.py @@ -135,7 +135,7 @@ def Build(a): FROM {colDef["table"](now)} WHERE project_id = %(project_id)s {"AND event_type=%(event_type)s" if params["event_type"] else ""} - {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" + {"AND " + colDef["condition"] if colDef.get("condition") else ""}""" q = f"""SELECT coalesce(value,0) AS value, coalesce(value,0) {a["query"]["operator"]} {a["query"]["right"]} AS valid""" @@ -200,7 +200,13 @@ def process(): if alerts_processor.can_check(alert): logging.info(f"Querying alertId:{alert['alertId']} name: {alert['name']}") query, params = Build(alert) - query = ch_cur.format(query, params) + try: + query = ch_cur.format(query, params) + except Exception as e: + logging.error( + f"!!!Error while building alert query for alertId:{alert['alertId']} name: {alert['name']}") + logging.error(e) + continue logging.debug(alert) logging.debug(query) try: