From aa0a24f56acc63fe73a3031908e305ae7fa3f231 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 26 Aug 2022 16:27:32 +0100 Subject: [PATCH] feat(alerts): CH search --- ee/api/.gitignore | 1 + ee/api/chalicelib/core/__init__.py | 6 + .../chalicelib/core/alerts_processor_exp.py | 208 ++++++++++++++++++ ee/api/clean.sh | 1 + ee/api/env.default | 3 +- 5 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 ee/api/chalicelib/core/alerts_processor_exp.py diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 59d7202a9..1afe8462f 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -184,6 +184,7 @@ Pipfile /chalicelib/core/announcements.py /chalicelib/core/autocomplete.py /chalicelib/core/collaboration_slack.py +/chalicelib/core/countries.py /chalicelib/core/errors.py /chalicelib/core/errors_favorite.py /chalicelib/core/events.py diff --git a/ee/api/chalicelib/core/__init__.py b/ee/api/chalicelib/core/__init__.py index ffccbb1a4..369d5f476 100644 --- a/ee/api/chalicelib/core/__init__.py +++ b/ee/api/chalicelib/core/__init__.py @@ -26,3 +26,9 @@ if config("EXP_METRICS", cast=bool, default=False): from . import metrics_exp as metrics else: from . import metrics as metrics + +if config("EXP_ALERTS", cast=bool, default=False): + print(">>> Using experimental alerts") + from . import alerts_processor_exp as alerts_processor +else: + from . import alerts_processor as alerts_processor diff --git a/ee/api/chalicelib/core/alerts_processor_exp.py b/ee/api/chalicelib/core/alerts_processor_exp.py new file mode 100644 index 000000000..afa453c2e --- /dev/null +++ b/ee/api/chalicelib/core/alerts_processor_exp.py @@ -0,0 +1,208 @@ +import decimal +import logging + +import schemas +from chalicelib.core import alerts_listener, alerts_processor +from chalicelib.core import sessions, alerts +from chalicelib.utils import pg_client, ch_client +from chalicelib.utils.TimeUTC import TimeUTC + +LeftToDb = { + schemas.AlertColumn.performance__dom_content_loaded__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "COALESCE(AVG(NULLIF(dom_content_loaded_time ,0)),0)"}, + schemas.AlertColumn.performance__first_meaningful_paint__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "COALESCE(AVG(NULLIF(first_contentful_paint_time,0)),0)"}, + schemas.AlertColumn.performance__page_load_time__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", "formula": "AVG(NULLIF(load_time ,0))"}, + schemas.AlertColumn.performance__dom_build_time__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(dom_building_time,0))"}, + schemas.AlertColumn.performance__speed_index__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", "formula": "AVG(NULLIF(speed_index,0))"}, + schemas.AlertColumn.performance__page_response_time__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(response_time,0))"}, + schemas.AlertColumn.performance__ttfb__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(first_paint_time,0))"}, + schemas.AlertColumn.performance__time_to_render__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(visually_complete,0))"}, + schemas.AlertColumn.performance__image_load_time__average: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='img'"}, + schemas.AlertColumn.performance__request_load_time__average: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='fetch'"}, + schemas.AlertColumn.resources__load_time__average: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(resources.duration,0))"}, + schemas.AlertColumn.resources__missing__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(DISTINCT url_hostpath)", "condition": "success= FALSE"}, + schemas.AlertColumn.errors__4xx_5xx__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(session_id)", + "condition": "status/100!=2"}, + schemas.AlertColumn.errors__4xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=4"}, + schemas.AlertColumn.errors__5xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=5"}, + schemas.AlertColumn.errors__javascript__impacted_sessions__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(DISTINCT session_id)", "condition": "success= FALSE AND type='script'"}, + schemas.AlertColumn.performance__crashes__count: { + "table": "(SELECT *, start_ts AS timestamp FROM public.sessions WHERE errors_count > 0) AS sessions", + "formula": "COUNT(DISTINCT session_id)", "condition": "errors_count > 0"}, + schemas.AlertColumn.errors__javascript__count: { + "table": "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", + "formula": "COUNT(DISTINCT session_id)", "condition": "source='js_exception'", "joinSessions": False}, + schemas.AlertColumn.errors__backend__count: { + "table": "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", + "formula": "COUNT(DISTINCT session_id)", "condition": "source!='js_exception'", "joinSessions": False}, +} + +# This is the frequency of execution for each threshold +TimeInterval = { + 15: 3, + 30: 5, + 60: 10, + 120: 20, + 240: 30, + 1440: 60, +} + + +def Build(a): + params = {"project_id": a["projectId"], "now": TimeUTC.now()} + full_args = {} + j_s = True + if a["seriesId"] is not None: + a["filter"]["sort"] = "session_id" + a["filter"]["order"] = schemas.SortOrderType.desc + a["filter"]["startDate"] = -1 + a["filter"]["endDate"] = TimeUTC.now() + full_args, query_part = sessions.search_query_parts( + data=schemas.SessionsSearchPayloadSchema.parse_obj(a["filter"]), error_status=None, errors_only=False, + issue=None, project_id=a["projectId"], user_id=None, favorite_only=False) + subQ = f"""SELECT COUNT(session_id) AS value + {query_part}""" + else: + colDef = LeftToDb[a["query"]["left"]] + subQ = f"""SELECT {colDef["formula"]} AS value + FROM {colDef["table"]} + WHERE project_id = %(project_id)s + {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" + j_s = colDef.get("joinSessions", True) + + q = f"""SELECT coalesce(value,0) AS value, coalesce(value,0) {a["query"]["operator"]} {a["query"]["right"]} AS valid""" + + if a["detectionMethod"] == schemas.AlertDetectionMethod.threshold: + if a["seriesId"] is not None: + q += f""" FROM ({subQ}) AS stat""" + else: + q += f""" FROM ({subQ} AND timestamp>=%(startDate)s AND timestamp<=%(now)s + {"AND sessions.start_ts >= %(startDate)s AND sessions.start_ts <= %(now)s" if j_s else ""}) AS stat""" + params = {**params, **full_args, "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000} + else: + if a["options"]["change"] == schemas.AlertDetectionChangeType.change: + if a["seriesId"] is not None: + sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") + sub1 = f"SELECT (({subQ})-({sub2})) AS value" + q += f" FROM ( {sub1} ) AS stat" + params = {**params, **full_args, + "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, + "timestamp_sub2": TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000} + else: + sub1 = f"""{subQ} AND timestamp>=%(startDate)s + {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}""" + params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 + sub2 = f"""{subQ} AND timestamp<%(startDate)s + AND timestamp>=%(timestamp_sub2)s + {"AND sessions.start_ts < %(startDate)s AND sessions.start_ts >= %(timestamp_sub2)s" if j_s else ""}""" + params["timestamp_sub2"] = TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000 + sub1 = f"SELECT (( {sub1} )-( {sub2} )) AS value" + q += f" FROM ( {sub1} ) AS stat" + + else: + if a["seriesId"] is not None: + sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") + sub1 = f"SELECT (({subQ})/NULLIF(({sub2}),0)-1)*100 AS value" + q += f" FROM ({sub1}) AS stat" + params = {**params, **full_args, + "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, + "timestamp_sub2": TimeUTC.now() \ + - (a["options"]["currentPeriod"] + a["options"]["currentPeriod"]) \ + * 60 * 1000} + else: + sub1 = f"""{subQ} AND timestamp>=%(startDate)s + {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}""" + params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 + sub2 = f"""{subQ} AND timestamp<%(startDate)s + AND timestamp>=%(timestamp_sub2)s + {"AND sessions.start_ts < %(startDate)s AND sessions.start_ts >= %(timestamp_sub2)s" if j_s else ""}""" + params["timestamp_sub2"] = TimeUTC.now() \ + - (a["options"]["currentPeriod"] + a["options"]["currentPeriod"]) * 60 * 1000 + sub1 = f"SELECT (({sub1})/NULLIF(({sub2}),0)-1)*100 AS value" + q += f" FROM ({sub1}) AS stat" + + return q, params + + +def process(): + notifications = [] + all_alerts = alerts_listener.get_all_alerts() + with pg_client.PostgresClient() as cur, ch_client.ClickHouseClient() as curc: + for alert in all_alerts: + if True or alerts_processor.can_check(alert): + logging.info(f"Querying alertId:{alert['alertId']} name: {alert['name']}") + query, params = Build(alert) + query = cur.mogrify(query, params) + logging.debug(alert) + logging.debug(query) + try: + print("------------------Alerts") + print(alert) + print(query) + print("------------------") + continue + cur.execute(query) + result = cur.fetchone() + if result["valid"]: + logging.info("Valid alert, notifying users") + notifications.append({ + "alertId": alert["alertId"], + "tenantId": alert["tenantId"], + "title": alert["name"], + "description": f"has been triggered, {alert['query']['left']} = {round(result['value'], 2)} ({alert['query']['operator']} {alert['query']['right']}).", + "buttonText": "Check metrics for more details", + "buttonUrl": f"/{alert['projectId']}/metrics", + "imageUrl": None, + "options": {"source": "ALERT", "sourceId": alert["alertId"], + "sourceMeta": alert["detectionMethod"], + "message": alert["options"]["message"], "projectId": alert["projectId"], + "data": {"title": alert["name"], + "limitValue": alert["query"]["right"], + "actualValue": float(result["value"]) \ + if isinstance(result["value"], decimal.Decimal) \ + else result["value"], + "operator": alert["query"]["operator"], + "trigger": alert["query"]["left"], + "alertId": alert["alertId"], + "detectionMethod": alert["detectionMethod"], + "currentPeriod": alert["options"]["currentPeriod"], + "previousPeriod": alert["options"]["previousPeriod"], + "createdAt": TimeUTC.now()}}, + }) + except Exception as e: + logging.error(f"!!!Error while running alert query for alertId:{alert['alertId']}") + logging.error(str(e)) + logging.error(query) + if len(notifications) > 0: + cur.execute( + cur.mogrify(f"""UPDATE public.alerts + SET options = options||'{{"lastNotification":{TimeUTC.now()}}}'::jsonb + WHERE alert_id IN %(ids)s;""", {"ids": tuple([n["alertId"] for n in notifications])})) + if len(notifications) > 0: + alerts.process_notifications(notifications) diff --git a/ee/api/clean.sh b/ee/api/clean.sh index 0974875cf..b05ce1ee4 100755 --- a/ee/api/clean.sh +++ b/ee/api/clean.sh @@ -5,6 +5,7 @@ rm -rf ./chalicelib/core/alerts_processor.py rm -rf ./chalicelib/core/announcements.py rm -rf ./chalicelib/core/autocomplete.py rm -rf ./chalicelib/core/collaboration_slack.py +rm -rf ./chalicelib/core/countries.py rm -rf ./chalicelib/core/errors.py rm -rf ./chalicelib/core/errors_favorite.py rm -rf ./chalicelib/core/events.py diff --git a/ee/api/env.default b/ee/api/env.default index 13976e436..385dd254f 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -65,4 +65,5 @@ EXP_SESSIONS_SEARCH=true EXP_AUTOCOMPLETE=true EXP_ERRORS_SEARCH=true EXP_METRICS=true -EXP_7D_MV=true \ No newline at end of file +EXP_7D_MV=true +EXP_ALERTS=true \ No newline at end of file