diff --git a/api/chalicelib/core/alerts.py b/api/chalicelib/core/alerts.py index f851751ba..da6211687 100644 --- a/api/chalicelib/core/alerts.py +++ b/api/chalicelib/core/alerts.py @@ -52,8 +52,8 @@ def create(project_id, data: schemas.AlertSchema): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify("""\ - INSERT INTO public.alerts(project_id, name, description, detection_method, query, options, series_id) - VALUES (%(project_id)s, %(name)s, %(description)s, %(detection_method)s, %(query)s, %(options)s::jsonb, %(series_id)s) + INSERT INTO public.alerts(project_id, name, description, detection_method, query, options, series_id, change) + VALUES (%(project_id)s, %(name)s, %(description)s, %(detection_method)s, %(query)s, %(options)s::jsonb, %(series_id)s, %(change)s) RETURNING *;""", {"project_id": project_id, **data}) ) @@ -75,7 +75,8 @@ def update(id, data: schemas.AlertSchema): detection_method = %(detection_method)s, query = %(query)s, options = %(options)s, - series_id = %(series_id)s + series_id = %(series_id)s, + change = %(change)s WHERE alert_id =%(id)s AND deleted_at ISNULL RETURNING *;""", {"id": id, **data}) diff --git a/api/chalicelib/core/alerts_listener.py b/api/chalicelib/core/alerts_listener.py index 419f0326d..0fa193964 100644 --- a/api/chalicelib/core/alerts_listener.py +++ b/api/chalicelib/core/alerts_listener.py @@ -12,7 +12,8 @@ def get_all_alerts(): (EXTRACT(EPOCH FROM alerts.created_at) * 1000)::BIGINT AS created_at, alerts.name, alerts.series_id, - filter + filter, + change FROM public.alerts LEFT JOIN metric_series USING (series_id) INNER JOIN projects USING (project_id) diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index ece75bfe5..dbc4aaf41 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -1,12 +1,16 @@ import decimal import logging +from decouple import config + import schemas from chalicelib.core import alerts_listener from chalicelib.core import sessions, alerts from chalicelib.utils import pg_client from chalicelib.utils.TimeUTC import TimeUTC +logging.basicConfig(level=config("LOGLEVEL", default=logging.INFO)) + LeftToDb = { schemas.AlertColumn.performance__dom_content_loaded__average: { "table": "events.pages INNER JOIN public.sessions USING(session_id)", @@ -41,7 +45,7 @@ LeftToDb = { "formula": "AVG(NULLIF(resources.duration,0))"}, schemas.AlertColumn.resources__missing__count: { "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(DISTINCT url_hostpath)", "condition": "success= FALSE"}, + "formula": "COUNT(DISTINCT url_hostpath)", "condition": "success= FALSE AND type='img'"}, schemas.AlertColumn.errors__4xx_5xx__count: { "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(session_id)", "condition": "status/100!=2"}, @@ -53,8 +57,9 @@ LeftToDb = { "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(DISTINCT session_id)", "condition": "success= FALSE AND type='script'"}, schemas.AlertColumn.performance__crashes__count: { - "table": "(SELECT *, start_ts AS timestamp FROM public.sessions WHERE errors_count > 0) AS sessions", - "formula": "COUNT(DISTINCT session_id)", "condition": "errors_count > 0"}, + "table": "public.sessions", + "formula": "COUNT(DISTINCT session_id)", + "condition": "errors_count > 0 AND duration>0"}, schemas.AlertColumn.errors__javascript__count: { "table": "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", "formula": "COUNT(DISTINCT session_id)", "condition": "source='js_exception'", "joinSessions": False}, @@ -94,7 +99,8 @@ def can_check(a) -> bool: def Build(a): - params = {"project_id": a["projectId"]} + now = TimeUTC.now() + params = {"project_id": a["projectId"], "now": now} full_args = {} j_s = True if a["seriesId"] is not None: @@ -121,11 +127,12 @@ def Build(a): if a["seriesId"] is not None: q += f""" FROM ({subQ}) AS stat""" else: - q += f""" FROM ({subQ} AND timestamp>=%(startDate)s - {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}) AS stat""" + q += f""" FROM ({subQ} AND timestamp>=%(startDate)s AND timestamp<=%(now)s + {"AND sessions.start_ts >= %(startDate)s" if j_s else ""} + {"AND sessions.start_ts <= %(now)s" if j_s else ""}) AS stat""" params = {**params, **full_args, "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000} else: - if a["options"]["change"] == schemas.AlertDetectionChangeType.change: + if a["change"] == schemas.AlertDetectionType.change: if a["seriesId"] is not None: sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") sub1 = f"SELECT (({subQ})-({sub2})) AS value" @@ -135,7 +142,9 @@ def Build(a): "timestamp_sub2": TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000} else: sub1 = f"""{subQ} AND timestamp>=%(startDate)s - {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}""" + AND datetime<=toDateTime(%(now)s/1000) + {"AND sessions.start_ts >= %(startDate)s" if j_s else ""} + {"AND sessions.start_ts <= %(now)s" if j_s else ""}""" params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 sub2 = f"""{subQ} AND timestamp<%(startDate)s AND timestamp>=%(timestamp_sub2)s @@ -155,8 +164,9 @@ def Build(a): - (a["options"]["currentPeriod"] + a["options"]["currentPeriod"]) \ * 60 * 1000} else: - sub1 = f"""{subQ} AND timestamp>=%(startDate)s - {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}""" + sub1 = f"""{subQ} AND timestamp>=%(startDate)s AND timestamp<=%(now)s + {"AND sessions.start_ts >= %(startDate)s" if j_s else ""} + {"AND sessions.start_ts <= %(now)s" if j_s else ""}""" params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 sub2 = f"""{subQ} AND timestamp<%(startDate)s AND timestamp>=%(timestamp_sub2)s @@ -185,30 +195,7 @@ def process(): result = cur.fetchone() if result["valid"]: logging.info("Valid alert, notifying users") - notifications.append({ - "alertId": alert["alertId"], - "tenantId": alert["tenantId"], - "title": alert["name"], - "description": f"has been triggered, {alert['query']['left']} = {round(result['value'], 2)} ({alert['query']['operator']} {alert['query']['right']}).", - "buttonText": "Check metrics for more details", - "buttonUrl": f"/{alert['projectId']}/metrics", - "imageUrl": None, - "options": {"source": "ALERT", "sourceId": alert["alertId"], - "sourceMeta": alert["detectionMethod"], - "message": alert["options"]["message"], "projectId": alert["projectId"], - "data": {"title": alert["name"], - "limitValue": alert["query"]["right"], - "actualValue": float(result["value"]) \ - if isinstance(result["value"], decimal.Decimal) \ - else result["value"], - "operator": alert["query"]["operator"], - "trigger": alert["query"]["left"], - "alertId": alert["alertId"], - "detectionMethod": alert["detectionMethod"], - "currentPeriod": alert["options"]["currentPeriod"], - "previousPeriod": alert["options"]["previousPeriod"], - "createdAt": TimeUTC.now()}}, - }) + notifications.append(generate_notification(alert, result)) except Exception as e: logging.error(f"!!!Error while running alert query for alertId:{alert['alertId']}") logging.error(str(e)) @@ -220,3 +207,30 @@ def process(): WHERE alert_id IN %(ids)s;""", {"ids": tuple([n["alertId"] for n in notifications])})) if len(notifications) > 0: alerts.process_notifications(notifications) + + +def generate_notification(alert, result): + return { + "alertId": alert["alertId"], + "tenantId": alert["tenantId"], + "title": alert["name"], + "description": f"has been triggered, {alert['query']['left']} = {round(result['value'], 2)} ({alert['query']['operator']} {alert['query']['right']}).", + "buttonText": "Check metrics for more details", + "buttonUrl": f"/{alert['projectId']}/metrics", + "imageUrl": None, + "options": {"source": "ALERT", "sourceId": alert["alertId"], + "sourceMeta": alert["detectionMethod"], + "message": alert["options"]["message"], "projectId": alert["projectId"], + "data": {"title": alert["name"], + "limitValue": alert["query"]["right"], + "actualValue": float(result["value"]) \ + if isinstance(result["value"], decimal.Decimal) \ + else result["value"], + "operator": alert["query"]["operator"], + "trigger": alert["query"]["left"], + "alertId": alert["alertId"], + "detectionMethod": alert["detectionMethod"], + "currentPeriod": alert["options"]["currentPeriod"], + "previousPeriod": alert["options"]["previousPeriod"], + "createdAt": TimeUTC.now()}}, + } diff --git a/api/schemas.py b/api/schemas.py index 591d8e905..42787c9e5 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -279,7 +279,7 @@ class _AlertMessageSchema(BaseModel): value: str = Field(...) -class AlertDetectionChangeType(str, Enum): +class AlertDetectionType(str, Enum): percent = "percent" change = "change" @@ -290,7 +290,6 @@ class _AlertOptionSchema(BaseModel): previousPeriod: Literal[15, 30, 60, 120, 240, 1440] = Field(15) lastNotification: Optional[int] = Field(None) renotifyInterval: Optional[int] = Field(720) - change: Optional[AlertDetectionChangeType] = Field(None) class AlertColumn(str, Enum): @@ -339,6 +338,7 @@ class AlertDetectionMethod(str, Enum): class AlertSchema(BaseModel): name: str = Field(...) detection_method: AlertDetectionMethod = Field(...) + change: Optional[AlertDetectionType] = Field(default=AlertDetectionType.change) description: Optional[str] = Field(None) options: _AlertOptionSchema = Field(...) query: _AlertQuerySchema = Field(...) @@ -356,11 +356,6 @@ class AlertSchema(BaseModel): def alert_validator(cls, values): if values.get("query") is not None and values["query"].left == AlertColumn.custom: assert values.get("series_id") is not None, "series_id should not be null for CUSTOM alert" - if values.get("detectionMethod") is not None \ - and values["detectionMethod"] == AlertDetectionMethod.change \ - and values.get("options") is not None: - assert values["options"].change is not None, \ - "options.change should not be null for detection method 'change'" return values class Config: diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 59d7202a9..1afe8462f 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -184,6 +184,7 @@ Pipfile /chalicelib/core/announcements.py /chalicelib/core/autocomplete.py /chalicelib/core/collaboration_slack.py +/chalicelib/core/countries.py /chalicelib/core/errors.py /chalicelib/core/errors_favorite.py /chalicelib/core/events.py diff --git a/ee/api/chalicelib/core/__init__.py b/ee/api/chalicelib/core/__init__.py index ffccbb1a4..369d5f476 100644 --- a/ee/api/chalicelib/core/__init__.py +++ b/ee/api/chalicelib/core/__init__.py @@ -26,3 +26,9 @@ if config("EXP_METRICS", cast=bool, default=False): from . import metrics_exp as metrics else: from . import metrics as metrics + +if config("EXP_ALERTS", cast=bool, default=False): + print(">>> Using experimental alerts") + from . import alerts_processor_exp as alerts_processor +else: + from . import alerts_processor as alerts_processor diff --git a/ee/api/chalicelib/core/alerts_listener.py b/ee/api/chalicelib/core/alerts_listener.py index 40241f51e..6a97daf93 100644 --- a/ee/api/chalicelib/core/alerts_listener.py +++ b/ee/api/chalicelib/core/alerts_listener.py @@ -12,7 +12,8 @@ def get_all_alerts(): (EXTRACT(EPOCH FROM alerts.created_at) * 1000)::BIGINT AS created_at, alerts.name, alerts.series_id, - filter + filter, + change FROM public.alerts LEFT JOIN metric_series USING (series_id) INNER JOIN projects USING (project_id) diff --git a/ee/api/chalicelib/core/alerts_processor_exp.py b/ee/api/chalicelib/core/alerts_processor_exp.py new file mode 100644 index 000000000..66448b1d9 --- /dev/null +++ b/ee/api/chalicelib/core/alerts_processor_exp.py @@ -0,0 +1,224 @@ +import logging + +from decouple import config + +import schemas +from chalicelib.core import alerts_listener, alerts_processor +from chalicelib.core import sessions, alerts +from chalicelib.utils import pg_client, ch_client, exp_ch_helper +from chalicelib.utils.TimeUTC import TimeUTC + +logging.basicConfig(level=config("LOGLEVEL", default=logging.INFO)) + +LeftToDb = { + schemas.AlertColumn.performance__dom_content_loaded__average: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS pages", + "formula": "COALESCE(AVG(NULLIF(dom_content_loaded_event_time ,0)),0)", + "eventType": "LOCATION" + }, + schemas.AlertColumn.performance__first_meaningful_paint__average: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS pages", + "formula": "COALESCE(AVG(NULLIF(first_contentful_paint_time,0)),0)", + "eventType": "LOCATION" + }, + schemas.AlertColumn.performance__page_load_time__average: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS pages", + "formula": "AVG(NULLIF(load_event_time ,0))", + "eventType": "LOCATION" + }, + schemas.AlertColumn.performance__dom_build_time__average: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS pages", + "formula": "AVG(NULLIF(dom_building_time,0))", + "eventType": "LOCATION" + }, + schemas.AlertColumn.performance__speed_index__average: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS pages", + "formula": "AVG(NULLIF(speed_index,0))", + "eventType": "LOCATION" + }, + schemas.AlertColumn.performance__page_response_time__average: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS pages", + "formula": "AVG(NULLIF(response_time,0))", + "eventType": "LOCATION" + }, + schemas.AlertColumn.performance__ttfb__average: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS pages", + "formula": "AVG(NULLIF(first_contentful_paint_time,0))", + "eventType": "LOCATION" + }, + schemas.AlertColumn.performance__time_to_render__average: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS pages", + "formula": "AVG(NULLIF(visually_complete,0))", + "eventType": "LOCATION" + }, + schemas.AlertColumn.performance__image_load_time__average: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_resources_table(timestamp)} AS resources", + "formula": "AVG(NULLIF(resources.duration,0))", + "condition": "type='img'" + }, + schemas.AlertColumn.performance__request_load_time__average: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_resources_table(timestamp)} AS resources", + "formula": "AVG(NULLIF(resources.duration,0))", + "condition": "type='fetch'" + }, + schemas.AlertColumn.resources__load_time__average: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_resources_table(timestamp)} AS resources", + "formula": "AVG(NULLIF(resources.duration,0))" + }, + schemas.AlertColumn.resources__missing__count: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_resources_table(timestamp)} AS resources", + "formula": "COUNT(DISTINCT url_hostpath)", + "condition": "success= FALSE AND type='img'" + }, + schemas.AlertColumn.errors__4xx_5xx__count: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS requests", + "eventType": "REQUEST", + "formula": "COUNT(1)", + "condition": "intDiv(requests.status, 100)!=2" + }, + schemas.AlertColumn.errors__4xx__count: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS requests", + "eventType": "REQUEST", + "formula": "COUNT(1)", + "condition": "intDiv(requests.status, 100)==4" + }, + schemas.AlertColumn.errors__5xx__count: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS requests", + "eventType": "REQUEST", + "formula": "COUNT(1)", + "condition": "intDiv(requests.status, 100)==5" + }, + schemas.AlertColumn.errors__javascript__impacted_sessions__count: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS errors", + "eventType": "ERROR", + "formula": "COUNT(DISTINCT session_id)", + "condition": "source='js_exception'" + }, + schemas.AlertColumn.performance__crashes__count: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_sessions_table(timestamp)} AS sessions", + "formula": "COUNT(DISTINCT session_id)", + "condition": "duration>0 AND errors_count>0" + }, + schemas.AlertColumn.errors__javascript__count: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS errors", + "eventType": "ERROR", + "formula": "COUNT(DISTINCT session_id)", + "condition": "source='js_exception'" + }, + schemas.AlertColumn.errors__backend__count: { + "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS errors", + "eventType": "ERROR", + "formula": "COUNT(DISTINCT session_id)", + "condition": "source!='js_exception'" + }, +} + + +def Build(a): + now = TimeUTC.now() + params = {"project_id": a["projectId"], "now": now} + full_args = {} + if a["seriesId"] is not None: + a["filter"]["sort"] = "session_id" + a["filter"]["order"] = schemas.SortOrderType.desc + a["filter"]["startDate"] = -1 + a["filter"]["endDate"] = TimeUTC.now() + full_args, query_part = sessions.search_query_parts_ch( + data=schemas.SessionsSearchPayloadSchema.parse_obj(a["filter"]), error_status=None, errors_only=False, + issue=None, project_id=a["projectId"], user_id=None, favorite_only=False) + subQ = f"""SELECT COUNT(session_id) AS value + {query_part}""" + else: + colDef = LeftToDb[a["query"]["left"]] + params["event_type"] = LeftToDb[a["query"]["left"]].get("eventType") + subQ = f"""SELECT {colDef["formula"]} AS value + FROM {colDef["table"](now)} + WHERE project_id = %(project_id)s + {"AND event_type=%(event_type)s" if params["event_type"] else ""} + {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" + + q = f"""SELECT coalesce(value,0) AS value, coalesce(value,0) {a["query"]["operator"]} {a["query"]["right"]} AS valid""" + + if a["detectionMethod"] == schemas.AlertDetectionMethod.threshold: + if a["seriesId"] is not None: + q += f""" FROM ({subQ}) AS stat""" + else: + q += f""" FROM ({subQ} + AND datetime>=toDateTime(%(startDate)s/1000) + AND datetime<=toDateTime(%(now)s/1000) ) AS stat""" + params = {**params, **full_args, "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000} + else: + if a["change"] == schemas.AlertDetectionType.change: + if a["seriesId"] is not None: + sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") + sub1 = f"SELECT (({subQ})-({sub2})) AS value" + q += f" FROM ( {sub1} ) AS stat" + params = {**params, **full_args, + "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, + "timestamp_sub2": TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000} + else: + sub1 = f"""{subQ} AND datetime>=toDateTime(%(startDate)s/1000) + AND datetime<=toDateTime(%(now)s/1000)""" + params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 + sub2 = f"""{subQ} AND datetime=toDateTime(%(timestamp_sub2)s/1000)""" + params["timestamp_sub2"] = TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000 + sub1 = f"SELECT (( {sub1} )-( {sub2} )) AS value" + q += f" FROM ( {sub1} ) AS stat" + + else: + if a["seriesId"] is not None: + sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") + sub1 = f"SELECT (({subQ})/NULLIF(({sub2}),0)-1)*100 AS value" + q += f" FROM ({sub1}) AS stat" + params = {**params, **full_args, + "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, + "timestamp_sub2": TimeUTC.now() \ + - (a["options"]["currentPeriod"] + a["options"]["currentPeriod"]) \ + * 60 * 1000} + else: + sub1 = f"""{subQ} AND datetime>=toDateTime(%(startDate)s/1000) + AND datetime<=toDateTime(%(now)s/1000)""" + params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 + sub2 = f"""{subQ} AND datetime=toDateTime(%(timestamp_sub2)s/1000)""" + params["timestamp_sub2"] = TimeUTC.now() \ + - (a["options"]["currentPeriod"] + a["options"]["currentPeriod"]) * 60 * 1000 + sub1 = f"SELECT (({sub1})/NULLIF(({sub2}),0)-1)*100 AS value" + q += f" FROM ({sub1}) AS stat" + + return q, params + + +def process(): + notifications = [] + all_alerts = alerts_listener.get_all_alerts() + with pg_client.PostgresClient() as cur, ch_client.ClickHouseClient() as ch_cur: + for alert in all_alerts: + if alert["query"]["left"] != "CUSTOM": + continue + if True or alerts_processor.can_check(alert): + logging.info(f"Querying alertId:{alert['alertId']} name: {alert['name']}") + query, params = Build(alert) + query = ch_cur.format(query, params) + logging.debug(alert) + logging.debug(query) + try: + result = ch_cur.execute(query) + if len(result) > 0: + result = result[0] + + if result["valid"]: + logging.info("Valid alert, notifying users") + notifications.append(alerts_processor.generate_notification(alert, result)) + except Exception as e: + logging.error(f"!!!Error while running alert query for alertId:{alert['alertId']}") + logging.error(str(e)) + logging.error(query) + if len(notifications) > 0: + cur.execute( + cur.mogrify(f"""UPDATE public.alerts + SET options = options||'{{"lastNotification":{TimeUTC.now()}}}'::jsonb + WHERE alert_id IN %(ids)s;""", {"ids": tuple([n["alertId"] for n in notifications])})) + if len(notifications) > 0: + alerts.process_notifications(notifications) diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index 5973a3a8e..2b78f46f2 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -361,9 +361,9 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ print("--------------------") main_query = cur.format(f"""SELECT DISTINCT er.error_id, COALESCE((SELECT TRUE - FROM final.user_viewed_errors AS ve - WHERE er.error_id = ve.error_id - AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + FROM {exp_ch_helper.get_user_viewed_errors_table()} AS ve + WHERE er.error_id = ve.error_id + AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed {query_part};""", full_args) elif count_only: @@ -1252,9 +1252,9 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, "isNotNull(s.duration)" ] if favorite_only: - extra_constraints.append("""s.session_id IN (SELECT session_id - FROM final.user_favorite_sessions - WHERE user_id = %(userId)s)""") + extra_constraints.append(f"""s.session_id IN (SELECT session_id + FROM {exp_ch_helper.get_user_favorite_sessions_table()} AS user_favorite_sessions + WHERE user_id = %(userId)s)""") extra_from = "" events_query_part = "" __events_where_basic = ["project_id = %(projectId)s", @@ -1526,7 +1526,7 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, "main.datetime >= toDateTime(%(startDate)s/1000)", "main.datetime <= toDateTime(%(endDate)s/1000)"] if favorite_only and not errors_only: - event_from += "INNER JOIN final.user_favorite_sessions AS fs USING(session_id)" + event_from += f"INNER JOIN {exp_ch_helper.get_user_favorite_sessions_table()} AS fs USING(session_id)" event_where.append("fs.user_id = %(userId)s") # else: # event_from = "%s" @@ -1940,9 +1940,9 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, else: events_extra_join = f"LEFT JOIN ({events_extra_join}) AS main1 USING(error_id)" if favorite_only and user_id is not None: - events_conditions_where.append("""main.session_id IN (SELECT session_id - FROM final.user_favorite_sessions - WHERE user_id = %(userId)s)""") + events_conditions_where.append(f"""main.session_id IN (SELECT session_id + FROM {exp_ch_helper.get_user_favorite_sessions_table()} AS user_favorite_sessions + WHERE user_id = %(userId)s)""") if data.events_order in [schemas.SearchEventOrder._then, schemas.SearchEventOrder._and]: sequence_pattern = [f'(?{i + 1}){c.get("time", "")}' for i, c in enumerate(events_conditions)] @@ -2067,8 +2067,8 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, extra_from += """INNER JOIN (SELECT 1 AS session_id) AS favorite_sessions ON (TRUE)""" elif not favorite_only and not errors_only and user_id is not None: - extra_from += """LEFT JOIN (SELECT session_id - FROM final.user_favorite_sessions + extra_from += f"""LEFT JOIN (SELECT session_id + FROM {exp_ch_helper.get_user_favorite_sessions_table()} AS user_favorite_sessions WHERE user_id = %(userId)s) AS favorite_sessions ON (s.session_id=favorite_sessions.session_id)""" extra_join = "" diff --git a/ee/api/chalicelib/utils/exp_ch_helper.py b/ee/api/chalicelib/utils/exp_ch_helper.py index fb6781184..709b5e926 100644 --- a/ee/api/chalicelib/utils/exp_ch_helper.py +++ b/ee/api/chalicelib/utils/exp_ch_helper.py @@ -21,7 +21,7 @@ def get_main_sessions_table(timestamp): def get_main_resources_table(timestamp): - return "experimental.resources_l7s_mv" \ + return "experimental.resources_l7d_mv" \ if config("EXP_7D_MV", cast=bool, default=True) \ and timestamp >= TimeUTC.now(delta_days=-7) else "experimental.resources" diff --git a/ee/api/clean.sh b/ee/api/clean.sh index 0974875cf..b05ce1ee4 100755 --- a/ee/api/clean.sh +++ b/ee/api/clean.sh @@ -5,6 +5,7 @@ rm -rf ./chalicelib/core/alerts_processor.py rm -rf ./chalicelib/core/announcements.py rm -rf ./chalicelib/core/autocomplete.py rm -rf ./chalicelib/core/collaboration_slack.py +rm -rf ./chalicelib/core/countries.py rm -rf ./chalicelib/core/errors.py rm -rf ./chalicelib/core/errors_favorite.py rm -rf ./chalicelib/core/events.py diff --git a/ee/api/env.default b/ee/api/env.default index 13976e436..385dd254f 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -65,4 +65,5 @@ EXP_SESSIONS_SEARCH=true EXP_AUTOCOMPLETE=true EXP_ERRORS_SEARCH=true EXP_METRICS=true -EXP_7D_MV=true \ No newline at end of file +EXP_7D_MV=true +EXP_ALERTS=true \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_events_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_events_mv.sql index 136aea1c5..9605c0c09 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_events_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_events_mv.sql @@ -1,10 +1,67 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv ENGINE = MergeTree - PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, session_id) - TTL datetime + INTERVAL 7 DAY + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, event_type, session_id) + TTL datetime + INTERVAL 7 DAY POPULATE AS -SELECT * +SELECT session_id, + project_id, + event_type, + datetime, + label, + hesitation_time, + name, + payload, + level, + source, + message, + error_id, + duration, + context, + container_type, + container_id, + container_name, + container_src, + url, + url_host, + url_path, + url_hostpath, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint_time, + speed_index, + visually_complete, + time_to_interactive, + ttfb, + ttlb, + response_time, + dom_building_time, + dom_content_loaded_event_time, + load_event_time, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + method, + status, + success, + request_body, + response_body, + _timestamp FROM experimental.events WHERE datetime >= now() - INTERVAL 7 DAY; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_resources_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_resources_mv.sql index a5a6fb403..839ffa4f7 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_resources_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_resources_mv.sql @@ -1,10 +1,26 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.resources_l7d_mv ENGINE = MergeTree PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, session_id) + ORDER BY (project_id, datetime, type, session_id) TTL datetime + INTERVAL 7 DAY POPULATE AS -SELECT * +SELECT session_id, + project_id, + datetime, + url, + url_host, + url_path, + url_hostpath, + type, + name, + duration, + ttfb, + header_size, + encoded_body_size, + decoded_body_size, + compression_ratio, + success, + _timestamp FROM experimental.resources WHERE datetime >= now() - INTERVAL 7 DAY; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_sessions_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_sessions_mv.sql index eea689374..6d0a2176c 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_sessions_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_sessions_mv.sql @@ -6,7 +6,41 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.sessions_l7d_mv SETTINGS index_granularity = 512 POPULATE AS -SELECT * +SELECT session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + pages_count, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign, + user_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + issue_types, + referrer, + base_referrer, + _timestamp FROM experimental.sessions WHERE datetime >= now() - INTERVAL 7 DAY AND isNotNull(duration) diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_events_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_events_mv.sql index 136aea1c5..9605c0c09 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_events_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_events_mv.sql @@ -1,10 +1,67 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv ENGINE = MergeTree - PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, session_id) - TTL datetime + INTERVAL 7 DAY + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, event_type, session_id) + TTL datetime + INTERVAL 7 DAY POPULATE AS -SELECT * +SELECT session_id, + project_id, + event_type, + datetime, + label, + hesitation_time, + name, + payload, + level, + source, + message, + error_id, + duration, + context, + container_type, + container_id, + container_name, + container_src, + url, + url_host, + url_path, + url_hostpath, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint_time, + speed_index, + visually_complete, + time_to_interactive, + ttfb, + ttlb, + response_time, + dom_building_time, + dom_content_loaded_event_time, + load_event_time, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + method, + status, + success, + request_body, + response_body, + _timestamp FROM experimental.events WHERE datetime >= now() - INTERVAL 7 DAY; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_resources_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_resources_mv.sql index a5a6fb403..839ffa4f7 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_resources_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_resources_mv.sql @@ -1,10 +1,26 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.resources_l7d_mv ENGINE = MergeTree PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, session_id) + ORDER BY (project_id, datetime, type, session_id) TTL datetime + INTERVAL 7 DAY POPULATE AS -SELECT * +SELECT session_id, + project_id, + datetime, + url, + url_host, + url_path, + url_hostpath, + type, + name, + duration, + ttfb, + header_size, + encoded_body_size, + decoded_body_size, + compression_ratio, + success, + _timestamp FROM experimental.resources WHERE datetime >= now() - INTERVAL 7 DAY; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_sessions_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_sessions_mv.sql index eea689374..6d0a2176c 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_sessions_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_sessions_mv.sql @@ -6,7 +6,41 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.sessions_l7d_mv SETTINGS index_granularity = 512 POPULATE AS -SELECT * +SELECT session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + pages_count, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign, + user_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + issue_types, + referrer, + base_referrer, + _timestamp FROM experimental.sessions WHERE datetime >= now() - INTERVAL 7 DAY AND isNotNull(duration) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.8.0/1.8.0.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.8.0/1.8.0.sql index 90c4322bc..0decabdce 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.8.0/1.8.0.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.8.0/1.8.0.sql @@ -9,6 +9,25 @@ ALTER TABLE IF EXISTS projects ADD COLUMN IF NOT EXISTS first_recorded_session_at timestamp without time zone NULL DEFAULT NULL, ADD COLUMN IF NOT EXISTS sessions_last_check_at timestamp without time zone NULL DEFAULT NULL; + +DO +$$ + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + INNER JOIN pg_namespace nsp + ON nsp.oid = typ.typnamespace + WHERE nsp.nspname = current_schema() + AND typ.typname = 'alert_change_type') THEN + CREATE TYPE alert_change_type AS ENUM ('percent', 'change'); + END IF; + END; +$$ +LANGUAGE plpgsql; + +ALTER TABLE IF EXISTS alerts + ADD COLUMN IF NOT EXISTS change alert_change_type NOT NULL DEFAULT 'change'; + COMMIT; CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS autocomplete_unique_project_id_md5value_type_idx ON autocomplete (project_id, md5(value), type); diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index dd5c380da..4a4e7b716 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -819,6 +819,13 @@ $$ WHERE typ.typname = 'alert_detection_method') THEN CREATE TYPE alert_detection_method AS ENUM ('threshold', 'change'); END IF; + + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'alert_detection_method') THEN + CREATE TYPE alert_change_type AS ENUM ('percent', 'change'); + END IF; + CREATE TABLE IF NOT EXISTS alerts ( alert_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, @@ -828,6 +835,7 @@ $$ description text NULL DEFAULT NULL, active boolean NOT NULL DEFAULT TRUE, detection_method alert_detection_method NOT NULL, + change alert_change_type NOT NULL DEFAULT 'change', query jsonb NOT NULL, deleted_at timestamp NULL DEFAULT NULL, created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), diff --git a/scripts/helm/db/init_dbs/postgresql/1.8.0/1.8.0.sql b/scripts/helm/db/init_dbs/postgresql/1.8.0/1.8.0.sql index bfe903ee5..64aa33581 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.8.0/1.8.0.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.8.0/1.8.0.sql @@ -9,6 +9,25 @@ ALTER TABLE IF EXISTS projects ADD COLUMN IF NOT EXISTS first_recorded_session_at timestamp without time zone NULL DEFAULT NULL, ADD COLUMN IF NOT EXISTS sessions_last_check_at timestamp without time zone NULL DEFAULT NULL; + +DO +$$ + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + INNER JOIN pg_namespace nsp + ON nsp.oid = typ.typnamespace + WHERE nsp.nspname = current_schema() + AND typ.typname = 'alert_change_type') THEN + CREATE TYPE alert_change_type AS ENUM ('percent', 'change'); + END IF; + END; +$$ +LANGUAGE plpgsql; + +ALTER TABLE IF EXISTS alerts + ADD COLUMN IF NOT EXISTS change alert_change_type NOT NULL DEFAULT 'change'; + COMMIT; CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS autocomplete_unique_project_id_md5value_type_idx ON autocomplete (project_id, md5(value), type); diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index d8fe2afcb..6b0340757 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -967,6 +967,7 @@ $$ CREATE INDEX searches_project_id_idx ON public.searches (project_id); CREATE TYPE alert_detection_method AS ENUM ('threshold', 'change'); + CREATE TYPE alert_change_type AS ENUM ('percent', 'change'); CREATE TABLE alerts ( @@ -977,6 +978,7 @@ $$ description text NULL DEFAULT NULL, active boolean NOT NULL DEFAULT TRUE, detection_method alert_detection_method NOT NULL, + change alert_change_type NOT NULL DEFAULT 'change', query jsonb NOT NULL, deleted_at timestamp NULL DEFAULT NULL, created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()),