From 56b2c6250b6444b647759203b03dea7c4d6d8886 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 16 May 2025 17:19:34 +0200 Subject: [PATCH] refactor(chalice): support incidents for replay --- api/chalicelib/core/events/__init__.py | 2 +- api/chalicelib/core/events/events_ch.py | 69 ++++++++++++++----- api/chalicelib/core/events/events_pg.py | 12 +++- api/chalicelib/core/issues/issues_ch.py | 27 +++++++- api/chalicelib/core/issues/issues_pg.py | 30 ++++++-- .../core/sessions/sessions_replay.py | 26 +------ api/schemas/schemas.py | 1 + 7 files changed, 116 insertions(+), 51 deletions(-) diff --git a/api/chalicelib/core/events/__init__.py b/api/chalicelib/core/events/__init__.py index d4df51f72..897cb7fa5 100644 --- a/api/chalicelib/core/events/__init__.py +++ b/api/chalicelib/core/events/__init__.py @@ -4,7 +4,7 @@ from decouple import config logger = logging.getLogger(__name__) -if config("EXP_EVENTS_REPLAY", cast=bool, default=False): +if config("EXP_EVENTS", cast=bool, default=False): logger.info(">>> Using experimental events replay") from . import events_ch as events else: diff --git a/api/chalicelib/core/events/events_ch.py b/api/chalicelib/core/events/events_ch.py index 4bca47a27..be90e230d 100644 --- a/api/chalicelib/core/events/events_ch.py +++ b/api/chalicelib/core/events/events_ch.py @@ -2,18 +2,27 @@ from chalicelib.utils import ch_client from .events_pg import * +def __explode_properties(rows): + for i in range(len(rows)): + rows[i] = {**rows[i], **rows[i]["$properties"]} + rows[i].pop("$properties") + return rows + + def get_customs_by_session_id(session_id, project_id): with ch_client.ClickHouseClient() as cur: rows = cur.execute(""" \ - SELECT c.*, + SELECT `$properties`, + created_at, 'CUSTOM' AS type - FROM product_analytics.events AS c - WHERE c.session_id = %(session_id)s + FROM product_analytics.events + WHERE session_id = %(session_id)s AND NOT `$auto_captured` - ORDER BY c.timestamp;""", - {"project_id": project_id, "session_id": session_id} - ) - return helper.dict_to_camel_case(rows) + AND `$event_name`!='INCIDENT' + ORDER BY created_at;""", + {"project_id": project_id, "session_id": session_id}) + rows = __explode_properties(rows) + return helper.list_to_camel_case(rows) def __merge_cells(rows, start, count, replacement): @@ -34,7 +43,7 @@ def __get_grouped_clickrage(rows, session_id, project_id): else: merge_count = 3 for i in range(len(rows)): - if rows[i]["timestamp"] == c["timestamp"]: + if rows[i]["created_at"] == c["createdAt"]: rows = __merge_cells(rows=rows, start=i, count=merge_count, @@ -48,19 +57,41 @@ def get_by_session_id(session_id, project_id, group_clickrage=False, event_type: select_events = ('CLICK', 'INPUT', 'LOCATION') if event_type is not None: select_events = (event_type,) - rows = cur.execute(""" \ - SELECT c.*, - `$event_name` AS type - FROM product_analytics.events - WHERE session_id = %(session_id)s - AND `$event_name` IN %(select_events)s - AND `$auto_captured` - ORDER BY c.timestamp;""", - {"project_id": project_id, "session_id": session_id, "select_events": select_events}) - + query = cur.format(query=""" \ + SELECT created_at, + `$properties`, + `$event_name` AS type + FROM product_analytics.events + WHERE session_id = %(session_id)s + AND `$event_name` IN %(select_events)s + AND `$auto_captured` + ORDER BY created_at;""", + parameters={"project_id": project_id, "session_id": session_id, + "select_events": select_events}) + rows = cur.execute(query) + rows = __explode_properties(rows) if group_clickrage and 'CLICK' in select_events: rows = __get_grouped_clickrage(rows=rows, session_id=session_id, project_id=project_id) rows = helper.list_to_camel_case(rows) - rows = sorted(rows, key=lambda k: (k["timestamp"], k["messageId"])) + rows = sorted(rows, key=lambda k: k["createdAt"]) + return rows + + +def get_incidents_by_session_id(session_id, project_id): + with ch_client.ClickHouseClient() as cur: + query = cur.format(query=""" \ + SELECT created_at, + `$properties`, + `properties`, + `$event_name` AS type + FROM product_analytics.events + WHERE session_id = %(session_id)s + AND `$event_name` = 'INCIDENT' + AND `$auto_captured` + ORDER BY created_at;""", + parameters={"project_id": project_id, "session_id": session_id}) + rows = cur.execute(query) + rows = helper.list_to_camel_case(rows) + rows = sorted(rows, key=lambda k: k["createdAt"]) return rows diff --git a/api/chalicelib/core/events/events_pg.py b/api/chalicelib/core/events/events_pg.py index d481e6259..06b524f51 100644 --- a/api/chalicelib/core/events/events_pg.py +++ b/api/chalicelib/core/events/events_pg.py @@ -1,14 +1,17 @@ +import logging from functools import cache from typing import Optional import schemas -from chalicelib.core.issues import issues from chalicelib.core.autocomplete import autocomplete +from chalicelib.core.issues import issues from chalicelib.core.sessions import sessions_metas from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC from chalicelib.utils.event_filter_definition import SupportedFilter +logger = logging.getLogger(__name__) + def get_customs_by_session_id(session_id, project_id): with pg_client.PostgresClient() as cur: @@ -21,7 +24,7 @@ def get_customs_by_session_id(session_id, project_id): {"project_id": project_id, "session_id": session_id}) ) rows = cur.fetchall() - return helper.dict_to_camel_case(rows) + return helper.list_to_camel_case(rows) def __merge_cells(rows, start, count, replacement): @@ -179,6 +182,11 @@ def get_errors_by_session_id(session_id, project_id): return helper.list_to_camel_case(errors) +def get_incidents_by_session_id(session_id, project_id): + logger.warning("INCIDENTS not supported in PG") + return [] + + def search(text, event_type, project_id, source, key): if not event_type: return {"data": autocomplete.__get_autocomplete_table(text, project_id)} diff --git a/api/chalicelib/core/issues/issues_ch.py b/api/chalicelib/core/issues/issues_ch.py index d705e3288..b6297f598 100644 --- a/api/chalicelib/core/issues/issues_ch.py +++ b/api/chalicelib/core/issues/issues_ch.py @@ -1,4 +1,5 @@ from chalicelib.utils import ch_client, helper +import datetime from .issues_pg import get_all_types @@ -18,14 +19,38 @@ def get(project_id, issue_id): def get_by_session_id(session_id, project_id, issue_type=None): - with ch_client.ClickHouseClient as cur: + with ch_client.ClickHouseClient() as cur: query = cur.format(query=f"""\ SELECT * FROM product_analytics.events WHERE session_id = %(session_id)s AND project_id= %(project_id)s + AND `$event_name`='ISSUE' {"AND issue_type = %(type)s" if issue_type is not None else ""} ORDER BY created_at;""", parameters={"session_id": session_id, "project_id": project_id, "type": issue_type}) data = cur.execute(query) return helper.list_to_camel_case(data) + + +# To reduce the number of issues in the replay; +# will be removed once we agree on how to show issues +def reduce_issues(issues_list): + if issues_list is None: + return None + i = 0 + # remove same-type issues if the time between them is <2s + while i < len(issues_list) - 1: + for j in range(i + 1, len(issues_list)): + if issues_list[i]["issueType"] == issues_list[j]["issueType"]: + break + else: + i += 1 + break + + if issues_list[i]["createdAt"] - issues_list[j]["createdAt"] < datetime.timedelta(seconds=2): + issues_list.pop(j) + else: + i += 1 + + return issues_list diff --git a/api/chalicelib/core/issues/issues_pg.py b/api/chalicelib/core/issues/issues_pg.py index 66edcdf45..b2ca0b142 100644 --- a/api/chalicelib/core/issues/issues_pg.py +++ b/api/chalicelib/core/issues/issues_pg.py @@ -4,12 +4,11 @@ from chalicelib.utils import pg_client, helper def get(project_id, issue_id): with pg_client.PostgresClient() as cur: query = cur.mogrify( - """\ - SELECT - * + """ \ + SELECT * FROM public.issues WHERE project_id = %(project_id)s - AND issue_id = %(issue_id)s;""", + AND issue_id = %(issue_id)s;""", {"project_id": project_id, "issue_id": issue_id} ) cur.execute(query=query) @@ -35,6 +34,29 @@ def get_by_session_id(session_id, project_id, issue_type=None): return helper.list_to_camel_case(cur.fetchall()) +# To reduce the number of issues in the replay; +# will be removed once we agree on how to show issues +def reduce_issues(issues_list): + if issues_list is None: + return None + i = 0 + # remove same-type issues if the time between them is <2s + while i < len(issues_list) - 1: + for j in range(i + 1, len(issues_list)): + if issues_list[i]["type"] == issues_list[j]["type"]: + break + else: + i += 1 + break + + if issues_list[i]["timestamp"] - issues_list[j]["timestamp"] < 2000: + issues_list.pop(j) + else: + i += 1 + + return issues_list + + def get_all_types(): return [ { diff --git a/api/chalicelib/core/sessions/sessions_replay.py b/api/chalicelib/core/sessions/sessions_replay.py index 6602cc434..b53d60a29 100644 --- a/api/chalicelib/core/sessions/sessions_replay.py +++ b/api/chalicelib/core/sessions/sessions_replay.py @@ -129,30 +129,8 @@ def get_events(project_id, session_id): data['userTesting'] = user_testing.get_test_signals(session_id=session_id, project_id=project_id) data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) - data['issues'] = reduce_issues(data['issues']) + data['issues'] = issues.reduce_issues(data['issues']) + data['incidents'] = events.get_incidents_by_session_id(session_id=session_id, project_id=project_id) return data else: return None - - -# To reduce the number of issues in the replay; -# will be removed once we agree on how to show issues -def reduce_issues(issues_list): - if issues_list is None: - return None - i = 0 - # remove same-type issues if the time between them is <2s - while i < len(issues_list) - 1: - for j in range(i + 1, len(issues_list)): - if issues_list[i]["type"] == issues_list[j]["type"]: - break - else: - i += 1 - break - - if issues_list[i]["timestamp"] - issues_list[j]["timestamp"] < 2000: - issues_list.pop(j) - else: - i += 1 - - return issues_list diff --git a/api/schemas/schemas.py b/api/schemas/schemas.py index 973dce8f1..7ab8f5ad4 100644 --- a/api/schemas/schemas.py +++ b/api/schemas/schemas.py @@ -406,6 +406,7 @@ class EventType(str, Enum): ERROR_MOBILE = "errorMobile" SWIPE_MOBILE = "swipeMobile" EVENT = "event" + INCIDENT = "incident" class PerformanceEventType(str, Enum):