From 06b4db94b5cd1963970059516f69291086108395 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 17 Jan 2025 17:54:28 +0100 Subject: [PATCH] change(api): ch query updates - use created_at --- api/chalicelib/core/errors/errors_ch.py | 16 +++---- api/chalicelib/core/sessions/sessions_ch.py | 46 ++++++++++----------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/api/chalicelib/core/errors/errors_ch.py b/api/chalicelib/core/errors/errors_ch.py index 37e22dd71..fc2825110 100644 --- a/api/chalicelib/core/errors/errors_ch.py +++ b/api/chalicelib/core/errors/errors_ch.py @@ -92,8 +92,8 @@ def __get_basic_constraints_events(platform=None, time_constraint=True, startTim if type_condition: ch_sub_query.append(f"{table_name}`$event_name`='ERROR'") if time_constraint: - ch_sub_query += [f"{table_name}_timestamp >= toDateTime(%({startTime_arg_name})s/1000)", - f"{table_name}_timestamp < toDateTime(%({endTime_arg_name})s/1000)"] + ch_sub_query += [f"{table_name}created_at >= toDateTime(%({startTime_arg_name})s/1000)", + f"{table_name}created_at < toDateTime(%({endTime_arg_name})s/1000)"] # if platform == schemas.PlatformType.MOBILE: # ch_sub_query.append("user_device_type = 'mobile'") # elif platform == schemas.PlatformType.DESKTOP: @@ -123,7 +123,7 @@ def search(data: schemas.SearchErrorsSchema, project_id, user_id): ch_sub_query.append("JSONExtractString(toString(`$properties`), 'source') = 'js_exception'") # To ignore Script error - ch_sub_query.append("JSONExtractString(toString(`$properties`)) != 'Script error.'") + ch_sub_query.append("JSONExtractString(toString(`$properties`), 'message') != 'Script error.'") error_ids = None if data.startTimestamp is None: @@ -361,8 +361,8 @@ def search(data: schemas.SearchErrorsSchema, project_id, user_id): JSONExtractString(toString(`$properties`), 'message') AS message, COUNT(DISTINCT user_id) AS users, COUNT(DISTINCT events.session_id) AS sessions, - MAX(_timestamp) AS max_datetime, - MIN(_timestamp) AS min_datetime, + MAX(created_at) AS max_datetime, + MIN(created_at) AS min_datetime, COUNT(DISTINCT JSONExtractString(toString(`$properties`), 'error_id')) OVER() AS total FROM {MAIN_EVENTS_TABLE} AS events @@ -376,8 +376,8 @@ def search(data: schemas.SearchErrorsSchema, project_id, user_id): ORDER BY {sort} {order} LIMIT %(errors_limit)s OFFSET %(errors_offset)s) AS details INNER JOIN (SELECT JSONExtractString(toString(`$properties`), 'error_id') AS error_id, - toUnixTimestamp(MAX(_timestamp))*1000 AS last_occurrence, - toUnixTimestamp(MIN(_timestamp))*1000 AS first_occurrence + toUnixTimestamp(MAX(created_at))*1000 AS last_occurrence, + toUnixTimestamp(MIN(created_at))*1000 AS first_occurrence FROM {MAIN_EVENTS_TABLE} WHERE project_id=%(project_id)s AND `$event_name`='ERROR' @@ -385,7 +385,7 @@ def search(data: schemas.SearchErrorsSchema, project_id, user_id): ON details.error_id=time_details.error_id INNER JOIN (SELECT error_id, groupArray([timestamp, count]) AS chart FROM (SELECT JSONExtractString(toString(`$properties`), 'error_id') AS error_id, - toUnixTimestamp(toStartOfInterval(_timestamp, INTERVAL %(step_size)s second)) * 1000 AS timestamp, + toUnixTimestamp(toStartOfInterval(created_at, INTERVAL %(step_size)s second)) * 1000 AS timestamp, COUNT(DISTINCT session_id) AS count FROM {MAIN_EVENTS_TABLE} WHERE {" AND ".join(ch_sub_query)} diff --git a/api/chalicelib/core/sessions/sessions_ch.py b/api/chalicelib/core/sessions/sessions_ch.py index c6dd00bb8..33cbbd4df 100644 --- a/api/chalicelib/core/sessions/sessions_ch.py +++ b/api/chalicelib/core/sessions/sessions_ch.py @@ -21,8 +21,8 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d if metric_of == schemas.MetricOfTable.VISITED_URL: extra_event = f"""SELECT DISTINCT ev.session_id, ev.url_path FROM {exp_ch_helper.get_main_events_table(data.startTimestamp)} AS ev - WHERE ev.`_timestamp` >= toDateTime(%(startDate)s / 1000) - AND ev.`_timestamp` <= toDateTime(%(endDate)s / 1000) + WHERE ev.created_at >= toDateTime(%(startDate)s / 1000) + AND ev.created_at <= toDateTime(%(endDate)s / 1000) AND ev.project_id = %(project_id)s AND ev.`$event_name` = 'LOCATION'""" elif metric_of == schemas.MetricOfTable.ISSUES and len(metric_value) > 0: @@ -138,8 +138,8 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de extra_event = f"""SELECT DISTINCT ev.session_id, JSONExtractString(toString(ev.`$properties`), 'url_path') AS url_path FROM {exp_ch_helper.get_main_events_table(data.startTimestamp)} AS ev - WHERE ev.`_timestamp` >= toDateTime(%(startDate)s / 1000) - AND ev.`_timestamp` <= toDateTime(%(endDate)s / 1000) + WHERE ev.created_at >= toDateTime(%(startDate)s / 1000) + AND ev.created_at <= toDateTime(%(endDate)s / 1000) AND ev.project_id = %(project_id)s AND ev.`$event_name` = 'LOCATION'""" extra_deduplication.append("url_path") @@ -162,8 +162,8 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de extra_event = f"""SELECT DISTINCT ev.session_id, JSONExtractString(toString(ev.`$properties`), 'url_path') AS url_path FROM {exp_ch_helper.get_main_events_table(data.startTimestamp)} AS ev - WHERE ev.`_timestamp` >= toDateTime(%(startDate)s / 1000) - AND ev.`_timestamp` <= toDateTime(%(endDate)s / 1000) + WHERE ev.created_at >= toDateTime(%(startDate)s / 1000) + AND ev.created_at <= toDateTime(%(endDate)s / 1000) AND ev.project_id = %(project_id)s AND ev.`$event_name` = 'REQUEST'""" @@ -367,11 +367,11 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu events_query_part = "" issues = [] __events_where_basic = ["project_id = %(projectId)s", - "`_timestamp` >= toDateTime(%(startDate)s/1000)", - "`_timestamp` <= toDateTime(%(endDate)s/1000)"] + "created_at >= toDateTime(%(startDate)s/1000)", + "created_at <= toDateTime(%(endDate)s/1000)"] events_conditions_where = ["main.project_id = %(projectId)s", - "main.`_timestamp` >= toDateTime(%(startDate)s/1000)", - "main.`_timestamp` <= toDateTime(%(endDate)s/1000)"] + "main.created_at >= toDateTime(%(startDate)s/1000)", + "main.created_at <= toDateTime(%(endDate)s/1000)"] if len(data.filters) > 0: meta_keys = None # to reduce include a sub-query of sessions inside events query, in order to reduce the selected data @@ -650,8 +650,8 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu # event_from = f"%s INNER JOIN {MAIN_SESSIONS_TABLE} AS ms USING (session_id)" event_from = "%s" event_where = ["main.project_id = %(projectId)s", - "main.`_timestamp` >= toDateTime(%(startDate)s/1000)", - "main.`_timestamp` <= toDateTime(%(endDate)s/1000)"] + "main.created_at >= toDateTime(%(startDate)s/1000)", + "main.created_at <= toDateTime(%(endDate)s/1000)"] e_k = f"e_value{i}" s_k = e_k + "_source" @@ -1224,7 +1224,7 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu pass else: events_query_from.append(f"""\ - (SELECT main.session_id, {"MIN" if event_index < (valid_events_count - 1) else "MAX"}(main.`_timestamp`) AS datetime + (SELECT main.session_id, {"MIN" if event_index < (valid_events_count - 1) else "MAX"}(main.created_at) AS datetime FROM {event_from} WHERE {" AND ".join(event_where)} GROUP BY session_id @@ -1292,13 +1292,13 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu del value_conditions_not if data.events_order == schemas.SearchEventOrder.THEN: - having = f"""HAVING sequenceMatch('{''.join(sequence_pattern)}')(main.`_timestamp`,{','.join(sequence_conditions)})""" + having = f"""HAVING sequenceMatch('{''.join(sequence_pattern)}')(toDateTime(main.created_at),{','.join(sequence_conditions)})""" else: having = f"""HAVING {" AND ".join([f"countIf({c})>0" for c in list(set(sequence_conditions))])}""" events_query_part = f"""SELECT main.session_id, - MIN(main.`_timestamp`) AS first_event_ts, - MAX(main.`_timestamp`) AS last_event_ts + MIN(main.created_at) AS first_event_ts, + MAX(main.created_at) AS last_event_ts FROM {MAIN_EVENTS_TABLE} AS main {events_extra_join} {sub_join} WHERE {" AND ".join(events_conditions_where)} @@ -1340,8 +1340,8 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu events_conditions_where.append(f"({' OR '.join(events_conditions)})") events_query_part = f"""SELECT main.session_id, - MIN(main.`_timestamp`) AS first_event_ts, - MAX(main.`_timestamp`) AS last_event_ts + MIN(main.created_at) AS first_event_ts, + MAX(main.created_at) AS last_event_ts FROM {MAIN_EVENTS_TABLE} AS main {events_extra_join} WHERE {" AND ".join(events_conditions_where)} GROUP BY session_id""" @@ -1364,8 +1364,8 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu AND issues.project_id = %(projectId)s AND events.project_id = %(projectId)s AND events.issue_type = %(issue_type)s - AND events.`_timestamp` >= toDateTime(%(startDate)s/1000) - AND events.`_timestamp` <= toDateTime(%(endDate)s/1000) + AND events.created_at >= toDateTime(%(startDate)s/1000) + AND events.created_at <= toDateTime(%(endDate)s/1000) ) AS issues ON (f.session_id = issues.session_id) """ full_args["issue_contextString"] = issue["contextString"] @@ -1384,8 +1384,8 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu INNER JOIN experimental.events USING (issue_id) WHERE issues.project_id = %(projectId)s AND events.project_id = %(projectId)s - AND events.`_timestamp` >= toDateTime(%(startDate)s/1000) - AND events.`_timestamp` <= toDateTime(%(endDate)s/1000) + AND events.created_at >= toDateTime(%(startDate)s/1000) + AND events.created_at <= toDateTime(%(endDate)s/1000) AND {" OR ".join(issues_conditions)} ) AS issues USING (session_id)""" @@ -1423,7 +1423,7 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu extra_join = f"""(SELECT * FROM {MAIN_SESSIONS_TABLE} AS s {extra_join} {extra_event} WHERE {" AND ".join(extra_constraints)} - ORDER BY _timestamp DESC + ORDER BY datetime DESC LIMIT 1 BY {",".join(deduplication_keys)}) AS s""" query_part = f"""\ FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else ""}