From c2405dfbb38c6a681f698b8d9dd8de3c20f09fc8 Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Wed, 6 Nov 2024 17:24:03 +0100 Subject: [PATCH] Dev (#2727) * fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * feat(chalice): autocomplete return top 10 with stats * fix(chalice): fixed autocomplete top 10 meta-filters * refactor(chalice): removed unused code --- ee/api/chalicelib/core/autocomplete_exp.py | 6 - ee/api/chalicelib/core/errors_exp.py | 176 --------------------- ee/api/chalicelib/core/log_tools.py | 16 +- ee/api/chalicelib/core/metrics.py | 36 ----- ee/api/chalicelib/core/projects.py | 23 --- ee/api/chalicelib/core/traces.py | 11 -- ee/api/chalicelib/utils/assist_helper.py | 4 +- ee/api/chalicelib/utils/exp_ch_helper.py | 4 - 8 files changed, 3 insertions(+), 273 deletions(-) diff --git a/ee/api/chalicelib/core/autocomplete_exp.py b/ee/api/chalicelib/core/autocomplete_exp.py index 494f07108..16fcdd3cb 100644 --- a/ee/api/chalicelib/core/autocomplete_exp.py +++ b/ee/api/chalicelib/core/autocomplete_exp.py @@ -264,12 +264,6 @@ def __search_metadata(project_id, value, key=None, source=None): return helper.list_to_camel_case(results) -class TableColumn: - def __init__(self, table, column): - self.table = table - self.column = column - - TYPE_TO_COLUMN = { schemas.EventType.CLICK: "label", schemas.EventType.INPUT: "label", diff --git a/ee/api/chalicelib/core/errors_exp.py b/ee/api/chalicelib/core/errors_exp.py index f36edc2e6..4ef3441df 100644 --- a/ee/api/chalicelib/core/errors_exp.py +++ b/ee/api/chalicelib/core/errors_exp.py @@ -373,122 +373,6 @@ def get_details(project_id, error_id, user_id, **data): return {"data": helper.dict_to_camel_case(row)} -def get_details_chart(project_id, error_id, user_id, **data): - ch_sub_query = __get_basic_constraints() - ch_sub_query.append("error_id = %(error_id)s") - with ch_client.ClickHouseClient() as ch: - if data.get("startDate") is None: - data["startDate"] = TimeUTC.now(-7) - else: - data["startDate"] = int(data["startDate"]) - if data.get("endDate") is None: - data["endDate"] = TimeUTC.now() - else: - data["endDate"] = int(data["endDate"]) - density = int(data.get("density", 7)) - step_size = __get_step_size(data["startDate"], data["endDate"], density) - params = { - "startDate": data['startDate'], - "endDate": data['endDate'], - "project_id": project_id, - "userId": user_id, - "step_size": step_size, - "error_id": error_id} - - main_ch_query = f"""\ - SELECT browser_details.error_id AS error_id, - browsers_partition, - os_partition, - device_partition, - country_partition, - chart - FROM (SELECT %(error_id)s AS error_id, - groupArray([[[user_browser]], [[toString(count_per_browser)]],versions_partition]) AS browsers_partition - FROM (SELECT user_browser, - COUNT(session_id) AS count_per_browser - FROM errors - WHERE {" AND ".join(ch_sub_query)} - GROUP BY user_browser - ORDER BY count_per_browser DESC) AS count_per_browser_query - INNER JOIN (SELECT user_browser, - groupArray([user_browser_version, toString(count_per_version)]) AS versions_partition - FROM (SELECT user_browser, - user_browser_version, - COUNT(session_id) AS count_per_version - FROM errors - WHERE {" AND ".join(ch_sub_query)} - GROUP BY user_browser, user_browser_version - ORDER BY count_per_version DESC) AS count_per_version_details - GROUP BY user_browser ) AS browesr_version_details USING (user_browser)) AS browser_details - INNER JOIN (SELECT %(error_id)s AS error_id, - groupArray( - [[[user_os]], [[toString(count_per_os)]],versions_partition]) AS os_partition - FROM (SELECT user_os, - COUNT(session_id) AS count_per_os - FROM errors - WHERE {" AND ".join(ch_sub_query)} - GROUP BY user_os - ORDER BY count_per_os DESC) AS count_per_os_details - INNER JOIN (SELECT user_os, - groupArray([user_os_version, toString(count_per_version)]) AS versions_partition - FROM (SELECT user_os, user_os_version, COUNT(session_id) AS count_per_version - FROM errors - WHERE {" AND ".join(ch_sub_query)} - GROUP BY user_os, user_os_version - ORDER BY count_per_version DESC) AS count_per_version_query - GROUP BY user_os ) AS os_version_query USING (user_os)) AS os_details - ON os_details.error_id = browser_details.error_id - INNER JOIN (SELECT %(error_id)s AS error_id, - groupArray( - [[[toString(user_device_type)]], [[toString(count_per_device)]],versions_partition]) AS device_partition - FROM (SELECT user_device_type, - COUNT(session_id) AS count_per_device - FROM errors - WHERE {" AND ".join(ch_sub_query)} - GROUP BY user_device_type - ORDER BY count_per_device DESC) AS count_per_device_details - INNER JOIN (SELECT user_device_type, - groupArray([user_device, toString(count_per_device)]) AS versions_partition - FROM (SELECT user_device_type, - coalesce(user_device,'unknown') AS user_device, - COUNT(session_id) AS count_per_device - FROM errors - WHERE {" AND ".join(ch_sub_query)} - GROUP BY user_device_type, user_device - ORDER BY count_per_device DESC) AS count_per_device_details - GROUP BY user_device_type ) AS device_version_details USING (user_device_type)) AS device_details - ON device_details.error_id = os_details.error_id - INNER JOIN (SELECT %(error_id)s AS error_id, - groupArray( - [[[toString(user_country)]], [[toString(count_per_country)]]]) AS country_partition - FROM (SELECT user_country, - COUNT(session_id) AS count_per_country - FROM errors - WHERE {" AND ".join(ch_sub_query)} - GROUP BY user_country - ORDER BY count_per_country DESC) AS count_per_country_details) AS country_details - ON country_details.error_id = device_details.error_id - INNER JOIN (SELECT %(error_id)s AS error_id, groupArray([timestamp, count]) AS chart - FROM (SELECT toUnixTimestamp(toStartOfInterval(datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(DISTINCT session_id) AS count - FROM errors - WHERE {" AND ".join(ch_sub_query)} - GROUP BY timestamp - ORDER BY timestamp) AS chart_details) AS chart_details - ON country_details.error_id = chart_details.error_id;""" - - # print("--------------------") - # print(main_ch_query % params) - row = ch.execute(query=main_ch_query, params=params) - if len(row) == 0: - return {"errors": ["error not found"]} - row = row[0] - row["tags"] = __process_tags(row) - row["chart"] = __rearrange_chart_details(start_at=data["startDate"], end_at=data["endDate"], density=density, - chart=row["chart"]) - return {"data": helper.dict_to_camel_case(row)} - - def __get_basic_constraints(platform=None, time_constraint=True, startTime_arg_name="startDate", endTime_arg_name="endDate", type_condition=True, project_key="project_id", table_name=None): ch_sub_query = [f"{project_key} =toUInt16(%(project_id)s)"] @@ -521,26 +405,6 @@ def __get_sort_key(key): }.get(key, 'max_datetime') -def __get_basic_constraints_pg(platform=None, time_constraint=True, startTime_arg_name="startDate", - endTime_arg_name="endDate", chart=False, step_size_name="step_size", - project_key="project_id"): - if project_key is None: - ch_sub_query = [] - else: - ch_sub_query = [f"{project_key} =%(project_id)s"] - if time_constraint: - ch_sub_query += [f"timestamp >= %({startTime_arg_name})s", - f"timestamp < %({endTime_arg_name})s"] - if chart: - ch_sub_query += [f"timestamp >= generated_timestamp", - f"timestamp < generated_timestamp + %({step_size_name})s"] - if platform == schemas.PlatformType.MOBILE: - ch_sub_query.append("user_device_type = 'mobile'") - elif platform == schemas.PlatformType.DESKTOP: - ch_sub_query.append("user_device_type = 'desktop'") - return ch_sub_query - - def search(data: schemas.SearchErrorsSchema, project_id, user_id): MAIN_EVENTS_TABLE = exp_ch_helper.get_main_events_table(data.startTimestamp) MAIN_SESSIONS_TABLE = exp_ch_helper.get_main_sessions_table(data.startTimestamp) @@ -907,43 +771,3 @@ def merge(error_ids): # row = cur.fetchone() return {"data": "success"} - - -def format_first_stack_frame(error): - error["stack"] = sourcemaps.format_payload(error.pop("payload"), truncate_to_first=True) - for s in error["stack"]: - for c in s.get("context", []): - for sci, sc in enumerate(c): - if isinstance(sc, str) and len(sc) > 1000: - c[sci] = sc[:1000] - # convert bytes to string: - if isinstance(s["filename"], bytes): - s["filename"] = s["filename"].decode("utf-8") - return error - - -def stats(project_id, user_id, startTimestamp=TimeUTC.now(delta_days=-7), endTimestamp=TimeUTC.now()): - with pg_client.PostgresClient() as cur: - query = cur.mogrify( - """WITH user_viewed AS (SELECT error_id FROM public.user_viewed_errors WHERE user_id = %(userId)s) - SELECT COUNT(timed_errors.*) AS unresolved_and_unviewed - FROM (SELECT root_error.error_id - FROM events.errors - INNER JOIN public.errors AS root_error USING (error_id) - LEFT JOIN user_viewed USING (error_id) - WHERE project_id = %(project_id)s - AND timestamp >= %(startTimestamp)s - AND timestamp <= %(endTimestamp)s - AND source = 'js_exception' - AND root_error.status = 'unresolved' - AND user_viewed.error_id ISNULL - LIMIT 1 - ) AS timed_errors;""", - {"project_id": project_id, "userId": user_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp}) - cur.execute(query=query) - row = cur.fetchone() - - return { - "data": helper.dict_to_camel_case(row) - } diff --git a/ee/api/chalicelib/core/log_tools.py b/ee/api/chalicelib/core/log_tools.py index a56a01c55..e1903f695 100644 --- a/ee/api/chalicelib/core/log_tools.py +++ b/ee/api/chalicelib/core/log_tools.py @@ -60,20 +60,6 @@ def get(project_id, integration): return helper.dict_to_camel_case(helper.flatten_nested_dicts(r)) -def get_all_by_type(integration): - with pg_client.PostgresClient() as cur: - cur.execute( - cur.mogrify( - """\ - SELECT integrations.* - FROM public.integrations INNER JOIN public.projects USING(project_id) - WHERE provider = %(provider)s AND projects.deleted_at ISNULL;""", - {"provider": integration}) - ) - r = cur.fetchall() - return helper.list_to_camel_case(r, flatten=True) - - def edit(project_id, integration, changes): if "projectId" in changes: changes.pop("project_id") @@ -117,4 +103,4 @@ def get_all_by_tenant(tenant_id, integration): {"tenant_id": tenant_id, "provider": integration}) ) r = cur.fetchall() - return helper.list_to_camel_case(r, flatten=True) \ No newline at end of file + return helper.list_to_camel_case(r, flatten=True) diff --git a/ee/api/chalicelib/core/metrics.py b/ee/api/chalicelib/core/metrics.py index 7d05c98f2..a0529fabc 100644 --- a/ee/api/chalicelib/core/metrics.py +++ b/ee/api/chalicelib/core/metrics.py @@ -85,16 +85,6 @@ def __complete_missing_steps(start_time, end_time, density, neutral, rows, time_ return result -def __merge_charts(list1, list2, time_key="timestamp"): - if len(list1) != len(list2): - raise Exception("cannot merge unequal lists") - result = [] - for i in range(len(list1)): - timestamp = min(list1[i][time_key], list2[i][time_key]) - result.append({**list1[i], **list2[i], time_key: timestamp}) - return result - - def __get_constraint(data, fields, table_name): constraints = [] # for k in fields.keys(): @@ -314,32 +304,6 @@ def get_errors_per_domains(project_id, limit, page, startTimestamp=TimeUTC.now(d return response -def __get_calls_errors_4xx_or_5xx(status, project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - platform=None, **args): - ch_sub_query = __get_basic_constraints(table_name="requests", data=args) - ch_sub_query.append("requests.event_type = 'REQUEST'") - ch_sub_query.append(f"intDiv(requests.status, 100) == {status}") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT requests.method, - requests.url_hostpath, - COUNT(1) AS all_requests - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS requests - WHERE {" AND ".join(ch_sub_query)} - GROUP BY requests.method, requests.url_hostpath - ORDER BY all_requests DESC - LIMIT 10;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - # print(ch.format(query=ch_query, params=params)) - rows = ch.execute(query=ch_query, params=params) - return helper.list_to_camel_case(rows) - - def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index a28ca71e3..499a2bc8a 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -149,29 +149,6 @@ def get_project(tenant_id, project_id, include_last_session=False, include_gdpr= return helper.dict_to_camel_case(row) -def get_project_by_key(tenant_id, project_key, include_last_session=False, include_gdpr=None): - with pg_client.PostgresClient() as cur: - extra_select = "" - if include_last_session: - extra_select += """,(SELECT max(ss.start_ts) - FROM public.sessions AS ss - WHERE ss.project_key = %(project_key)s) AS last_recorded_session_at""" - if include_gdpr: - extra_select += ",s.gdpr" - query = cur.mogrify(f"""SELECT s.project_key, - s.name - {extra_select} - FROM public.projects AS s - WHERE s.project_key =%(project_key)s - AND s.tenant_id =%(tenant_id)s - AND s.deleted_at IS NULL - LIMIT 1;""", - {"project_key": project_key, "tenant_id": tenant_id}) - cur.execute(query=query) - row = cur.fetchone() - return helper.dict_to_camel_case(row) - - def create(tenant_id, user_id, data: schemas.CreateProjectSchema, skip_authorization=False): if __exists_by_name(name=data.name, exclude_id=None, tenant_id=tenant_id): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"name already exists.") diff --git a/ee/api/chalicelib/core/traces.py b/ee/api/chalicelib/core/traces.py index 04194a237..a6f9e52ca 100644 --- a/ee/api/chalicelib/core/traces.py +++ b/ee/api/chalicelib/core/traces.py @@ -71,17 +71,6 @@ def __process_trace(trace: TraceSchema): return data -async def write_trace(trace: TraceSchema): - data = __process_trace(trace) - with pg_client.PostgresClient() as cur: - cur.execute( - cur.mogrify( - f"""INSERT INTO traces(user_id, tenant_id, created_at, auth, action, method, path_format, endpoint, payload, parameters, status) - VALUES (%(user_id)s, %(tenant_id)s, %(created_at)s, %(auth)s, %(action)s, %(method)s, %(path_format)s, %(endpoint)s, %(payload)s::jsonb, %(parameters)s::jsonb, %(status)s);""", - data) - ) - - async def write_traces_batch(traces: List[TraceSchema]): if len(traces) == 0: return diff --git a/ee/api/chalicelib/utils/assist_helper.py b/ee/api/chalicelib/utils/assist_helper.py index 061b329ef..0173e17b8 100644 --- a/ee/api/chalicelib/utils/assist_helper.py +++ b/ee/api/chalicelib/utils/assist_helper.py @@ -5,7 +5,7 @@ from time import time from decouple import config -from chalicelib.core import assist +from chalicelib.core import assist_ice from chalicelib.utils import helper_ee @@ -29,7 +29,7 @@ def get_temporary_credentials(): def get_full_config(): - servers = assist.get_ice_servers() + servers = assist_ice.get_ice_servers() if servers is None: return None servers = servers.split("|") diff --git a/ee/api/chalicelib/utils/exp_ch_helper.py b/ee/api/chalicelib/utils/exp_ch_helper.py index 31115d3f5..e5eccd012 100644 --- a/ee/api/chalicelib/utils/exp_ch_helper.py +++ b/ee/api/chalicelib/utils/exp_ch_helper.py @@ -26,10 +26,6 @@ def get_main_sessions_table(timestamp=0): and timestamp and timestamp >= TimeUTC.now(delta_days=-7) else "experimental.sessions" -def get_autocomplete_table(timestamp=0): - return "experimental.autocomplete" - - def get_user_favorite_sessions_table(timestamp=0): return "experimental.user_favorite_sessions"