From f0488edf831f2061179eb7cf1b5b122cb48fbf8f Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Tue, 9 Apr 2024 16:56:56 +0200 Subject: [PATCH] Updated patch build from main 41318269f7551cffa61171b27e98cf219a4b3be2 (#2069) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(buil): Cherrypicking build script * fix(chalice): fixed mouse_thrashing title (#2014) * fix(chalice): fixed vault with exp_search * refactor(chalice): enhanced CH exception handler * fix(chalice): fixed table of URLs-values not filtered according to the specified sessions' filters CH (#2055) * fix(chalice): fixed cards-table error (#2057) * fix(chalice): fixed 1 stage results VS multi-stages result (#2060) * fix(chalice): fixed funnels negative-filter's operators (#2061) * refactor(chalice): changed JWT_REFRESH_EXPIRATION default value (#2062) * refactor(chalice): delete global session notes (#2064) * fix(chalice): support issues step-filters and tab-filters at the same… (#2065) * fix(chalice): support issues step-filters and tab-filters at the same time * Increment chalice chart version --------- Co-authored-by: Taha Yassine Kraiem Co-authored-by: GitHub Action --- api/chalicelib/core/sessions.py | 46 +- api/chalicelib/core/sessions_notes.py | 5 +- api/chalicelib/core/significance.py | 65 +- api/chalicelib/utils/helper.py | 3 +- api/env.dev | 2 +- api/routers/core_dynamic.py | 3 +- api/schemas/schemas.py | 5 +- ee/api/.gitignore | 1 + ee/api/chalicelib/core/sessions_exp.py | 76 ++- ee/api/chalicelib/core/sessions_notes.py | 5 +- ee/api/chalicelib/core/significance.py | 619 ----------------- ee/api/chalicelib/core/significance_exp.py | 620 +----------------- ee/api/chalicelib/utils/ch_client.py | 7 +- ee/api/clean-dev.sh | 1 + ee/api/routers/core_dynamic.py | 3 +- .../openreplay/charts/chalice/Chart.yaml | 5 +- 16 files changed, 145 insertions(+), 1321 deletions(-) delete mode 100644 ee/api/chalicelib/core/significance.py diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index dff3e8f5d..6c7314103 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -282,14 +282,31 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de step_size = int(metrics_helper.__get_step_size(endTimestamp=data.endTimestamp, startTimestamp=data.startTimestamp, density=density, factor=1, decimal=True)) extra_event = None + extra_conditions = None if metric_of == schemas.MetricOfTable.visited_url: extra_event = "events.pages" + extra_conditions = {} + for e in data.events: + if e.type == schemas.EventType.location: + if e.operator not in extra_conditions: + extra_conditions[e.operator] = schemas.SessionSearchEventSchema2.model_validate({ + "type": e.type, + "isEvent": True, + "value": [], + "operator": e.operator, + "filters": [] + }) + for v in e.value: + if v not in extra_conditions[e.operator].value: + extra_conditions[e.operator].value.append(v) + extra_conditions = list(extra_conditions.values()) + elif metric_of == schemas.MetricOfTable.issues and len(metric_value) > 0: data.filters.append(schemas.SessionSearchFilterSchema(value=metric_value, type=schemas.FilterType.issue, operator=schemas.SearchEventOperator._is)) full_args, query_part = search_query_parts(data=data, error_status=None, errors_only=False, favorite_only=False, issue=None, project_id=project_id, - user_id=None, extra_event=extra_event) + user_id=None, extra_event=extra_event, extra_conditions=extra_conditions) full_args["step_size"] = step_size with pg_client.PostgresClient() as cur: if isinstance(metric_of, schemas.MetricOfTable): @@ -400,14 +417,7 @@ def __is_valid_event(is_any: bool, event: schemas.SessionSearchEventSchema2): # this function generates the query and return the generated-query with the dict of query arguments def search_query_parts(data: schemas.SessionsSearchPayloadSchema, error_status, errors_only, favorite_only, issue, - project_id, user_id, platform="web", extra_event=None): - if issue: - data.filters.append( - schemas.SessionSearchFilterSchema(value=[issue['type']], - type=schemas.FilterType.issue.value, - operator='is') - ) - + project_id, user_id, platform="web", extra_event=None, extra_conditions=None): ss_constraints = [] full_args = {"project_id": project_id, "startDate": data.startTimestamp, "endDate": data.endTimestamp, "projectId": project_id, "userId": user_id} @@ -1092,6 +1102,24 @@ def search_query_parts(data: schemas.SessionsSearchPayloadSchema, error_status, extra_join += f"""INNER JOIN {extra_event} AS ev USING(session_id)""" extra_constraints.append("ev.timestamp>=%(startDate)s") extra_constraints.append("ev.timestamp<=%(endDate)s") + if extra_conditions and len(extra_conditions) > 0: + _extra_or_condition = [] + for i, c in enumerate(extra_conditions): + if sh.isAny_opreator(c.operator): + continue + e_k = f"ec_value{i}" + op = sh.get_sql_operator(c.operator) + c.value = helper.values_for_operator(value=c.value, op=c.operator) + full_args = {**full_args, + **sh.multi_values(c.value, value_key=e_k)} + if c.type == events.EventType.LOCATION.ui_type: + _extra_or_condition.append( + sh.multi_conditions(f"ev.{events.EventType.LOCATION.column} {op} %({e_k})s", + c.value, value_key=e_k)) + else: + logging.warning(f"unsupported extra_event type:${c.type}") + if len(_extra_or_condition) > 0: + extra_constraints.append("(" + " OR ".join(_extra_or_condition) + ")") query_part = f"""\ FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else "public.sessions AS s"} {extra_join} diff --git a/api/chalicelib/core/sessions_notes.py b/api/chalicelib/core/sessions_notes.py index 89e5c6d3d..4bc2138e1 100644 --- a/api/chalicelib/core/sessions_notes.py +++ b/api/chalicelib/core/sessions_notes.py @@ -125,16 +125,15 @@ def edit(tenant_id, user_id, project_id, note_id, data: schemas.SessionUpdateNot return {"errors": ["Note not found"]} -def delete(tenant_id, user_id, project_id, note_id): +def delete(project_id, note_id): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify(""" UPDATE public.sessions_notes SET deleted_at = timezone('utc'::text, now()) WHERE note_id = %(note_id)s AND project_id = %(project_id)s - AND user_id = %(user_id)s AND deleted_at ISNULL;""", - {"project_id": project_id, "user_id": user_id, "note_id": note_id}) + {"project_id": project_id, "note_id": note_id}) ) return {"data": {"state": "success"}} diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index 2418f816a..90dbcec5f 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -1,10 +1,7 @@ -__author__ = "AZNAUROV David" -__maintainer__ = "KRAIEM Taha Yassine" - import logging import schemas -from chalicelib.core import events, metadata, sessions +from chalicelib.core import events, metadata from chalicelib.utils import sql_helper as sh """ @@ -57,30 +54,27 @@ def get_stages_and_events(filter_d: schemas.CardSeriesFilterSchema, project_id) op = sh.get_sql_operator(f.operator) filter_type = f.type - # values[f_k] = sessions.__get_sql_value_multiple(f["value"]) f_k = f"f_value{i}" values = {**values, - **sh.multi_values(helper.values_for_operator(value=f.value, op=f.operator), - value_key=f_k)} + **sh.multi_values(f.value, value_key=f_k)} + is_not = False + if sh.is_negation_operator(f.operator): + is_not = True if filter_type == schemas.FilterType.user_browser: - # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_browser {op} %({f_k})s', f.value, value_key=f_k)) + sh.multi_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: - # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_os {op} %({f_k})s', f.value, value_key=f_k)) + sh.multi_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: - # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_device {op} %({f_k})s', f.value, value_key=f_k)) + sh.multi_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: - # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_country {op} %({f_k})s', f.value, value_key=f_k)) + sh.multi_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type == schemas.FilterType.duration: if len(f.value) > 0 and f.value[0] is not None: first_stage_extra_constraints.append(f's.duration >= %(minDuration)s') @@ -91,35 +85,30 @@ def get_stages_and_events(filter_d: schemas.CardSeriesFilterSchema, project_id) elif filter_type == schemas.FilterType.referrer: # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" filter_extra_from = [f"INNER JOIN {events.EventType.LOCATION.table} AS p USING(session_id)"] - # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( - sh.multi_conditions(f"p.base_referrer {op} %({f_k})s", f.value, value_key=f_k)) + sh.multi_conditions(f"p.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) elif filter_type == events.EventType.METADATA.ui_type: if meta_keys is None: meta_keys = metadata.get(project_id=project_id) meta_keys = {m["key"]: m["index"] for m in meta_keys} - # op = sessions.__get_sql_operator(f["operator"]) if f.source in meta_keys.keys(): first_stage_extra_constraints.append( sh.multi_conditions( f's.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s', f.value, - value_key=f_k)) + is_not=is_not, value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: - # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_id {op} %({f_k})s', f.value, value_key=f_k)) + sh.multi_conditions(f's.user_id {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) elif filter_type in [schemas.FilterType.user_anonymous_id, schemas.FilterType.user_anonymous_id_ios]: - # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_anonymous_id {op} %({f_k})s', f.value, value_key=f_k)) + sh.multi_conditions(f's.user_anonymous_id {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: - # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( - sh.multi_conditions(f's.rev_id {op} %({f_k})s', f.value, value_key=f_k)) + sh.multi_conditions(f's.rev_id {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) i = -1 for s in stages: @@ -553,35 +542,11 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False) def get_top_insights(filter_d: schemas.CardSeriesFilterSchema, project_id): output = [] stages = filter_d.events - # TODO: handle 1 stage alone + if len(stages) == 0: logging.debug("no stages found") return output, 0 - elif len(stages) == 1: - # TODO: count sessions, and users for single stage - output = [{ - "type": stages[0].type, - "value": stages[0].value, - "dropPercentage": None, - "operator": stages[0].operator, - "sessionsCount": 0, - "dropPct": 0, - "usersCount": 0, - "dropDueToIssues": 0 - }] - # original - # counts = sessions.search_sessions(data=schemas.SessionsSearchCountSchema.parse_obj(filter_d), - # project_id=project_id, user_id=None, count_only=True) - # first change - # counts = sessions.search_sessions(data=schemas.FlatSessionsSearchPayloadSchema.parse_obj(filter_d), - # project_id=project_id, user_id=None, count_only=True) - # last change - counts = sessions.search_sessions(data=schemas.SessionsSearchPayloadSchema.model_validate(filter_d), - project_id=project_id, user_id=None, count_only=True) - output[0]["sessionsCount"] = counts["countSessions"] - output[0]["usersCount"] = counts["countUsers"] - return output, 0 # The result of the multi-stage query rows = get_stages_and_events(filter_d=filter_d, project_id=project_id) if len(rows) == 0: diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py index 5370c8f67..08a8ce286 100644 --- a/api/chalicelib/utils/helper.py +++ b/api/chalicelib/utils/helper.py @@ -249,7 +249,8 @@ def get_issue_title(issue_type): 'custom': "Custom Event", 'js_exception': "Error", 'custom_event_error': "Custom Error", - 'js_error': "Error"}.get(issue_type, issue_type) + 'js_error': "Error", + "mouse_thrashing": "Mouse Thrashing"}.get(issue_type, issue_type) def __progress(old_val, new_val): diff --git a/api/env.dev b/api/env.dev index ab74f5c0b..33ac05dcf 100644 --- a/api/env.dev +++ b/api/env.dev @@ -29,7 +29,7 @@ js_cache_bucket= jwt_algorithm=HS512 JWT_EXPIRATION=6000 JWT_ISSUER=openReplay-dev -JWT_REFRESH_EXPIRATION=60 +JWT_REFRESH_EXPIRATION=604800 JWT_REFRESH_SECRET=SECRET2 jwt_secret=SECRET LOCAL_DEV=true diff --git a/api/routers/core_dynamic.py b/api/routers/core_dynamic.py index dfadd05c7..a8a615f1f 100644 --- a/api/routers/core_dynamic.py +++ b/api/routers/core_dynamic.py @@ -481,8 +481,7 @@ def edit_note(projectId: int, noteId: int, data: schemas.SessionUpdateNoteSchema @app.delete('/{projectId}/notes/{noteId}', tags=["sessions", "notes"]) def delete_note(projectId: int, noteId: int, _=Body(None), context: schemas.CurrentContext = Depends(OR_context)): - data = sessions_notes.delete(tenant_id=context.tenant_id, project_id=projectId, user_id=context.user_id, - note_id=noteId) + data = sessions_notes.delete(project_id=projectId, note_id=noteId) return data diff --git a/api/schemas/schemas.py b/api/schemas/schemas.py index cdb71ee75..3f4d9c0ce 100644 --- a/api/schemas/schemas.py +++ b/api/schemas/schemas.py @@ -784,9 +784,12 @@ class SessionsSearchPayloadSchema(_TimedSchema, _PaginatedSchema): @field_validator("filters", mode="after") def merge_identical_filters(cls, values): + # ignore 'issue' type as it could be used for step-filters and tab-filters at the same time i = 0 while i < len(values): - if values[i].is_event: + if values[i].is_event or values[i].type == FilterType.issue: + if values[i].type == FilterType.issue: + values[i] = remove_duplicate_values(values[i]) i += 1 continue j = i + 1 diff --git a/ee/api/.gitignore b/ee/api/.gitignore index a0743c6db..9fd6db379 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -227,6 +227,7 @@ Pipfile.lock /chalicelib/core/sessions.py /chalicelib/core/sessions_assignments.py /chalicelib/core/sessions_mobs.py +/chalicelib/core/significance.py /chalicelib/core/socket_ios.py /chalicelib/core/sourcemaps.py /chalicelib/core/sourcemaps_parser.py diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index 7c498142c..1fdeccb6c 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -3,7 +3,7 @@ import logging from typing import List, Union import schemas -from chalicelib.core import events, metadata, projects, performance_event, metrics +from chalicelib.core import events, metadata, projects, performance_event, metrics, sessions_favorite, sessions_legacy from chalicelib.utils import pg_client, helper, metrics_helper, ch_client, exp_ch_helper logger = logging.getLogger(__name__) @@ -110,6 +110,8 @@ def _isUndefined_operator(op: schemas.SearchEventOperator): def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False, platform="web"): + if data.bookmarked: + data.startTimestamp, data.endTimestamp = sessions_favorite.get_start_end_timestamp(project_id, user_id) full_args, query_part = search_query_parts_ch(data=data, error_status=error_status, errors_only=errors_only, favorite_only=data.bookmarked, issue=issue, project_id=project_id, user_id=user_id, platform=platform) @@ -354,6 +356,7 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de density=density)) extra_event = None extra_deduplication = [] + extra_conditions = None if metric_of == schemas.MetricOfTable.visited_url: extra_event = f"""SELECT DISTINCT ev.session_id, ev.url_path FROM {exp_ch_helper.get_main_events_table(data.startTimestamp)} AS ev @@ -362,13 +365,30 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de AND ev.project_id = %(project_id)s AND ev.event_type = 'LOCATION'""" extra_deduplication.append("url_path") + extra_conditions = {} + for e in data.events: + if e.type == schemas.EventType.location: + if e.operator not in extra_conditions: + extra_conditions[e.operator] = schemas.SessionSearchEventSchema2.model_validate({ + "type": e.type, + "isEvent": True, + "value": [], + "operator": e.operator, + "filters": [] + }) + for v in e.value: + if v not in extra_conditions[e.operator].value: + extra_conditions[e.operator].value.append(v) + extra_conditions = list(extra_conditions.values()) + elif metric_of == schemas.MetricOfTable.issues and len(metric_value) > 0: data.filters.append(schemas.SessionSearchFilterSchema(value=metric_value, type=schemas.FilterType.issue, operator=schemas.SearchEventOperator._is)) full_args, query_part = search_query_parts_ch(data=data, error_status=None, errors_only=False, favorite_only=False, issue=None, project_id=project_id, user_id=None, extra_event=extra_event, - extra_deduplication=extra_deduplication) + extra_deduplication=extra_deduplication, + extra_conditions=extra_conditions) full_args["step_size"] = step_size sessions = [] with ch_client.ClickHouseClient() as cur: @@ -521,7 +541,8 @@ def __get_event_type(event_type: Union[schemas.EventType, schemas.PerformanceEve # this function generates the query and return the generated-query with the dict of query arguments def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_status, errors_only, favorite_only, issue, - project_id, user_id, platform="web", extra_event=None, extra_deduplication=[]): + project_id, user_id, platform="web", extra_event=None, extra_deduplication=[], + extra_conditions=None): if issue: data.filters.append( schemas.SessionSearchFilterSchema(value=[issue['type']], @@ -1462,7 +1483,7 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu AND events.issue_type = %(issue_type)s AND events.datetime >= toDateTime(%(startDate)s/1000) AND events.datetime <= toDateTime(%(endDate)s/1000) - ) AS issues ON (s.session_id = issues.session_id) + ) AS issues ON (f.session_id = issues.session_id) """ full_args["issue_contextString"] = issue["contextString"] full_args["issue_type"] = issue["type"] @@ -1487,9 +1508,24 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu if extra_event: extra_event = f"INNER JOIN ({extra_event}) AS extra_event USING(session_id)" - # extra_join = f"""INNER JOIN {extra_event} AS ev USING(session_id)""" - # extra_constraints.append("ev.timestamp>=%(startDate)s") - # extra_constraints.append("ev.timestamp<=%(endDate)s") + if extra_conditions and len(extra_conditions) > 0: + _extra_or_condition = [] + for i, c in enumerate(extra_conditions): + if _isAny_opreator(c.operator): + continue + e_k = f"ec_value{i}" + op = __get_sql_operator(c.operator) + c.value = helper.values_for_operator(value=c.value, op=c.operator) + full_args = {**full_args, + **_multiple_values(c.value, value_key=e_k)} + if c.type == events.EventType.LOCATION.ui_type: + _extra_or_condition.append( + _multiple_conditions(f"extra_event.url_path {op} %({e_k})s", + c.value, value_key=e_k)) + else: + logging.warning(f"unsupported extra_event type:${c.type}") + if len(_extra_or_condition) > 0: + extra_constraints.append("(" + " OR ".join(_extra_or_condition) + ")") else: extra_event = "" if errors_only: @@ -1679,3 +1715,29 @@ def check_recording_status(project_id: int) -> dict: "recordingStatus": row["recording_status"], "sessionsCount": row["sessions_count"] } + + +# TODO: rewrite this function to use ClickHouse +def search_sessions_by_ids(project_id: int, session_ids: list, sort_by: str = 'session_id', + ascending: bool = False) -> dict: + if session_ids is None or len(session_ids) == 0: + return {"total": 0, "sessions": []} + with pg_client.PostgresClient() as cur: + meta_keys = metadata.get(project_id=project_id) + params = {"project_id": project_id, "session_ids": tuple(session_ids)} + order_direction = 'ASC' if ascending else 'DESC' + main_query = cur.mogrify(f"""SELECT {sessions_legacy.SESSION_PROJECTION_BASE_COLS} + {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} + FROM public.sessions AS s + WHERE project_id=%(project_id)s + AND session_id IN %(session_ids)s + ORDER BY {sort_by} {order_direction};""", params) + + cur.execute(main_query) + rows = cur.fetchall() + if len(meta_keys) > 0: + for s in rows: + s["metadata"] = {} + for m in meta_keys: + s["metadata"][m["key"]] = s.pop(f'metadata_{m["index"]}') + return {"total": len(rows), "sessions": helper.list_to_camel_case(rows)} diff --git a/ee/api/chalicelib/core/sessions_notes.py b/ee/api/chalicelib/core/sessions_notes.py index 46c974c1a..97ab14f9c 100644 --- a/ee/api/chalicelib/core/sessions_notes.py +++ b/ee/api/chalicelib/core/sessions_notes.py @@ -128,16 +128,15 @@ def edit(tenant_id, user_id, project_id, note_id, data: schemas.SessionUpdateNot return row -def delete(tenant_id, user_id, project_id, note_id): +def delete(project_id, note_id): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify(""" UPDATE public.sessions_notes SET deleted_at = timezone('utc'::text, now()) WHERE note_id = %(note_id)s AND project_id = %(project_id)s - AND user_id = %(user_id)s AND deleted_at ISNULL;""", - {"project_id": project_id, "user_id": user_id, "note_id": note_id}) + {"project_id": project_id, "note_id": note_id}) ) return {"data": {"state": "success"}} diff --git a/ee/api/chalicelib/core/significance.py b/ee/api/chalicelib/core/significance.py deleted file mode 100644 index 0d5f1b70d..000000000 --- a/ee/api/chalicelib/core/significance.py +++ /dev/null @@ -1,619 +0,0 @@ -__author__ = "AZNAUROV David" -__maintainer__ = "KRAIEM Taha Yassine" - -import logging - -from decouple import config - -import schemas -from chalicelib.core import events, metadata -from chalicelib.utils import sql_helper as sh - -if config("EXP_SESSIONS_SEARCH", cast=bool, default=False): - from chalicelib.core import sessions_legacy as sessions -else: - from chalicelib.core import sessions - -""" -todo: remove LIMIT from the query -""" - -from typing import List -import math -import warnings -from collections import defaultdict - -from psycopg2.extras import RealDictRow -from chalicelib.utils import pg_client, helper - -logger = logging.getLogger(__name__) -SIGNIFICANCE_THRSH = 0.4 -# Taha: the value 24 was estimated in v1.15 -T_VALUES = {1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228, - 11: 2.201, 12: 2.179, 13: 2.160, 14: 2.145, 15: 2.13, 16: 2.120, 17: 2.110, 18: 2.101, 19: 2.093, 20: 2.086, - 21: 2.080, 22: 2.074, 23: 2.069, 24: 2.067, 25: 2.064, 26: 2.060, 27: 2.056, 28: 2.052, 29: 2.045, - 30: 2.042} - - -def get_stages_and_events(filter_d: schemas.CardSeriesFilterSchema, project_id) -> List[RealDictRow]: - """ - Add minimal timestamp - :param filter_d: dict contains events&filters&... - :return: - """ - stages: [dict] = filter_d.events - filters: [dict] = filter_d.filters - filter_issues = [] - # TODO: enable this if needed by an endpoint - # filter_issues = filter_d.get("issueTypes") - # if filter_issues is None or len(filter_issues) == 0: - # filter_issues = [] - stage_constraints = ["main.timestamp <= %(endTimestamp)s"] - first_stage_extra_constraints = ["s.project_id=%(project_id)s", "s.start_ts >= %(startTimestamp)s", - "s.start_ts <= %(endTimestamp)s"] - filter_extra_from = [] - n_stages_query = [] - values = {} - if len(filters) > 0: - meta_keys = None - for i, f in enumerate(filters): - if len(f.value) == 0: - continue - f.value = helper.values_for_operator(value=f.value, op=f.operator) - # filter_args = _multiple_values(f["value"]) - op = sh.get_sql_operator(f.operator) - - filter_type = f.type - # values[f_k] = sessions.__get_sql_value_multiple(f["value"]) - f_k = f"f_value{i}" - values = {**values, - **sh.multi_values(helper.values_for_operator(value=f.value, op=f.operator), - value_key=f_k)} - if filter_type == schemas.FilterType.user_browser: - # op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_browser {op} %({f_k})s', f.value, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: - # op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_os {op} %({f_k})s', f.value, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: - # op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_device {op} %({f_k})s', f.value, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: - # op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_country {op} %({f_k})s', f.value, value_key=f_k)) - elif filter_type == schemas.FilterType.duration: - if len(f.value) > 0 and f.value[0] is not None: - first_stage_extra_constraints.append(f's.duration >= %(minDuration)s') - values["minDuration"] = f.value[0] - if len(f["value"]) > 1 and f.value[1] is not None and int(f.value[1]) > 0: - first_stage_extra_constraints.append('s.duration <= %(maxDuration)s') - values["maxDuration"] = f.value[1] - elif filter_type == schemas.FilterType.referrer: - # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" - filter_extra_from = [f"INNER JOIN {events.EventType.LOCATION.table} AS p USING(session_id)"] - # op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f"p.base_referrer {op} %({f_k})s", f.value, value_key=f_k)) - elif filter_type == events.EventType.METADATA.ui_type: - if meta_keys is None: - meta_keys = metadata.get(project_id=project_id) - meta_keys = {m["key"]: m["index"] for m in meta_keys} - # op = sessions.__get_sql_operator(f["operator"]) - if f.source in meta_keys.keys(): - first_stage_extra_constraints.append( - sh.multi_conditions( - f's.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s', f.value, - value_key=f_k)) - # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: - # op = sessions.__get_sql_operator(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_id {op} %({f_k})s', f.value, value_key=f_k)) - # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [schemas.FilterType.user_anonymous_id, - schemas.FilterType.user_anonymous_id_ios]: - # op = sessions.__get_sql_operator(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_anonymous_id {op} %({f_k})s', f.value, value_key=f_k)) - # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: - # op = sessions.__get_sql_operator(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.rev_id {op} %({f_k})s', f.value, value_key=f_k)) - # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - i = -1 - for s in stages: - - if s.operator is None: - s.operator = schemas.SearchEventOperator._is - - if not isinstance(s.value, list): - s.value = [s.value] - is_any = sh.isAny_opreator(s.operator) - if not is_any and isinstance(s.value, list) and len(s.value) == 0: - continue - i += 1 - if i == 0: - extra_from = filter_extra_from + ["INNER JOIN public.sessions AS s USING (session_id)"] - else: - extra_from = [] - op = sh.get_sql_operator(s.operator) - # event_type = s["type"].upper() - event_type = s.type - if event_type == events.EventType.CLICK.ui_type: - next_table = events.EventType.CLICK.table - next_col_name = events.EventType.CLICK.column - elif event_type == events.EventType.INPUT.ui_type: - next_table = events.EventType.INPUT.table - next_col_name = events.EventType.INPUT.column - elif event_type == events.EventType.LOCATION.ui_type: - next_table = events.EventType.LOCATION.table - next_col_name = events.EventType.LOCATION.column - elif event_type == events.EventType.CUSTOM.ui_type: - next_table = events.EventType.CUSTOM.table - next_col_name = events.EventType.CUSTOM.column - # IOS -------------- - elif event_type == events.EventType.CLICK_IOS.ui_type: - next_table = events.EventType.CLICK_IOS.table - next_col_name = events.EventType.CLICK_IOS.column - elif event_type == events.EventType.INPUT_IOS.ui_type: - next_table = events.EventType.INPUT_IOS.table - next_col_name = events.EventType.INPUT_IOS.column - elif event_type == events.EventType.VIEW_IOS.ui_type: - next_table = events.EventType.VIEW_IOS.table - next_col_name = events.EventType.VIEW_IOS.column - elif event_type == events.EventType.CUSTOM_IOS.ui_type: - next_table = events.EventType.CUSTOM_IOS.table - next_col_name = events.EventType.CUSTOM_IOS.column - else: - logging.warning(f"=================UNDEFINED:{event_type}") - continue - - values = {**values, **sh.multi_values(helper.values_for_operator(value=s.value, op=s.operator), - value_key=f"value{i + 1}")} - if sh.is_negation_operator(s.operator) and i > 0: - op = sh.reverse_sql_operator(op) - main_condition = "left_not.session_id ISNULL" - extra_from.append(f"""LEFT JOIN LATERAL (SELECT session_id - FROM {next_table} AS s_main - WHERE - {sh.multi_conditions(f"s_main.{next_col_name} {op} %(value{i + 1})s", - values=s.value, value_key=f"value{i + 1}")} - AND s_main.timestamp >= T{i}.stage{i}_timestamp - AND s_main.session_id = T1.session_id) AS left_not ON (TRUE)""") - else: - if is_any: - main_condition = "TRUE" - else: - main_condition = sh.multi_conditions(f"main.{next_col_name} {op} %(value{i + 1})s", - values=s.value, value_key=f"value{i + 1}") - n_stages_query.append(f""" - (SELECT main.session_id, - {"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp - FROM {next_table} AS main {" ".join(extra_from)} - WHERE main.timestamp >= {f"T{i}.stage{i}_timestamp" if i > 0 else "%(startTimestamp)s"} - {f"AND main.session_id=T1.session_id" if i > 0 else ""} - AND {main_condition} - {(" AND " + " AND ".join(stage_constraints)) if len(stage_constraints) > 0 else ""} - {(" AND " + " AND ".join(first_stage_extra_constraints)) if len(first_stage_extra_constraints) > 0 and i == 0 else ""} - GROUP BY main.session_id) - AS T{i + 1} {"ON (TRUE)" if i > 0 else ""} - """) - n_stages = len(n_stages_query) - if n_stages == 0: - return [] - n_stages_query = " LEFT JOIN LATERAL ".join(n_stages_query) - n_stages_query += ") AS stages_t" - - n_stages_query = f""" - SELECT stages_and_issues_t.*, sessions.user_uuid - FROM ( - SELECT * FROM ( - SELECT T1.session_id, {",".join([f"stage{i + 1}_timestamp" for i in range(n_stages)])} - FROM {n_stages_query} - LEFT JOIN LATERAL - ( SELECT ISS.type as issue_type, - ISE.timestamp AS issue_timestamp, - COALESCE(ISS.context_string,'') as issue_context, - ISS.issue_id as issue_id - FROM events_common.issues AS ISE INNER JOIN issues AS ISS USING (issue_id) - WHERE ISE.timestamp >= stages_t.stage1_timestamp - AND ISE.timestamp <= stages_t.stage{i + 1}_timestamp - AND ISS.project_id=%(project_id)s - AND ISE.session_id = stages_t.session_id - AND ISS.type!='custom' -- ignore custom issues because they are massive - {"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""} - LIMIT 10 -- remove the limit to get exact stats - ) AS issues_t ON (TRUE) - ) AS stages_and_issues_t INNER JOIN sessions USING(session_id); - """ - - # LIMIT 10000 - params = {"project_id": project_id, "startTimestamp": filter_d.startTimestamp, - "endTimestamp": filter_d.endTimestamp, - "issueTypes": tuple(filter_issues), **values} - with pg_client.PostgresClient() as cur: - query = cur.mogrify(n_stages_query, params) - logging.debug("---------------------------------------------------") - logging.debug(query) - logging.debug("---------------------------------------------------") - try: - cur.execute(query) - rows = cur.fetchall() - except Exception as err: - logging.warning("--------- FUNNEL SEARCH QUERY EXCEPTION -----------") - logging.warning(query.decode('UTF-8')) - logging.warning("--------- PAYLOAD -----------") - logging.warning(filter_d.model_dump_json()) - logging.warning("--------------------") - raise err - return rows - - -def pearson_corr(x: list, y: list): - n = len(x) - if n != len(y): - raise ValueError(f'x and y must have the same length. Got {len(x)} and {len(y)} instead') - - if n < 2: - warnings.warn(f'x and y must have length at least 2. Got {n} instead') - return None, None, False - - # If an input is constant, the correlation coefficient is not defined. - if all(t == x[0] for t in x) or all(t == y[0] for t in y): - warnings.warn("An input array is constant; the correlation coefficent is not defined.") - return None, None, False - - if n == 2: - return math.copysign(1, x[1] - x[0]) * math.copysign(1, y[1] - y[0]), 1.0, True - - xmean = sum(x) / len(x) - ymean = sum(y) / len(y) - - xm = [el - xmean for el in x] - ym = [el - ymean for el in y] - - normxm = math.sqrt((sum([xm[i] * xm[i] for i in range(len(xm))]))) - normym = math.sqrt((sum([ym[i] * ym[i] for i in range(len(ym))]))) - - threshold = 1e-8 - if normxm < threshold * abs(xmean) or normym < threshold * abs(ymean): - # If all the values in x (likewise y) are very close to the mean, - # the loss of precision that occurs in the subtraction xm = x - xmean - # might result in large errors in r. - warnings.warn("An input array is constant; the correlation coefficent is not defined.") - - r = sum( - i[0] * i[1] for i in zip([xm[i] / normxm for i in range(len(xm))], [ym[i] / normym for i in range(len(ym))])) - - # Presumably, if abs(r) > 1, then it is only some small artifact of floating point arithmetic. - # However, if r < 0, we don't care, as our problem is to find only positive correlations - r = max(min(r, 1.0), 0.0) - - # approximated confidence - if n < 31: - t_c = T_VALUES[n] - elif n < 50: - t_c = 2.02 - else: - t_c = 2 - if r >= 0.999: - confidence = 1 - else: - confidence = r * math.sqrt(n - 2) / math.sqrt(1 - r ** 2) - - if confidence > SIGNIFICANCE_THRSH: - return r, confidence, True - else: - return r, confidence, False - - -# def tuple_or(t: tuple): -# x = 0 -# for el in t: -# x |= el # | is for bitwise OR -# return x -# -# The following function is correct optimization of the previous function because t is a list of 0,1 -def tuple_or(t: tuple): - for el in t: - if el > 0: - return 1 - return 0 - - -def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues, first_stage, last_stage): - """ - Returns two lists with binary values 0/1: - - transitions ::: if transited from the first stage to the last - 1 - else - 0 - errors ::: a dictionary WHERE the keys are all unique issues (currently context-wise) - the values are lists - if an issue happened between the first stage to the last - 1 - else - 0 - - For a small task of calculating a total drop due to issues, - we need to disregard the issue type when creating the `errors`-like array. - The `all_errors` array can be obtained by logical OR statement applied to all errors by issue - The `transitions` array stays the same - """ - transitions = [] - n_sess_affected = 0 - errors = {} - - for row in rows: - t = 0 - first_ts = row[f'stage{first_stage}_timestamp'] - last_ts = row[f'stage{last_stage}_timestamp'] - if first_ts is None: - continue - elif last_ts is not None: - t = 1 - transitions.append(t) - - ic_present = False - for error_id in all_issues: - if error_id not in errors: - errors[error_id] = [] - ic = 0 - row_issue_id = row['issue_id'] - if row_issue_id is not None: - if last_ts is None or (first_ts < row['issue_timestamp'] < last_ts): - if error_id == row_issue_id: - ic = 1 - ic_present = True - errors[error_id].append(ic) - - if ic_present and t: - n_sess_affected += 1 - - all_errors = [tuple_or(t) for t in zip(*errors.values())] - - return transitions, errors, all_errors, n_sess_affected - - -def get_affected_users_for_all_issues(rows, first_stage, last_stage): - """ - - :param rows: - :param first_stage: - :param last_stage: - :return: - """ - affected_users = defaultdict(lambda: set()) - affected_sessions = defaultdict(lambda: set()) - all_issues = {} - n_affected_users_dict = defaultdict(lambda: None) - n_affected_sessions_dict = defaultdict(lambda: None) - n_issues_dict = defaultdict(lambda: 0) - issues_by_session = defaultdict(lambda: 0) - - for row in rows: - - # check that the session has reached the first stage of subfunnel: - if row[f'stage{first_stage}_timestamp'] is None: - continue - - iss = row['issue_type'] - iss_ts = row['issue_timestamp'] - - # check that the issue exists and belongs to subfunnel: - if iss is not None and (row[f'stage{last_stage}_timestamp'] is None or - (row[f'stage{first_stage}_timestamp'] < iss_ts < row[f'stage{last_stage}_timestamp'])): - if row["issue_id"] not in all_issues: - all_issues[row["issue_id"]] = {"context": row['issue_context'], "issue_type": row["issue_type"]} - n_issues_dict[row["issue_id"]] += 1 - if row['user_uuid'] is not None: - affected_users[row["issue_id"]].add(row['user_uuid']) - - affected_sessions[row["issue_id"]].add(row['session_id']) - issues_by_session[row[f'session_id']] += 1 - - if len(affected_users) > 0: - n_affected_users_dict.update({ - iss: len(affected_users[iss]) for iss in affected_users - }) - if len(affected_sessions) > 0: - n_affected_sessions_dict.update({ - iss: len(affected_sessions[iss]) for iss in affected_sessions - }) - return all_issues, n_issues_dict, n_affected_users_dict, n_affected_sessions_dict - - -def count_sessions(rows, n_stages): - session_counts = {i: set() for i in range(1, n_stages + 1)} - for row in rows: - for i in range(1, n_stages + 1): - if row[f"stage{i}_timestamp"] is not None: - session_counts[i].add(row[f"session_id"]) - - session_counts = {i: len(session_counts[i]) for i in session_counts} - return session_counts - - -def count_users(rows, n_stages): - users_in_stages = {i: set() for i in range(1, n_stages + 1)} - for row in rows: - for i in range(1, n_stages + 1): - if row[f"stage{i}_timestamp"] is not None: - users_in_stages[i].add(row["user_uuid"]) - - users_count = {i: len(users_in_stages[i]) for i in range(1, n_stages + 1)} - return users_count - - -def get_stages(stages, rows): - n_stages = len(stages) - session_counts = count_sessions(rows, n_stages) - users_counts = count_users(rows, n_stages) - - stages_list = [] - for i, stage in enumerate(stages): - - drop = None - if i != 0: - if session_counts[i] == 0: - drop = 0 - elif session_counts[i] > 0: - drop = int(100 * (session_counts[i] - session_counts[i + 1]) / session_counts[i]) - - stages_list.append( - {"value": stage.value, - "type": stage.type, - "operator": stage.operator, - "sessionsCount": session_counts[i + 1], - "drop_pct": drop, - "usersCount": users_counts[i + 1], - "dropDueToIssues": 0 - } - ) - return stages_list - - -def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False): - """ - - :param stages: - :param rows: - :param first_stage: If it's a part of the initial funnel, provide a number of the first stage (starting from 1) - :param last_stage: If it's a part of the initial funnel, provide a number of the last stage (starting from 1) - :return: - """ - - n_stages = len(stages) - - if first_stage is None: - first_stage = 1 - if last_stage is None: - last_stage = n_stages - if last_stage > n_stages: - logging.debug( - "The number of the last stage provided is greater than the number of stages. Using n_stages instead") - last_stage = n_stages - - n_critical_issues = 0 - issues_dict = {"significant": [], - "insignificant": []} - session_counts = count_sessions(rows, n_stages) - drop = session_counts[first_stage] - session_counts[last_stage] - - all_issues, n_issues_dict, affected_users_dict, affected_sessions = get_affected_users_for_all_issues( - rows, first_stage, last_stage) - transitions, errors, all_errors, n_sess_affected = get_transitions_and_issues_of_each_type(rows, - all_issues, - first_stage, last_stage) - - del rows - - if any(all_errors): - total_drop_corr, conf, is_sign = pearson_corr(transitions, all_errors) - if total_drop_corr is not None and drop is not None: - total_drop_due_to_issues = int(total_drop_corr * n_sess_affected) - else: - total_drop_due_to_issues = 0 - else: - total_drop_due_to_issues = 0 - - if drop_only: - return total_drop_due_to_issues - for issue_id in all_issues: - - if not any(errors[issue_id]): - continue - r, confidence, is_sign = pearson_corr(transitions, errors[issue_id]) - - if r is not None and drop is not None and is_sign: - lost_conversions = int(r * affected_sessions[issue_id]) - else: - lost_conversions = None - if r is None: - r = 0 - issues_dict['significant' if is_sign else 'insignificant'].append({ - "type": all_issues[issue_id]["issue_type"], - "title": helper.get_issue_title(all_issues[issue_id]["issue_type"]), - "affected_sessions": affected_sessions[issue_id], - "unaffected_sessions": session_counts[1] - affected_sessions[issue_id], - "lost_conversions": lost_conversions, - "affected_users": affected_users_dict[issue_id], - "conversion_impact": round(r * 100), - "context_string": all_issues[issue_id]["context"], - "issue_id": issue_id - }) - - if is_sign: - n_critical_issues += n_issues_dict[issue_id] - # To limit the number of returned issues to the frontend - issues_dict["significant"] = issues_dict["significant"][:20] - issues_dict["insignificant"] = issues_dict["insignificant"][:20] - - return n_critical_issues, issues_dict, total_drop_due_to_issues - - -def get_top_insights(filter_d: schemas.CardSeriesFilterSchema, project_id): - output = [] - stages = filter_d.events - # TODO: handle 1 stage alone - if len(stages) == 0: - logging.debug("no stages found") - return output, 0 - elif len(stages) == 1: - # TODO: count sessions, and users for single stage - output = [{ - "type": stages[0].type, - "value": stages[0].value, - "dropPercentage": None, - "operator": stages[0].operator, - "sessionsCount": 0, - "dropPct": 0, - "usersCount": 0, - "dropDueToIssues": 0 - - }] - # original - # counts = sessions.search_sessions(data=schemas.SessionsSearchCountSchema.parse_obj(filter_d), - # project_id=project_id, user_id=None, count_only=True) - # first change - # counts = sessions.search_sessions(data=schemas.FlatSessionsSearchPayloadSchema.parse_obj(filter_d), - # project_id=project_id, user_id=None, count_only=True) - # last change - counts = sessions.search_sessions(data=schemas.SessionsSearchPayloadSchema.model_validate(filter_d), - project_id=project_id, user_id=None, count_only=True) - output[0]["sessionsCount"] = counts["countSessions"] - output[0]["usersCount"] = counts["countUsers"] - return output, 0 - # The result of the multi-stage query - rows = get_stages_and_events(filter_d=filter_d, project_id=project_id) - if len(rows) == 0: - return get_stages(stages, []), 0 - # Obtain the first part of the output - stages_list = get_stages(stages, rows) - # Obtain the second part of the output - total_drop_due_to_issues = get_issues(stages, rows, - first_stage=1, - last_stage=len(filter_d.events), - drop_only=True) - return stages_list, total_drop_due_to_issues - - -def get_issues_list(filter_d: schemas.CardSeriesFilterSchema, project_id, first_stage=None, last_stage=None): - output = dict({"total_drop_due_to_issues": 0, "critical_issues_count": 0, "significant": [], "insignificant": []}) - stages = filter_d.events - # The result of the multi-stage query - rows = get_stages_and_events(filter_d=filter_d, project_id=project_id) - if len(rows) == 0: - return output - # Obtain the second part of the output - n_critical_issues, issues_dict, total_drop_due_to_issues = get_issues(stages, rows, first_stage=first_stage, - last_stage=last_stage) - output['total_drop_due_to_issues'] = total_drop_due_to_issues - # output['critical_issues_count'] = n_critical_issues - output = {**output, **issues_dict} - return output diff --git a/ee/api/chalicelib/core/significance_exp.py b/ee/api/chalicelib/core/significance_exp.py index 058998d3f..132ff881f 100644 --- a/ee/api/chalicelib/core/significance_exp.py +++ b/ee/api/chalicelib/core/significance_exp.py @@ -1,618 +1,2 @@ -__maintainer__ = "KRAIEM Taha Yassine" - -import logging - -from decouple import config - -import schemas -from chalicelib.core import events, metadata -from chalicelib.utils import sql_helper as sh - -if config("EXP_SESSIONS_SEARCH", cast=bool, default=False): - from chalicelib.core import sessions_legacy as sessions -else: - from chalicelib.core import sessions - -""" -todo: remove LIMIT from the query -""" - -from typing import List -import math -import warnings -from collections import defaultdict - -from psycopg2.extras import RealDictRow -from chalicelib.utils import pg_client, helper - -logger = logging.getLogger(__name__) -SIGNIFICANCE_THRSH = 0.4 -# Taha: the value 24 was estimated in v1.15 -T_VALUES = {1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228, - 11: 2.201, 12: 2.179, 13: 2.160, 14: 2.145, 15: 2.13, 16: 2.120, 17: 2.110, 18: 2.101, 19: 2.093, 20: 2.086, - 21: 2.080, 22: 2.074, 23: 2.069, 24: 2.067, 25: 2.064, 26: 2.060, 27: 2.056, 28: 2.052, 29: 2.045, - 30: 2.042} - - -def get_stages_and_events(filter_d: schemas.CardSeriesFilterSchema, project_id) -> List[RealDictRow]: - """ - Add minimal timestamp - :param filter_d: dict contains events&filters&... - :return: - """ - stages: [dict] = filter_d.events - filters: [dict] = filter_d.filters - filter_issues = [] - # TODO: enable this if needed by an endpoint - # filter_issues = filter_d.get("issueTypes") - # if filter_issues is None or len(filter_issues) == 0: - # filter_issues = [] - stage_constraints = ["main.timestamp <= %(endTimestamp)s"] - first_stage_extra_constraints = ["s.project_id=%(project_id)s", "s.start_ts >= %(startTimestamp)s", - "s.start_ts <= %(endTimestamp)s"] - filter_extra_from = [] - n_stages_query = [] - values = {} - if len(filters) > 0: - meta_keys = None - for i, f in enumerate(filters): - if len(f.value) == 0: - continue - f.value = helper.values_for_operator(value=f.value, op=f.operator) - # filter_args = _multiple_values(f["value"]) - op = sh.get_sql_operator(f.operator) - - filter_type = f.type - # values[f_k] = sessions.__get_sql_value_multiple(f["value"]) - f_k = f"f_value{i}" - values = {**values, - **sh.multi_values(helper.values_for_operator(value=f.value, op=f.operator), - value_key=f_k)} - if filter_type == schemas.FilterType.user_browser: - # op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_browser {op} %({f_k})s', f.value, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: - # op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_os {op} %({f_k})s', f.value, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: - # op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_device {op} %({f_k})s', f.value, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: - # op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_country {op} %({f_k})s', f.value, value_key=f_k)) - elif filter_type == schemas.FilterType.duration: - if len(f.value) > 0 and f.value[0] is not None: - first_stage_extra_constraints.append(f's.duration >= %(minDuration)s') - values["minDuration"] = f.value[0] - if len(f["value"]) > 1 and f.value[1] is not None and int(f.value[1]) > 0: - first_stage_extra_constraints.append('s.duration <= %(maxDuration)s') - values["maxDuration"] = f.value[1] - elif filter_type == schemas.FilterType.referrer: - # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" - filter_extra_from = [f"INNER JOIN {events.EventType.LOCATION.table} AS p USING(session_id)"] - # op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f"p.base_referrer {op} %({f_k})s", f.value, value_key=f_k)) - elif filter_type == events.EventType.METADATA.ui_type: - if meta_keys is None: - meta_keys = metadata.get(project_id=project_id) - meta_keys = {m["key"]: m["index"] for m in meta_keys} - # op = sessions.__get_sql_operator(f["operator"]) - if f.source in meta_keys.keys(): - first_stage_extra_constraints.append( - sh.multi_conditions( - f's.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s', f.value, - value_key=f_k)) - # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: - # op = sessions.__get_sql_operator(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_id {op} %({f_k})s', f.value, value_key=f_k)) - # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [schemas.FilterType.user_anonymous_id, - schemas.FilterType.user_anonymous_id_ios]: - # op = sessions.__get_sql_operator(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.user_anonymous_id {op} %({f_k})s', f.value, value_key=f_k)) - # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: - # op = sessions.__get_sql_operator(f["operator"]) - first_stage_extra_constraints.append( - sh.multi_conditions(f's.rev_id {op} %({f_k})s', f.value, value_key=f_k)) - # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - i = -1 - for s in stages: - - if s.operator is None: - s.operator = schemas.SearchEventOperator._is - - if not isinstance(s.value, list): - s.value = [s.value] - is_any = sh.isAny_opreator(s.operator) - if not is_any and isinstance(s.value, list) and len(s.value) == 0: - continue - i += 1 - if i == 0: - extra_from = filter_extra_from + ["INNER JOIN public.sessions AS s USING (session_id)"] - else: - extra_from = [] - op = sh.get_sql_operator(s.operator) - # event_type = s["type"].upper() - event_type = s.type - if event_type == events.EventType.CLICK.ui_type: - next_table = events.EventType.CLICK.table - next_col_name = events.EventType.CLICK.column - elif event_type == events.EventType.INPUT.ui_type: - next_table = events.EventType.INPUT.table - next_col_name = events.EventType.INPUT.column - elif event_type == events.EventType.LOCATION.ui_type: - next_table = events.EventType.LOCATION.table - next_col_name = events.EventType.LOCATION.column - elif event_type == events.EventType.CUSTOM.ui_type: - next_table = events.EventType.CUSTOM.table - next_col_name = events.EventType.CUSTOM.column - # IOS -------------- - elif event_type == events.EventType.CLICK_IOS.ui_type: - next_table = events.EventType.CLICK_IOS.table - next_col_name = events.EventType.CLICK_IOS.column - elif event_type == events.EventType.INPUT_IOS.ui_type: - next_table = events.EventType.INPUT_IOS.table - next_col_name = events.EventType.INPUT_IOS.column - elif event_type == events.EventType.VIEW_IOS.ui_type: - next_table = events.EventType.VIEW_IOS.table - next_col_name = events.EventType.VIEW_IOS.column - elif event_type == events.EventType.CUSTOM_IOS.ui_type: - next_table = events.EventType.CUSTOM_IOS.table - next_col_name = events.EventType.CUSTOM_IOS.column - else: - logging.warning(f"=================UNDEFINED:{event_type}") - continue - - values = {**values, **sh.multi_values(helper.values_for_operator(value=s.value, op=s.operator), - value_key=f"value{i + 1}")} - if sh.is_negation_operator(s.operator) and i > 0: - op = sh.reverse_sql_operator(op) - main_condition = "left_not.session_id ISNULL" - extra_from.append(f"""LEFT JOIN LATERAL (SELECT session_id - FROM {next_table} AS s_main - WHERE - {sh.multi_conditions(f"s_main.{next_col_name} {op} %(value{i + 1})s", - values=s.value, value_key=f"value{i + 1}")} - AND s_main.timestamp >= T{i}.stage{i}_timestamp - AND s_main.session_id = T1.session_id) AS left_not ON (TRUE)""") - else: - if is_any: - main_condition = "TRUE" - else: - main_condition = sh.multi_conditions(f"main.{next_col_name} {op} %(value{i + 1})s", - values=s.value, value_key=f"value{i + 1}") - n_stages_query.append(f""" - (SELECT main.session_id, - {"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp - FROM {next_table} AS main {" ".join(extra_from)} - WHERE main.timestamp >= {f"T{i}.stage{i}_timestamp" if i > 0 else "%(startTimestamp)s"} - {f"AND main.session_id=T1.session_id" if i > 0 else ""} - AND {main_condition} - {(" AND " + " AND ".join(stage_constraints)) if len(stage_constraints) > 0 else ""} - {(" AND " + " AND ".join(first_stage_extra_constraints)) if len(first_stage_extra_constraints) > 0 and i == 0 else ""} - GROUP BY main.session_id) - AS T{i + 1} {"ON (TRUE)" if i > 0 else ""} - """) - n_stages = len(n_stages_query) - if n_stages == 0: - return [] - n_stages_query = " LEFT JOIN LATERAL ".join(n_stages_query) - n_stages_query += ") AS stages_t" - - n_stages_query = f""" - SELECT stages_and_issues_t.*, sessions.user_uuid - FROM ( - SELECT * FROM ( - SELECT T1.session_id, {",".join([f"stage{i + 1}_timestamp" for i in range(n_stages)])} - FROM {n_stages_query} - LEFT JOIN LATERAL - ( SELECT ISS.type as issue_type, - ISE.timestamp AS issue_timestamp, - COALESCE(ISS.context_string,'') as issue_context, - ISS.issue_id as issue_id - FROM events_common.issues AS ISE INNER JOIN issues AS ISS USING (issue_id) - WHERE ISE.timestamp >= stages_t.stage1_timestamp - AND ISE.timestamp <= stages_t.stage{i + 1}_timestamp - AND ISS.project_id=%(project_id)s - AND ISE.session_id = stages_t.session_id - AND ISS.type!='custom' -- ignore custom issues because they are massive - {"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""} - LIMIT 10 -- remove the limit to get exact stats - ) AS issues_t ON (TRUE) - ) AS stages_and_issues_t INNER JOIN sessions USING(session_id); - """ - - # LIMIT 10000 - params = {"project_id": project_id, "startTimestamp": filter_d.startTimestamp, - "endTimestamp": filter_d.endTimestamp, - "issueTypes": tuple(filter_issues), **values} - with pg_client.PostgresClient() as cur: - query = cur.mogrify(n_stages_query, params) - logging.debug("---------------------------------------------------") - logging.debug(query) - logging.debug("---------------------------------------------------") - try: - cur.execute(query) - rows = cur.fetchall() - except Exception as err: - logging.warning("--------- FUNNEL SEARCH QUERY EXCEPTION -----------") - logging.warning(query.decode('UTF-8')) - logging.warning("--------- PAYLOAD -----------") - logging.warning(filter_d.model_dump_json()) - logging.warning("--------------------") - raise err - return rows - - -def pearson_corr(x: list, y: list): - n = len(x) - if n != len(y): - raise ValueError(f'x and y must have the same length. Got {len(x)} and {len(y)} instead') - - if n < 2: - warnings.warn(f'x and y must have length at least 2. Got {n} instead') - return None, None, False - - # If an input is constant, the correlation coefficient is not defined. - if all(t == x[0] for t in x) or all(t == y[0] for t in y): - warnings.warn("An input array is constant; the correlation coefficent is not defined.") - return None, None, False - - if n == 2: - return math.copysign(1, x[1] - x[0]) * math.copysign(1, y[1] - y[0]), 1.0, True - - xmean = sum(x) / len(x) - ymean = sum(y) / len(y) - - xm = [el - xmean for el in x] - ym = [el - ymean for el in y] - - normxm = math.sqrt((sum([xm[i] * xm[i] for i in range(len(xm))]))) - normym = math.sqrt((sum([ym[i] * ym[i] for i in range(len(ym))]))) - - threshold = 1e-8 - if normxm < threshold * abs(xmean) or normym < threshold * abs(ymean): - # If all the values in x (likewise y) are very close to the mean, - # the loss of precision that occurs in the subtraction xm = x - xmean - # might result in large errors in r. - warnings.warn("An input array is constant; the correlation coefficent is not defined.") - - r = sum( - i[0] * i[1] for i in zip([xm[i] / normxm for i in range(len(xm))], [ym[i] / normym for i in range(len(ym))])) - - # Presumably, if abs(r) > 1, then it is only some small artifact of floating point arithmetic. - # However, if r < 0, we don't care, as our problem is to find only positive correlations - r = max(min(r, 1.0), 0.0) - - # approximated confidence - if n < 31: - t_c = T_VALUES[n] - elif n < 50: - t_c = 2.02 - else: - t_c = 2 - if r >= 0.999: - confidence = 1 - else: - confidence = r * math.sqrt(n - 2) / math.sqrt(1 - r ** 2) - - if confidence > SIGNIFICANCE_THRSH: - return r, confidence, True - else: - return r, confidence, False - - -# def tuple_or(t: tuple): -# x = 0 -# for el in t: -# x |= el # | is for bitwise OR -# return x -# -# The following function is correct optimization of the previous function because t is a list of 0,1 -def tuple_or(t: tuple): - for el in t: - if el > 0: - return 1 - return 0 - - -def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues, first_stage, last_stage): - """ - Returns two lists with binary values 0/1: - - transitions ::: if transited from the first stage to the last - 1 - else - 0 - errors ::: a dictionary WHERE the keys are all unique issues (currently context-wise) - the values are lists - if an issue happened between the first stage to the last - 1 - else - 0 - - For a small task of calculating a total drop due to issues, - we need to disregard the issue type when creating the `errors`-like array. - The `all_errors` array can be obtained by logical OR statement applied to all errors by issue - The `transitions` array stays the same - """ - transitions = [] - n_sess_affected = 0 - errors = {} - - for row in rows: - t = 0 - first_ts = row[f'stage{first_stage}_timestamp'] - last_ts = row[f'stage{last_stage}_timestamp'] - if first_ts is None: - continue - elif last_ts is not None: - t = 1 - transitions.append(t) - - ic_present = False - for error_id in all_issues: - if error_id not in errors: - errors[error_id] = [] - ic = 0 - row_issue_id = row['issue_id'] - if row_issue_id is not None: - if last_ts is None or (first_ts < row['issue_timestamp'] < last_ts): - if error_id == row_issue_id: - ic = 1 - ic_present = True - errors[error_id].append(ic) - - if ic_present and t: - n_sess_affected += 1 - - all_errors = [tuple_or(t) for t in zip(*errors.values())] - - return transitions, errors, all_errors, n_sess_affected - - -def get_affected_users_for_all_issues(rows, first_stage, last_stage): - """ - - :param rows: - :param first_stage: - :param last_stage: - :return: - """ - affected_users = defaultdict(lambda: set()) - affected_sessions = defaultdict(lambda: set()) - all_issues = {} - n_affected_users_dict = defaultdict(lambda: None) - n_affected_sessions_dict = defaultdict(lambda: None) - n_issues_dict = defaultdict(lambda: 0) - issues_by_session = defaultdict(lambda: 0) - - for row in rows: - - # check that the session has reached the first stage of subfunnel: - if row[f'stage{first_stage}_timestamp'] is None: - continue - - iss = row['issue_type'] - iss_ts = row['issue_timestamp'] - - # check that the issue exists and belongs to subfunnel: - if iss is not None and (row[f'stage{last_stage}_timestamp'] is None or - (row[f'stage{first_stage}_timestamp'] < iss_ts < row[f'stage{last_stage}_timestamp'])): - if row["issue_id"] not in all_issues: - all_issues[row["issue_id"]] = {"context": row['issue_context'], "issue_type": row["issue_type"]} - n_issues_dict[row["issue_id"]] += 1 - if row['user_uuid'] is not None: - affected_users[row["issue_id"]].add(row['user_uuid']) - - affected_sessions[row["issue_id"]].add(row['session_id']) - issues_by_session[row[f'session_id']] += 1 - - if len(affected_users) > 0: - n_affected_users_dict.update({ - iss: len(affected_users[iss]) for iss in affected_users - }) - if len(affected_sessions) > 0: - n_affected_sessions_dict.update({ - iss: len(affected_sessions[iss]) for iss in affected_sessions - }) - return all_issues, n_issues_dict, n_affected_users_dict, n_affected_sessions_dict - - -def count_sessions(rows, n_stages): - session_counts = {i: set() for i in range(1, n_stages + 1)} - for row in rows: - for i in range(1, n_stages + 1): - if row[f"stage{i}_timestamp"] is not None: - session_counts[i].add(row[f"session_id"]) - - session_counts = {i: len(session_counts[i]) for i in session_counts} - return session_counts - - -def count_users(rows, n_stages): - users_in_stages = {i: set() for i in range(1, n_stages + 1)} - for row in rows: - for i in range(1, n_stages + 1): - if row[f"stage{i}_timestamp"] is not None: - users_in_stages[i].add(row["user_uuid"]) - - users_count = {i: len(users_in_stages[i]) for i in range(1, n_stages + 1)} - return users_count - - -def get_stages(stages, rows): - n_stages = len(stages) - session_counts = count_sessions(rows, n_stages) - users_counts = count_users(rows, n_stages) - - stages_list = [] - for i, stage in enumerate(stages): - - drop = None - if i != 0: - if session_counts[i] == 0: - drop = 0 - elif session_counts[i] > 0: - drop = int(100 * (session_counts[i] - session_counts[i + 1]) / session_counts[i]) - - stages_list.append( - {"value": stage.value, - "type": stage.type, - "operator": stage.operator, - "sessionsCount": session_counts[i + 1], - "drop_pct": drop, - "usersCount": users_counts[i + 1], - "dropDueToIssues": 0 - } - ) - return stages_list - - -def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False): - """ - - :param stages: - :param rows: - :param first_stage: If it's a part of the initial funnel, provide a number of the first stage (starting from 1) - :param last_stage: If it's a part of the initial funnel, provide a number of the last stage (starting from 1) - :return: - """ - - n_stages = len(stages) - - if first_stage is None: - first_stage = 1 - if last_stage is None: - last_stage = n_stages - if last_stage > n_stages: - logging.debug( - "The number of the last stage provided is greater than the number of stages. Using n_stages instead") - last_stage = n_stages - - n_critical_issues = 0 - issues_dict = {"significant": [], - "insignificant": []} - session_counts = count_sessions(rows, n_stages) - drop = session_counts[first_stage] - session_counts[last_stage] - - all_issues, n_issues_dict, affected_users_dict, affected_sessions = get_affected_users_for_all_issues( - rows, first_stage, last_stage) - transitions, errors, all_errors, n_sess_affected = get_transitions_and_issues_of_each_type(rows, - all_issues, - first_stage, last_stage) - - del rows - - if any(all_errors): - total_drop_corr, conf, is_sign = pearson_corr(transitions, all_errors) - if total_drop_corr is not None and drop is not None: - total_drop_due_to_issues = int(total_drop_corr * n_sess_affected) - else: - total_drop_due_to_issues = 0 - else: - total_drop_due_to_issues = 0 - - if drop_only: - return total_drop_due_to_issues - for issue_id in all_issues: - - if not any(errors[issue_id]): - continue - r, confidence, is_sign = pearson_corr(transitions, errors[issue_id]) - - if r is not None and drop is not None and is_sign: - lost_conversions = int(r * affected_sessions[issue_id]) - else: - lost_conversions = None - if r is None: - r = 0 - issues_dict['significant' if is_sign else 'insignificant'].append({ - "type": all_issues[issue_id]["issue_type"], - "title": helper.get_issue_title(all_issues[issue_id]["issue_type"]), - "affected_sessions": affected_sessions[issue_id], - "unaffected_sessions": session_counts[1] - affected_sessions[issue_id], - "lost_conversions": lost_conversions, - "affected_users": affected_users_dict[issue_id], - "conversion_impact": round(r * 100), - "context_string": all_issues[issue_id]["context"], - "issue_id": issue_id - }) - - if is_sign: - n_critical_issues += n_issues_dict[issue_id] - # To limit the number of returned issues to the frontend - issues_dict["significant"] = issues_dict["significant"][:20] - issues_dict["insignificant"] = issues_dict["insignificant"][:20] - - return n_critical_issues, issues_dict, total_drop_due_to_issues - - -def get_top_insights(filter_d: schemas.CardSeriesFilterSchema, project_id): - output = [] - stages = filter_d.events - # TODO: handle 1 stage alone - if len(stages) == 0: - logging.debug("no stages found") - return output, 0 - elif len(stages) == 1: - # TODO: count sessions, and users for single stage - output = [{ - "type": stages[0].type, - "value": stages[0].value, - "dropPercentage": None, - "operator": stages[0].operator, - "sessionsCount": 0, - "dropPct": 0, - "usersCount": 0, - "dropDueToIssues": 0 - - }] - # original - # counts = sessions.search_sessions(data=schemas.SessionsSearchCountSchema.parse_obj(filter_d), - # project_id=project_id, user_id=None, count_only=True) - # first change - # counts = sessions.search_sessions(data=schemas.FlatSessionsSearchPayloadSchema.parse_obj(filter_d), - # project_id=project_id, user_id=None, count_only=True) - # last change - counts = sessions.search_sessions(data=schemas.SessionsSearchPayloadSchema.model_validate(filter_d), - project_id=project_id, user_id=None, count_only=True) - output[0]["sessionsCount"] = counts["countSessions"] - output[0]["usersCount"] = counts["countUsers"] - return output, 0 - # The result of the multi-stage query - rows = get_stages_and_events(filter_d=filter_d, project_id=project_id) - if len(rows) == 0: - return get_stages(stages, []), 0 - # Obtain the first part of the output - stages_list = get_stages(stages, rows) - # Obtain the second part of the output - total_drop_due_to_issues = get_issues(stages, rows, - first_stage=1, - last_stage=len(filter_d.events), - drop_only=True) - return stages_list, total_drop_due_to_issues - - -def get_issues_list(filter_d: schemas.CardSeriesFilterSchema, project_id, first_stage=None, last_stage=None): - output = dict({"total_drop_due_to_issues": 0, "critical_issues_count": 0, "significant": [], "insignificant": []}) - stages = filter_d.events - # The result of the multi-stage query - rows = get_stages_and_events(filter_d=filter_d, project_id=project_id) - if len(rows) == 0: - return output - # Obtain the second part of the output - n_critical_issues, issues_dict, total_drop_due_to_issues = get_issues(stages, rows, first_stage=first_stage, - last_stage=last_stage) - output['total_drop_due_to_issues'] = total_drop_due_to_issues - # output['critical_issues_count'] = n_critical_issues - output = {**output, **issues_dict} - return output +from .significance import * +# TODO: use clickhouse for funnels diff --git a/ee/api/chalicelib/utils/ch_client.py b/ee/api/chalicelib/utils/ch_client.py index cbd27d235..b7d19b4f9 100644 --- a/ee/api/chalicelib/utils/ch_client.py +++ b/ee/api/chalicelib/utils/ch_client.py @@ -41,8 +41,13 @@ class ClickHouseClient: keys = tuple(x for x, y in results[1]) return [dict(zip(keys, i)) for i in results[0]] except Exception as err: + logging.error("--------- CH EXCEPTION -----------") + logging.error(err) logging.error("--------- CH QUERY EXCEPTION -----------") - logging.error(self.format(query=query, params=params)) + logging.error(self.format(query=query, params=params) + .replace('\n', '\\n') + .replace(' ', ' ') + .replace(' ', ' ')) logging.error("--------------------") raise err diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index cdd294386..f4da5ecd2 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -48,6 +48,7 @@ rm -rf ./chalicelib/core/saved_search.py rm -rf ./chalicelib/core/sessions.py rm -rf ./chalicelib/core/sessions_assignments.py rm -rf ./chalicelib/core/sessions_mobs.py +rm -rf ./chalicelib/core/significance.py rm -rf ./chalicelib/core/socket_ios.py rm -rf ./chalicelib/core/sourcemaps.py rm -rf ./chalicelib/core/sourcemaps_parser.py diff --git a/ee/api/routers/core_dynamic.py b/ee/api/routers/core_dynamic.py index bc51a803d..e2a9c90ab 100644 --- a/ee/api/routers/core_dynamic.py +++ b/ee/api/routers/core_dynamic.py @@ -514,8 +514,7 @@ def edit_note(projectId: int, noteId: int, data: schemas.SessionUpdateNoteSchema @app.delete('/{projectId}/notes/{noteId}', tags=["sessions", "notes"], dependencies=[OR_scope(Permissions.session_replay)]) def delete_note(projectId: int, noteId: int, _=Body(None), context: schemas.CurrentContext = Depends(OR_context)): - data = sessions_notes.delete(tenant_id=context.tenant_id, project_id=projectId, user_id=context.user_id, - note_id=noteId) + data = sessions_notes.delete(project_id=projectId, note_id=noteId) return data diff --git a/scripts/helmcharts/openreplay/charts/chalice/Chart.yaml b/scripts/helmcharts/openreplay/charts/chalice/Chart.yaml index c231e6ae7..af0e4427b 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/Chart.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/Chart.yaml @@ -1,7 +1,6 @@ apiVersion: v2 name: chalice description: A Helm chart for Kubernetes - # A chart can be either an 'application' or a 'library' chart. # # Application charts are a collection of templates that can be packaged into versioned archives @@ -11,14 +10,12 @@ description: A Helm chart for Kubernetes # a dependency of application charts to inject those utilities and functions into the rendering # pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application - # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) version: 0.1.18 - # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -AppVersion: "v1.17.11" +AppVersion: "v1.17.12"