From f26c9e6b806002d1a48e7d6b9ab1aaf35ff5ac19 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 26 Jan 2022 16:55:44 +0100 Subject: [PATCH 1/2] feat(api): search sessions group by userId --- api/chalicelib/core/sessions.py | 26 ++++++++++++++++++++------ api/schemas.py | 6 +++++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 5f7cd23af..bbb0e6670 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -183,10 +183,24 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f main_query = cur.mogrify(f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, COUNT(DISTINCT s.user_uuid) AS count_users {query_part};""", full_args) + elif data.group_by_user: + main_query = cur.mogrify(f"""SELECT COUNT(*) AS count, jsonb_agg(users_sessions) FILTER ( WHERE rn <= 200 ) AS sessions + FROM (SELECT user_id, + count(full_sessions) AS user_sessions_count, + jsonb_agg(full_sessions) FILTER (WHERE rn <= 1) AS last_session, + ROW_NUMBER() OVER (ORDER BY count(full_sessions) DESC) AS rn + FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY start_ts DESC) AS rn + FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} + {query_part} + ORDER BY s.session_id desc) AS filtred_sessions + ORDER BY favorite DESC, issue_score DESC, {sort} {data.order}) AS full_sessions + GROUP BY user_id + ORDER BY user_sessions_count DESC) AS users_sessions;""", + full_args) else: main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, COALESCE(JSONB_AGG(full_sessions) FILTER (WHERE rn <= 200), '[]'::JSONB) AS sessions - FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY favorite DESC, issue_score DESC, session_id desc, start_ts desc) AS rn FROM - (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} + FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY favorite DESC, issue_score DESC, session_id desc, start_ts desc) AS rn + FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} {query_part} ORDER BY s.session_id desc) AS filtred_sessions ORDER BY favorite DESC, issue_score DESC, {sort} {data.order}) AS full_sessions;""", @@ -199,11 +213,11 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # ORDER BY favorite DESC, issue_score DESC, {sort} {order};""", # full_args) - # print("--------------------") - # print(main_query) + print("--------------------") + print(main_query) cur.execute(main_query) - # print("--------------------") + print("--------------------") if count_only: return helper.dict_to_camel_case(cur.fetchone()) sessions = cur.fetchone() @@ -221,7 +235,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f if errors_only: return sessions - if data.sort is not None and data.sort != "session_id": + if not data.group_by_user and data.sort is not None and data.sort != "session_id": sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data.sort)], reverse=data.order.upper() == "DESC") return { diff --git a/api/schemas.py b/api/schemas.py index 11d882e79..a9e1059a1 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -470,7 +470,7 @@ class _SessionSearchEventRaw(__MixedSearchFilter): value: Union[str, List[str]] = Field(...) type: Union[EventType, PerformanceEventType] = Field(...) operator: SearchEventOperator = Field(...) - source: Optional[Union[ErrorSource,List[Union[int, str]]]] = Field(default=ErrorSource.js_exception) + source: Optional[Union[ErrorSource, List[Union[int, str]]]] = Field(default=ErrorSource.js_exception) sourceOperator: Optional[MathOperator] = Field(None) @root_validator @@ -536,6 +536,7 @@ class SessionsSearchPayloadSchema(BaseModel): sort: str = Field(default="startTs") order: str = Field(default="DESC") events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then) + group_by_user: bool = Field(default=False) class Config: alias_generator = attribute_to_camel_case @@ -577,6 +578,7 @@ class FunnelSearchPayloadSchema(FlatSessionsSearchPayloadSchema): range_value: Optional[str] = Field(None) sort: Optional[str] = Field(None) order: Optional[str] = Field(None) + group_by_user: Optional[bool] = Field(default=False, const=True) class Config: alias_generator = attribute_to_camel_case @@ -601,6 +603,7 @@ class FunnelInsightsPayloadSchema(FlatSessionsSearchPayloadSchema): # class FunnelInsightsPayloadSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(None) order: Optional[str] = Field(None) + group_by_user: Optional[bool] = Field(default=False, const=True) class MetricPayloadSchema(BaseModel): @@ -634,6 +637,7 @@ class CustomMetricSeriesFilterSchema(FlatSessionsSearchPayloadSchema): endDate: Optional[int] = Field(None) sort: Optional[str] = Field(None) order: Optional[str] = Field(None) + group_by_user: Optional[bool] = Field(default=False, const=True) class CustomMetricCreateSeriesSchema(BaseModel): From e77cc2176f3c7ff89014b462501a1825b5e49218 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 2 Feb 2022 17:21:15 +0100 Subject: [PATCH 2/2] feat(api): fixed schema for Funnel-onTheFly endpoint feat(api): fixed schema for search sessions by error feat(api): search sessions by error support multiple values feat(api): search sessions by error support multiple sources --- api/chalicelib/core/sessions.py | 31 ++++++++++++++++--------------- api/schemas.py | 12 +++++++++--- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index bbb0e6670..96ccfe4f2 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -150,9 +150,10 @@ def _multiple_conditions(condition, values, value_key="value", is_not=False): def _multiple_values(values, value_key="value"): query_values = {} - for i in range(len(values)): - k = f"{value_key}_{i}" - query_values[k] = values[i] + if values is not None and isinstance(values, list): + for i in range(len(values)): + k = f"{value_key}_{i}" + query_values[k] = values[i] return query_values @@ -480,9 +481,12 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr if data.events_order == schemas.SearchEventOrder._then: event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") e_k = f"e_value{i}" + s_k = e_k + "_source" if event.type != schemas.PerformanceEventType.time_between_events: event.value = helper.values_for_operator(value=event.value, op=event.operator) - full_args = {**full_args, **_multiple_values(event.value, value_key=e_k)} + full_args = {**full_args, + **_multiple_values(event.value, value_key=e_k), + **_multiple_values(event.source, value_key=s_k)} # if event_type not in list(events.SUPPORTED_TYPES.keys()) \ # or event.value in [None, "", "*"] \ @@ -538,18 +542,15 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", event.value, value_key=e_k)) elif event_type == events.event_type.ERROR.ui_type: - # if event.source in [None, "*", ""]: - # event.source = "js_exception" event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" - if event.value not in [None, "*", ""]: - if not is_any: - event_where.append(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)") - if event.source not in [None, "*", ""]: - event_where.append(f"main1.source = %(source)s") - full_args["source"] = event.source - elif event.source not in [None, "*", ""]: - event_where.append(f"main1.source = %(source)s") - full_args["source"] = event.source + event.source = tuple(event.source) + if not is_any and event.value not in [None, "*", ""]: + event_where.append( + _multiple_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) + if event.source[0] not in [None, "*", ""]: + event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.value, value_key=s_k)) + # ----- IOS elif event_type == events.event_type.CLICK_IOS.ui_type: diff --git a/api/schemas.py b/api/schemas.py index fce6c2241..c87cd7f75 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -466,11 +466,11 @@ class __MixedSearchFilter(BaseModel): class _SessionSearchEventRaw(__MixedSearchFilter): - is_event: bool = Field(True, const=True) - value: Union[str, List[str]] = Field(...) + is_event: bool = Field(default=True, const=True) + value: List[str] = Field(...) type: Union[EventType, PerformanceEventType] = Field(...) operator: SearchEventOperator = Field(...) - source: Optional[Union[ErrorSource, List[Union[int, str]]]] = Field(default=ErrorSource.js_exception) + source: Optional[List[Union[ErrorSource, int, str]]] = Field(None) sourceOperator: Optional[MathOperator] = Field(None) @root_validator @@ -492,6 +492,9 @@ class _SessionSearchEventRaw(__MixedSearchFilter): else: for c in values["source"]: assert isinstance(c, int), f"source value should be of type int for {values.get('type')}" + elif values.get("type") == EventType.error and values.get("source") is None: + values["source"] = [ErrorSource.js_exception] + return values @@ -586,6 +589,7 @@ class FunnelSearchPayloadSchema(FlatSessionsSearchPayloadSchema): @root_validator(pre=True) def enforce_default_values(cls, values): values["eventsOrder"] = SearchEventOrder._then + values["groupByUser"] = False return values class Config: @@ -611,6 +615,7 @@ class FunnelInsightsPayloadSchema(FlatSessionsSearchPayloadSchema): # class FunnelInsightsPayloadSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(None) order: Optional[str] = Field(None) + events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then, const=True) group_by_user: Optional[bool] = Field(default=False, const=True) @@ -645,6 +650,7 @@ class CustomMetricSeriesFilterSchema(FlatSessionsSearchPayloadSchema): endDate: Optional[int] = Field(None) sort: Optional[str] = Field(None) order: Optional[str] = Field(None) + events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then, const=True) group_by_user: Optional[bool] = Field(default=False, const=True)