diff --git a/LICENSE b/LICENSE index a92396f23..b348b0c8f 100644 --- a/LICENSE +++ b/LICENSE @@ -5,7 +5,7 @@ OpenReplay monorepo uses multiple licenses. Portions of this software are licens - All content that resides under the "ee/" directory of this repository, is licensed under the license defined in "ee/LICENSE". - Content outside of the above mentioned directories or restrictions above is available under the "Elastic License 2.0 (ELv2)" license as defined below. -[Reach out](mailto:license@openreplay.com) if you have any questions regarding licenses. +Reach out (license@openreplay.com) if you have any questions regarding licenses. ------------------------------------------------------------------------------------ Elastic License 2.0 (ELv2) diff --git a/api/chalicelib/core/errors.py b/api/chalicelib/core/errors.py index c4825e163..f64ea017f 100644 --- a/api/chalicelib/core/errors.py +++ b/api/chalicelib/core/errors.py @@ -1,5 +1,6 @@ import json +import schemas from chalicelib.core import sourcemaps, sessions from chalicelib.utils import pg_client, helper, dev from chalicelib.utils.TimeUTC import TimeUTC @@ -405,9 +406,9 @@ def __get_basic_constraints(platform=None, time_constraint=True, startTime_arg_n if chart: ch_sub_query += [f"timestamp >= generated_timestamp", f"timestamp < generated_timestamp + %({step_size_name})s"] - if platform == 'mobile': + if platform == schemas.PlatformType.mobile: ch_sub_query.append("user_device_type = 'mobile'") - elif platform == 'desktop': + elif platform == schemas.PlatformType.desktop: ch_sub_query.append("user_device_type = 'desktop'") return ch_sub_query @@ -421,23 +422,28 @@ def __get_sort_key(key): @dev.timed -def search(data, project_id, user_id, flows=False, status="ALL", favorite_only=False): +def search(data: schemas.SearchErrorsSchema, project_id, user_id, flows=False, status="ALL", favorite_only=False): status = status.upper() if status.lower() not in ['all', 'unresolved', 'resolved', 'ignored']: return {"errors": ["invalid error status"]} - pg_sub_query = __get_basic_constraints(data.get('platform'), project_key="sessions.project_id") + platform = None + for f in data.filters: + if f.type == schemas.FilterType.platform and len(f.value) > 0: + platform = f.value[0] + pg_sub_query = __get_basic_constraints(platform, project_key="sessions.project_id") pg_sub_query += ["sessions.start_ts>=%(startDate)s", "sessions.start_ts<%(endDate)s", "source ='js_exception'", "pe.project_id=%(project_id)s"] - pg_sub_query_chart = __get_basic_constraints(data.get('platform'), time_constraint=False, chart=True) + pg_sub_query_chart = __get_basic_constraints(platform, time_constraint=False, chart=True) pg_sub_query_chart.append("source ='js_exception'") pg_sub_query_chart.append("errors.error_id =details.error_id") statuses = [] error_ids = None - if data.get("startDate") is None: - data["startDate"] = TimeUTC.now(-30) - if data.get("endDate") is None: - data["endDate"] = TimeUTC.now(1) - if len(data.get("events", [])) > 0 or len(data.get("filters", [])) > 0 or status != "ALL" or favorite_only: + if data.startDate is None: + data.startDate = TimeUTC.now(-30) + if data.endDate is None: + data.endDate = TimeUTC.now(1) + if len(data.events) > 0 or len(data.filters) > 0 or status != "ALL": + # if favorite_only=True search for sessions associated with favorite_error statuses = sessions.search2_pg(data=data, project_id=project_id, user_id=user_id, errors_only=True, error_status=status) if len(statuses) == 0: @@ -447,28 +453,30 @@ def search(data, project_id, user_id, flows=False, status="ALL", favorite_only=F }} error_ids = [e["error_id"] for e in statuses] with pg_client.PostgresClient() as cur: - if data.get("startDate") is None: - data["startDate"] = TimeUTC.now(-7) - if data.get("endDate") is None: - data["endDate"] = TimeUTC.now() - density = data.get("density", 7) - step_size = __get_step_size(data["startDate"], data["endDate"], density, factor=1) + if data.startDate is None: + data.startDate = TimeUTC.now(-7) + if data.endDate is None: + data.endDate = TimeUTC.now() + step_size = __get_step_size(data.startDate, data.endDate, data.density, factor=1) sort = __get_sort_key('datetime') - if data.get("sort") is not None: - sort = __get_sort_key(data["sort"]) + if data.sort is not None: + sort = __get_sort_key(data.sort) order = "DESC" - if data.get("order") is not None: - order = data["order"] - + if data.order is not None: + order = data.order + extra_join = "" params = { - "startDate": data['startDate'], - "endDate": data['endDate'], + "startDate": data.startDate, + "endDate": data.endDate, "project_id": project_id, "userId": user_id, "step_size": step_size} if error_ids is not None: params["error_ids"] = tuple(error_ids) pg_sub_query.append("error_id IN %(error_ids)s") + if favorite_only: + pg_sub_query.append("ufe.user_id = %(userId)s") + extra_join += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" main_pg_query = f"""\ SELECT error_id, name, @@ -488,6 +496,7 @@ def search(data, project_id, user_id, flows=False, status="ALL", favorite_only=F FROM events.errors INNER JOIN public.errors AS pe USING (error_id) INNER JOIN public.sessions USING (session_id) + {extra_join} WHERE {" AND ".join(pg_sub_query)} GROUP BY error_id, name, message ORDER BY {sort} {order}) AS details @@ -581,7 +590,7 @@ def __save_stacktrace(error_id, data): def get_trace(project_id, error_id): - error = get(error_id=error_id) + error = get(error_id=error_id, family=False) if error is None: return {"errors": ["error not found"]} if error.get("source", "") != "js_exception": diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index cdd6cec20..adcb1167e 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -1,4 +1,5 @@ import json +from typing import List import chalicelib.utils.helper import schemas @@ -12,12 +13,38 @@ REMOVE_KEYS = ["key", "_key", "startDate", "endDate"] ALLOW_UPDATE_FOR = ["name", "filter"] -# def filter_stages(stages): -# ALLOW_TYPES = [events.event_type.CLICK.ui_type, events.event_type.INPUT.ui_type, -# events.event_type.LOCATION.ui_type, events.event_type.CUSTOM.ui_type, -# events.event_type.CLICK_IOS.ui_type, events.event_type.INPUT_IOS.ui_type, -# events.event_type.VIEW_IOS.ui_type, events.event_type.CUSTOM_IOS.ui_type, ] -# return [s for s in stages if s["type"] in ALLOW_TYPES and s.get("value") is not None] +def filter_stages(stages: List[schemas._SessionSearchEventSchema]): + ALLOW_TYPES = [schemas.EventType.click, schemas.EventType.input, + schemas.EventType.location, schemas.EventType.custom, + schemas.EventType.click_ios, schemas.EventType.input_ios, + schemas.EventType.view_ios, schemas.EventType.custom_ios, ] + return [s for s in stages if s.type in ALLOW_TYPES and s.value is not None] + + +def __parse_events(f_events: List[dict]): + return [schemas._SessionSearchEventSchema.parse_obj(e) for e in f_events] + + +def __unparse_events(f_events: List[schemas._SessionSearchEventSchema]): + return [e.dict() for e in f_events] + + +def __fix_stages(f_events: List[schemas._SessionSearchEventSchema]): + if f_events is None: + return + events = [] + for e in f_events: + if e.operator is None: + e.operator = schemas.SearchEventOperator._is + + if not isinstance(e.value, list): + e.value = [e.value] + is_any = sessions._isAny_opreator(e.operator) + if not is_any and isinstance(e.value, list) and len(e.value) == 0: + continue + events.append(e) + return events + def __transform_old_funnels(events): for e in events: @@ -28,7 +55,7 @@ def __transform_old_funnels(events): def create(project_id, user_id, name, filter: schemas.FunnelSearchPayloadSchema, is_public): helper.delete_keys_from_dict(filter, REMOVE_KEYS) - # filter.events = filter_stages(stages=filter.events) + filter.events = filter_stages(stages=filter.events) with pg_client.PostgresClient() as cur: query = cur.mogrify("""\ INSERT INTO public.funnels (project_id, user_id, name, filter,is_public) @@ -76,6 +103,8 @@ def update(funnel_id, user_id, project_id, name=None, filter=None, is_public=Non query ) r = cur.fetchone() + if r is None: + return {"errors": ["funnel not found"]} r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) r = helper.dict_to_camel_case(r) r["filter"]["startDate"], r["filter"]["endDate"] = TimeUTC.get_start_end_from_range(r["filter"]["rangeValue"]) @@ -102,9 +131,9 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date for row in rows: row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"]) if details: - # row["filter"]["events"] = filter_stages(row["filter"]["events"]) + row["filter"]["events"] = filter_stages(__parse_events(row["filter"]["events"])) if row.get("filter") is not None and row["filter"].get("events") is not None: - row["filter"]["events"] = __transform_old_funnels(row["filter"]["events"]) + row["filter"]["events"] = __transform_old_funnels(__unparse_events(row["filter"]["events"])) get_start_end_time(filter_d=row["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -168,7 +197,8 @@ def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=No def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.FunnelSearchPayloadSchema): - # data.events = filter_stages(data.events) + data.events = filter_stages(data.events) + data.events = __fix_stages(data.events) if len(data.events) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: @@ -192,17 +222,18 @@ def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_dat "totalDropDueToIssues": total_drop_due_to_issues}} -def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data): - # data["events"] = filter_stages(data.get("events", [])) - if len(data["events"]) == 0: +def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data: schemas.FunnelInsightsPayloadSchema): + data.events = filter_stages(__parse_events(data.events)) + if len(data.events) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), - start_date=data.get('startDate', None), - end_date=data.get('endDate', None)) - data = f["filter"] - insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data, project_id=project_id) + get_start_end_time(filter_d=f["filter"], range_value=data.rangeValue, + start_date=data.startDate, + end_date=data.endDate) + data = schemas.FunnelInsightsPayloadSchema.parse_obj(f["filter"]) + data.events = __fix_stages(data.events) + insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data.dict(), project_id=project_id) if len(insights) > 0: insights[-1]["dropDueToIssues"] = total_drop_due_to_issues return {"data": {"stages": helper.list_to_camel_case(insights), @@ -220,25 +251,26 @@ def get_issues(project_id, user_id, funnel_id, range_value=None, start_date=None @dev.timed -def get_issues_on_the_fly(funnel_id, user_id, project_id, data): - first_stage = data.get("firstStage") - last_stage = data.get("lastStage") - # data["events"] = filter_stages(data.get("events", [])) - if len(data["events"]) == 0: +def get_issues_on_the_fly(funnel_id, user_id, project_id, data: schemas.FunnelSearchPayloadSchema): + data.events = filter_stages(data.events) + data.events = __fix_stages(data.events) + if len(data.events) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), - start_date=data.get('startDate', None), - end_date=data.get('endDate', None)) - data = f["filter"] + get_start_end_time(filter_d=f["filter"], range_value=data.rangeValue, + start_date=data.startDate, + end_date=data.endDate) + data = schemas.FunnelSearchPayloadSchema.parse_obj(f["filter"]) + if len(data.events) < 2: + return {"issues": []} return { "issues": helper.dict_to_camel_case( - significance.get_issues_list(filter_d=data, project_id=project_id, first_stage=first_stage, - last_stage=last_stage))} + significance.get_issues_list(filter_d=data.dict(), project_id=project_id, first_stage=1, + last_stage=len(data.events)))} -def get(funnel_id, project_id, user_id, flatten=True): +def get(funnel_id, project_id, user_id, flatten=True, fix_stages=True): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify( @@ -260,7 +292,11 @@ def get(funnel_id, project_id, user_id, flatten=True): if f.get("filter") is not None and f["filter"].get("events") is not None: f["filter"]["events"] = __transform_old_funnels(f["filter"]["events"]) f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) - # f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) + f["filter"]["events"] = __parse_events(f["filter"]["events"]) + f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) + if fix_stages: + f["filter"]["events"] = __fix_stages(f["filter"]["events"]) + f["filter"]["events"] = [e.dict() for e in f["filter"]["events"]] if flatten: f["filter"] = helper.old_search_payload_to_flat(f["filter"]) return f @@ -279,7 +315,7 @@ def search_by_issue(user_id, project_id, funnel_id, issue_id, data: schemas.Funn end_date=data.endDate) data = schemas.FunnelSearchPayloadSchema.parse_obj(f["filter"]) - issues = get_issues_on_the_fly(funnel_id=funnel_id, user_id=user_id, project_id=project_id, data=data.dict()) \ + issues = get_issues_on_the_fly(funnel_id=funnel_id, user_id=user_id, project_id=project_id, data=data) \ .get("issues", {}) issues = issues.get("significant", []) + issues.get("insignificant", []) issue = None diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 6c9ec24e6..b213a55f6 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -228,14 +228,15 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, e ORDER BY favorite DESC, issue_score DESC, {sort} {data.order}) AS full_sessions;""", full_args) - print("--------------------") - print(main_query) - print("--------------------") - cur.execute(main_query) + # print("--------------------") + # print(main_query) + # print("--------------------") - if count_only: - return helper.dict_to_camel_case(cur.fetchone()) + cur.execute(main_query) sessions = cur.fetchone() + if count_only: + return helper.dict_to_camel_case(sessions) + total = sessions["count"] sessions = sessions["sessions"] # sessions = [] @@ -281,7 +282,7 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d data.filters.append(schemas.SessionSearchFilterSchema(value=metric_value, type=schemas.FilterType.issue, operator=schemas.SearchEventOperator._is)) full_args, query_part, sort = search_query_parts(data=data, error_status=None, errors_only=False, - issue=None, project_id=project_id, + favorite_only=False, issue=None, project_id=project_id, user_id=None, extra_event=extra_event) full_args["step_size"] = step_size sessions = [] @@ -365,7 +366,7 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d return sessions -def search_query_parts(data, error_status, errors_only, issue, project_id, user_id, extra_event=None): +def search_query_parts(data, error_status, errors_only, favorite_only, issue, project_id, user_id, extra_event=None): ss_constraints = [] full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, "projectId": project_id, "userId": user_id} @@ -375,7 +376,7 @@ def search_query_parts(data, error_status, errors_only, issue, project_id, user_ ] extra_from = "" fav_only_join = "" - if data.bookmarked and not errors_only: + if favorite_only and not errors_only: fav_only_join = "LEFT JOIN public.user_favorite_sessions AS fs ON fs.session_id = s.session_id" # extra_constraints.append("fs.user_id = %(userId)s") events_query_part = "" @@ -969,9 +970,9 @@ def search_query_parts(data, error_status, errors_only, issue, project_id, user_ if error_status != "ALL": extra_constraints.append("ser.status = %(error_status)s") full_args["status"] = error_status.lower() - if data.bookmarked: + if favorite_only: extra_from += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" - extra_constraints.append("ufe.user_id = %(user_id)s") + extra_constraints.append("ufe.user_id = %(userId)s") # extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] if data.bookmarked and not errors_only and user_id is not None: extra_from += """INNER JOIN (SELECT user_id, session_id @@ -1199,11 +1200,11 @@ def get_session_user(project_id, user_id): "public".sessions WHERE project_id = %(project_id)s - AND user_id = %(user_id)s + AND user_id = %(userId)s AND duration is not null GROUP BY user_id; """, - {"project_id": project_id, "user_id": user_id} + {"project_id": project_id, "userId": user_id} ) cur.execute(query=query) data = cur.fetchone() @@ -1216,8 +1217,8 @@ def get_session_ids_by_user_ids(project_id, user_ids): """\ SELECT session_id FROM public.sessions WHERE - project_id = %(project_id)s AND user_id IN %(user_id)s;""", - {"project_id": project_id, "user_id": tuple(user_ids)} + project_id = %(project_id)s AND user_id IN %(userId)s;""", + {"project_id": project_id, "userId": tuple(user_ids)} ) ids = cur.execute(query=query) return ids @@ -1243,8 +1244,8 @@ def delete_sessions_by_user_ids(project_id, user_ids): """\ DELETE FROM public.sessions WHERE - project_id = %(project_id)s AND user_id IN %(user_id)s;""", - {"project_id": project_id, "user_id": tuple(user_ids)} + project_id = %(project_id)s AND user_id IN %(userId)s;""", + {"project_id": project_id, "userId": tuple(user_ids)} ) cur.execute(query=query) diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index ab242d7e8..035890e2f 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -118,12 +118,9 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: first_stage_extra_constraints.append( sessions._multiple_conditions(f's.rev_id {op} %({f_k})s', f["value"], value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) + i = -1 + for s in stages: - for i, s in enumerate(stages): - if i == 0: - extra_from = filter_extra_from + ["INNER JOIN public.sessions AS s USING (session_id)"] - else: - extra_from = [] if s.get("operator") is None: s["operator"] = "is" @@ -132,6 +129,11 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: is_any = sessions._isAny_opreator(s["operator"]) if not is_any and isinstance(s["value"], list) and len(s["value"]) == 0: continue + i += 1 + if i == 0: + extra_from = filter_extra_from + ["INNER JOIN public.sessions AS s USING (session_id)"] + else: + extra_from = [] op = sessions.__get_sql_operator(s["operator"]) event_type = s["type"].upper() if event_type == events.event_type.CLICK.ui_type: @@ -213,7 +215,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: ISS.issue_id as issue_id FROM events_common.issues AS ISE INNER JOIN issues AS ISS USING (issue_id) WHERE ISE.timestamp >= stages_t.stage1_timestamp - AND ISE.timestamp <= stages_t.stage{len(stages)}_timestamp + AND ISE.timestamp <= stages_t.stage{i + 1}_timestamp AND ISS.project_id=%(project_id)s {"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""}) AS base_t ) AS issues_t diff --git a/api/routers/core.py b/api/routers/core.py index 86ba57daf..999222c3e 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -716,7 +716,7 @@ def get_funnel_insights(projectId: int, funnelId: int, rangeValue: str = None, s def get_funnel_insights_on_the_fly(projectId: int, funnelId: int, data: schemas.FunnelInsightsPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): return funnels.get_top_insights_on_the_fly(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, - data=data.dict()) + data=data) @app.get('/{projectId}/funnels/{funnelId}/issues', tags=["funnels"]) @@ -731,7 +731,7 @@ def get_funnel_issues(projectId: int, funnelId, rangeValue: str = None, startDat def get_funnel_issues_on_the_fly(projectId: int, funnelId: int, data: schemas.FunnelSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): return {"data": funnels.get_issues_on_the_fly(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, - data=data.dict())} + data=data)} @app.get('/{projectId}/funnels/{funnelId}/sessions', tags=["funnels"]) @@ -755,10 +755,11 @@ def get_funnel_sessions_on_the_fly(projectId: int, funnelId: int, data: schemas. def get_issue_sessions(projectId: int, issueId: str, startDate: int = None, endDate: int = None, context: schemas.CurrentContext = Depends(OR_context)): issue = issues.get(project_id=projectId, issue_id=issueId) + if issue is None: + return {"errors": ["issue not found"]} return { "data": {"sessions": sessions.search_by_issue(user_id=context.user_id, project_id=projectId, issue=issue, - start_date=startDate, - end_date=endDate), + start_date=startDate, end_date=endDate), "issue": issue}} @@ -906,8 +907,7 @@ def errors_search(projectId: int, status: str = "ALL", favorite: Union[str, bool context: schemas.CurrentContext = Depends(OR_context)): if isinstance(favorite, str): favorite = True if len(favorite) == 0 else False - return errors.search(data.dict(), projectId, user_id=context.user_id, status=status, - favorite_only=favorite) + return errors.search(data, projectId, user_id=context.user_id, status=status, favorite_only=favorite) @app.get('/{projectId}/errors/stats', tags=['errors']) diff --git a/api/schemas.py b/api/schemas.py index c89d9ad56..767a53866 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -83,15 +83,6 @@ class EditSlackSchema(BaseModel): url: HttpUrl = Field(...) -class SearchErrorsSchema(BaseModel): - platform: Optional[str] = Field(None) - startDate: Optional[int] = Field(TimeUTC.now(-7)) - endDate: Optional[int] = Field(TimeUTC.now()) - density: Optional[int] = Field(7) - sort: Optional[str] = Field(None) - order: Optional[str] = Field(None) - - class CreateNotificationSchema(BaseModel): token: str = Field(...) notifications: List = Field(...) @@ -663,6 +654,7 @@ class FunnelSearchPayloadSchema(FlatSessionsSearchPayloadSchema): order: Optional[str] = Field(None) events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then, const=True) group_by_user: Optional[bool] = Field(default=False, const=True) + rangeValue: Optional[str] = Field(None) @root_validator(pre=True) def enforce_default_values(cls, values): @@ -695,6 +687,11 @@ class FunnelInsightsPayloadSchema(FlatSessionsSearchPayloadSchema): order: Optional[str] = Field(None) events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then, const=True) group_by_user: Optional[bool] = Field(default=False, const=True) + rangeValue: Optional[str] = Field(None) + + +class SearchErrorsSchema(SessionsSearchPayloadSchema): + density: Optional[int] = Field(7) class MetricPayloadSchema(BaseModel): diff --git a/ee/api/chalicelib/core/errors.py b/ee/api/chalicelib/core/errors.py index 04efdbb32..dc4ea17b2 100644 --- a/ee/api/chalicelib/core/errors.py +++ b/ee/api/chalicelib/core/errors.py @@ -1,5 +1,6 @@ import json +import schemas from chalicelib.core import dashboard from chalicelib.core import sourcemaps, sessions from chalicelib.utils import ch_client @@ -265,7 +266,7 @@ def get_details(project_id, error_id, user_id, **data): COALESCE((SELECT TRUE FROM public.user_favorite_errors AS fe WHERE pe.error_id = fe.error_id - AND fe.user_id = %(user_id)s), FALSE) AS favorite, + AND fe.user_id = %(userId)s), FALSE) AS favorite, True AS viewed FROM public.errors AS pe INNER JOIN events.errors AS ee USING (error_id) @@ -274,7 +275,7 @@ def get_details(project_id, error_id, user_id, **data): AND error_id = %(error_id)s ORDER BY start_ts DESC LIMIT 1;""", - {"project_id": project_id, "error_id": error_id, "user_id": user_id}) + {"project_id": project_id, "error_id": error_id, "userId": user_id}) cur.execute(query=query) status = cur.fetchone() @@ -443,54 +444,74 @@ def __get_sort_key(key): }.get(key, 'max_datetime') -def search(data, project_id, user_id, flows=False, status="ALL", favorite_only=False): +def search(data: schemas.SearchErrorsSchema, project_id, user_id, flows=False, status="ALL", favorite_only=False): + empty_response = {"data": { + 'total': 0, + 'errors': [] + }} status = status.upper() if status.lower() not in ['all', 'unresolved', 'resolved', 'ignored']: return {"errors": ["invalid error status"]} - ch_sub_query = __get_basic_constraints(data.get('platform')) + platform = None + for f in data.filters: + if f.type == schemas.FilterType.platform and len(f.value) > 0: + platform = f.value[0] + ch_sub_query = __get_basic_constraints(platform) ch_sub_query.append("source ='js_exception'") statuses = [] error_ids = None - if data.get("startDate") is None: - data["startDate"] = TimeUTC.now(-30) - if data.get("endDate") is None: - data["endDate"] = TimeUTC.now(1) - if len(data.get("events", [])) > 0 or len(data.get("filters", [])) > 0 or status != "ALL" or favorite_only: + if data.startDate is None: + data.startDate = TimeUTC.now(-30) + if data.endDate is None: + data.endDate = TimeUTC.now(1) + if len(data.events) > 0 or len(data.filters) > 0 or status != "ALL": + # if favorite_only=True search for sessions associated with favorite_error statuses = sessions.search2_pg(data=data, project_id=project_id, user_id=user_id, errors_only=True, error_status=status, favorite_only=favorite_only) - error_ids = [e["error_id"] for e in statuses] if len(statuses) == 0: - return {"data": { - 'total': 0, - 'errors': [] - }} - with ch_client.ClickHouseClient() as ch: - if data.get("startDate") is None: - data["startDate"] = TimeUTC.now(-7) - if data.get("endDate") is None: - data["endDate"] = TimeUTC.now() - density = data.get("density", 7) - step_size = __get_step_size(data["startDate"], data["endDate"], density) + return empty_response + error_ids = [e["error_id"] for e in statuses] + with ch_client.ClickHouseClient() as ch, pg_client.PostgresClient() as cur: + if data.startDate is None: + data.startDate = TimeUTC.now(-7) + if data.endDate is None: + data.endDate = TimeUTC.now() + step_size = __get_step_size(data.startDate, data.endDate, data.density) sort = __get_sort_key('datetime') - if data.get("sort") is not None: - sort = __get_sort_key(data["sort"]) + if data.sort is not None: + sort = __get_sort_key(data.sort) order = "DESC" - if data.get("order") is not None: - order = data["order"] - + if data.order is not None: + order = data.order + extra_join = "" params = { - "startDate": data['startDate'], - "endDate": data['endDate'], + "startDate": data.startDate, + "endDate": data.endDate, "project_id": project_id, "userId": user_id, "step_size": step_size} + if favorite_only: + cur.execute(cur.mogrify(f"""SELECT error_id + FROM public.user_favorite_errors + WHERE user_id = %(userId)s + {"" if error_ids is None else "AND error_id IN %(error_ids)s"}""", + {"userId": user_id, "error_ids": tuple(error_ids or [])})) + error_ids = cur.fetchall() + if len(error_ids) == 0: + return empty_response + error_ids = [e["error_id"] for e in error_ids] + if error_ids is not None: params["error_ids"] = tuple(error_ids) ch_sub_query.append("error_id IN %(error_ids)s") + main_ch_query = f"""\ SELECT COUNT(DISTINCT error_id) AS count FROM errors WHERE {" AND ".join(ch_sub_query)};""" + # print("------------") + # print(ch.client().substitute_params(main_ch_query, params)) + # print("------------") total = ch.execute(query=main_ch_query, params=params)[0]["count"] if flows: return {"data": {"count": total}} @@ -510,7 +531,7 @@ def search(data, project_id, user_id, flows=False, status="ALL", favorite_only=F WHERE {" AND ".join(ch_sub_query)} GROUP BY error_id, name, message ORDER BY {sort} {order} - LIMIT 1001) AS details INNER JOIN (SELECT error_id AS error_id, toUnixTimestamp(MAX(datetime))*1000 AS last_occurrence, toUnixTimestamp(MIN(datetime))*1000 AS first_occurrence + LIMIT 200) AS details INNER JOIN (SELECT error_id AS error_id, toUnixTimestamp(MAX(datetime))*1000 AS last_occurrence, toUnixTimestamp(MIN(datetime))*1000 AS first_occurrence FROM errors GROUP BY error_id) AS time_details ON details.error_id=time_details.error_id @@ -527,23 +548,22 @@ def search(data, project_id, user_id, flows=False, status="ALL", favorite_only=F # print(main_ch_query % params) rows = ch.execute(query=main_ch_query, params=params) if len(statuses) == 0: - with pg_client.PostgresClient() as cur: - query = cur.mogrify( - """SELECT error_id, status, parent_error_id, payload, - COALESCE((SELECT TRUE - FROM public.user_favorite_errors AS fe - WHERE errors.error_id = fe.error_id - AND fe.user_id = %(user_id)s LIMIT 1), FALSE) AS favorite, - COALESCE((SELECT TRUE - FROM public.user_viewed_errors AS ve - WHERE errors.error_id = ve.error_id - AND ve.user_id = %(user_id)s LIMIT 1), FALSE) AS viewed - FROM public.errors - WHERE project_id = %(project_id)s AND error_id IN %(error_ids)s;""", - {"project_id": project_id, "error_ids": tuple([r["error_id"] for r in rows]), - "user_id": user_id}) - cur.execute(query=query) - statuses = cur.fetchall() + query = cur.mogrify( + """SELECT error_id, status, parent_error_id, payload, + COALESCE((SELECT TRUE + FROM public.user_favorite_errors AS fe + WHERE errors.error_id = fe.error_id + AND fe.user_id = %(userId)s LIMIT 1), FALSE) AS favorite, + COALESCE((SELECT TRUE + FROM public.user_viewed_errors AS ve + WHERE errors.error_id = ve.error_id + AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + FROM public.errors + WHERE project_id = %(project_id)s AND error_id IN %(error_ids)s;""", + {"project_id": project_id, "error_ids": tuple([r["error_id"] for r in rows]), + "userId": user_id}) + cur.execute(query=query) + statuses = cur.fetchall() statuses = { s["error_id"]: s for s in statuses } @@ -565,9 +585,9 @@ def search(data, project_id, user_id, flows=False, status="ALL", favorite_only=F r["chart"] = list(r["chart"]) for i in range(len(r["chart"])): r["chart"][i] = {"timestamp": r["chart"][i][0], "count": r["chart"][i][1]} - r["chart"] = dashboard.__complete_missing_steps(rows=r["chart"], start_time=data["startDate"], - end_time=data["endDate"], - density=density, neutral={"count": 0}) + r["chart"] = dashboard.__complete_missing_steps(rows=r["chart"], start_time=data.startDate, + end_time=data.endDate, + density=data.density, neutral={"count": 0}) offset = len(rows) rows = [r for r in rows if r["stack"] is None or (len(r["stack"]) == 0 or len(r["stack"]) > 1 @@ -593,7 +613,7 @@ def __save_stacktrace(error_id, data): def get_trace(project_id, error_id): - error = get(error_id=error_id) + error = get(error_id=error_id, family=False) if error is None: return {"errors": ["error not found"]} if error.get("source", "") != "js_exception": @@ -766,7 +786,7 @@ def format_first_stack_frame(error): def stats(project_id, user_id, startTimestamp=TimeUTC.now(delta_days=-7), endTimestamp=TimeUTC.now()): with pg_client.PostgresClient() as cur: query = cur.mogrify( - """WITH user_viewed AS (SELECT error_id FROM public.user_viewed_errors WHERE user_id = %(user_id)s) + """WITH user_viewed AS (SELECT error_id FROM public.user_viewed_errors WHERE user_id = %(userId)s) SELECT COUNT(timed_errors.*) AS unresolved_and_unviewed FROM (SELECT root_error.error_id FROM events.errors @@ -780,7 +800,7 @@ def stats(project_id, user_id, startTimestamp=TimeUTC.now(delta_days=-7), endTim AND user_viewed.error_id ISNULL LIMIT 1 ) AS timed_errors;""", - {"project_id": project_id, "user_id": user_id, "startTimestamp": startTimestamp, + {"project_id": project_id, "userId": user_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp}) cur.execute(query=query) row = cur.fetchone() diff --git a/ee/utilities/servers/websocket.js b/ee/utilities/servers/websocket.js index 34f045a51..e087dba31 100644 --- a/ee/utilities/servers/websocket.js +++ b/ee/utilities/servers/websocket.js @@ -55,7 +55,7 @@ const socketsListByProject = function (req, res) { if (process.env.uws !== "true") { res.statusCode = 200; res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify()); + res.end(JSON.stringify(result)); } else { res.writeStatus('200 OK').writeHeader('Content-Type', 'application/json').end(JSON.stringify(result)); } diff --git a/frontend/app/components/Funnels/FunnelGraph/FunnelGraph.js b/frontend/app/components/Funnels/FunnelGraph/FunnelGraph.js index ec94451d7..906843394 100644 --- a/frontend/app/components/Funnels/FunnelGraph/FunnelGraph.js +++ b/frontend/app/components/Funnels/FunnelGraph/FunnelGraph.js @@ -211,9 +211,9 @@ function FunnelGraph(props) { strokeWidth={0} interval={0} tick ={{ fill: '#666', fontSize: 12 }} - xAxisId={0} + xAxisId={0} /> - '"' + val + '"'} - /> + /> */} Styles.tickFormatter(val)} /> diff --git a/frontend/app/components/Funnels/FunnelHeader/FunnelHeader.js b/frontend/app/components/Funnels/FunnelHeader/FunnelHeader.js index a94845294..6130351e2 100644 --- a/frontend/app/components/Funnels/FunnelHeader/FunnelHeader.js +++ b/frontend/app/components/Funnels/FunnelHeader/FunnelHeader.js @@ -1,7 +1,7 @@ import React, { useEffect, useState } from 'react'; import { Icon, BackLink, IconButton, Dropdown, Popup, TextEllipsis, Button } from 'UI'; import { remove as deleteFunnel, fetch, fetchInsights, fetchIssuesFiltered, fetchSessionsFiltered } from 'Duck/funnels'; -import { editFilter, addFilter } from 'Duck/funnels'; +import { editFilter, refresh, addFilter } from 'Duck/funnels'; import DateRange from 'Shared/DateRange'; import { connect } from 'react-redux'; import { confirm } from 'UI/Confirmation'; @@ -19,15 +19,11 @@ const Info = ({ label = '', value = '', className = 'mx-4' }) => { const FunnelHeader = (props) => { const { funnel, insights, funnels, onBack, funnelId, showFilters = false, renameHandler } = props; - const [showSaveModal, setShowSaveModal] = useState(false) const writeOption = (e, { name, value }) => { - props.fetch(value) - props.fetchInsights(value, {}) - props.fetchIssuesFiltered(value, {}) - props.fetchSessionsFiltered(value, {}) props.redirect(value) + props.fetch(value).then(() => props.refresh(value)) } const deleteFunnel = async (e, funnel) => { @@ -44,11 +40,12 @@ const FunnelHeader = (props) => { } const onDateChange = (e) => { - props.editFilter(e, funnel.funnelId); + props.editFilter(e, funnelId); } const options = funnels.map(({ funnelId, name }) => ({ text: name, value: funnelId })).toJS(); const selectedFunnel = funnels.filter(i => i.funnelId === parseInt(funnelId)).first() || {}; + const eventsCount = funnel.filter.filters.filter(i => i.isEvent).size; return (
@@ -75,7 +72,7 @@ const FunnelHeader = (props) => { selectOnBlur={false} icon={ } /> - + - @@ -113,4 +110,4 @@ const FunnelHeader = (props) => { export default connect(state => ({ funnel: state.getIn([ 'funnels', 'instance' ]), -}), { editFilter, deleteFunnel, fetch, fetchInsights, fetchIssuesFiltered, fetchSessionsFiltered })(FunnelHeader) +}), { editFilter, deleteFunnel, fetch, fetchInsights, fetchIssuesFiltered, fetchSessionsFiltered, refresh })(FunnelHeader) diff --git a/frontend/app/components/shared/Filters/FilterAutoComplete/FilterAutoComplete.tsx b/frontend/app/components/shared/Filters/FilterAutoComplete/FilterAutoComplete.tsx index 92baa3d51..62ec51c9d 100644 --- a/frontend/app/components/shared/Filters/FilterAutoComplete/FilterAutoComplete.tsx +++ b/frontend/app/components/shared/Filters/FilterAutoComplete/FilterAutoComplete.tsx @@ -60,7 +60,7 @@ function FilterAutoComplete(props: Props) { .finally(() => setLoading(false)); } - const debouncedRequestValues = React.useCallback(debounce(requestValues, 300), []); + const debouncedRequestValues = React.useCallback(debounce(requestValues, 1000), []); const onInputChange = ({ target: { value } }) => { setQuery(value); diff --git a/frontend/app/components/shared/MainSearchBar/MainSearchBar.tsx b/frontend/app/components/shared/MainSearchBar/MainSearchBar.tsx index 0ed764339..e6cdbd00c 100644 --- a/frontend/app/components/shared/MainSearchBar/MainSearchBar.tsx +++ b/frontend/app/components/shared/MainSearchBar/MainSearchBar.tsx @@ -8,15 +8,16 @@ import { connect } from 'react-redux'; interface Props { clearSearch: () => void; appliedFilter: any; + optionsReady: boolean; } const MainSearchBar = (props: Props) => { - const { appliedFilter } = props; + const { appliedFilter, optionsReady } = props; const hasFilters = appliedFilter && appliedFilter.filters && appliedFilter.filters.size > 0; return (
- + {optionsReady && } { } export default connect(state => ({ appliedFilter: state.getIn(['search', 'instance']), + optionsReady: state.getIn(['customFields', 'optionsReady']) }), { clearSearch })(MainSearchBar); \ No newline at end of file diff --git a/frontend/app/duck/customField.js b/frontend/app/duck/customField.js index ad93c21c6..9a8fdcc0e 100644 --- a/frontend/app/duck/customField.js +++ b/frontend/app/duck/customField.js @@ -31,6 +31,7 @@ const initialState = Map({ list: List(), instance: CustomField(), sources: List(), + optionsReady: false }); const reducer = (state = initialState, action = {}) => { @@ -40,7 +41,8 @@ const reducer = (state = initialState, action = {}) => { addElementToFiltersMap(FilterCategory.METADATA, item.key); addElementToLiveFiltersMap(FilterCategory.METADATA, item.key); }); - return state.set('list', List(action.data).map(CustomField)) //.concat(defaultMeta)) + return state.set('list', List(action.data).map(CustomField)) + .set('optionsReady', true) //.concat(defaultMeta)) case FETCH_SOURCES_SUCCESS: return state.set('sources', List(action.data.map(({ value, ...item}) => ({label: value, key: value, ...item}))).map(CustomField)) case SAVE_SUCCESS: diff --git a/frontend/app/types/filter/newFilter.js b/frontend/app/types/filter/newFilter.js index 29cb324b1..6963643f7 100644 --- a/frontend/app/types/filter/newFilter.js +++ b/frontend/app/types/filter/newFilter.js @@ -6,6 +6,7 @@ import { capitalize } from 'App/utils'; const countryOptions = Object.keys(countries).map(i => ({ text: countries[i], value: i })); const containsFilters = [{ key: 'contains', text: 'contains', value: 'contains' }] +export const metaFilter = { key: FilterKey.METADATA, type: FilterType.MULTIPLE, category: FilterCategory.METADATA, label: 'Metadata', operator: 'is', operatorOptions: filterOptions.stringOperators, icon: 'filters/metadata' }; export const filtersMap = { // EVENTS [FilterKey.CLICK]: { key: FilterKey.CLICK, type: FilterType.MULTIPLE, category: FilterCategory.INTERACTIONS, label: 'Click', operator: 'on', operatorOptions: filterOptions.targetOperators, icon: 'filters/click', isEvent: true }, @@ -31,7 +32,6 @@ export const filtersMap = { [FilterKey.ERROR]: { key: FilterKey.ERROR, type: FilterType.MULTIPLE, category: FilterCategory.JAVASCRIPT, label: 'Error', operator: 'is', operatorOptions: filterOptions.stringOperators, icon: 'filters/error', isEvent: true }, // [FilterKey.METADATA]: { key: FilterKey.METADATA, type: FilterType.MULTIPLE, category: FilterCategory.METADATA, label: 'Metadata', operator: 'is', operatorOptions: filterOptions.stringOperators, icon: 'filters/metadata', isEvent: true }, - // FILTERS [FilterKey.USER_OS]: { key: FilterKey.USER_OS, type: FilterType.MULTIPLE, category: FilterCategory.GEAR, label: 'User OS', operator: 'is', operatorOptions: filterOptions.stringOperators, icon: 'filters/os' }, [FilterKey.USER_BROWSER]: { key: FilterKey.USER_BROWSER, type: FilterType.MULTIPLE, category: FilterCategory.GEAR, label: 'User Browser', operator: 'is', operatorOptions: filterOptions.stringOperators, icon: 'filters/browser' }, @@ -133,7 +133,11 @@ export default Record({ }) _filter = subFilterMap[type] } else { - _filter = filtersMap[type]; + if (type === FilterKey.METADATA) { + _filter = filtersMap[filter.source]; + } else { + _filter = filtersMap[type]; + } } return { ...filter, diff --git a/frontend/app/types/funnel.js b/frontend/app/types/funnel.js index 0a97cf944..d9a27e44d 100644 --- a/frontend/app/types/funnel.js +++ b/frontend/app/types/funnel.js @@ -51,8 +51,9 @@ export default Record({ } }, fromJS: ({ stages = [], filter, activeStages = null, ...rest }) => { - let _stages = stages.map(stage => { - stage.label = getRedableName(stage.type, stage.value); + let _stages = stages.map((stage, index) => { + // stage.label = getRedableName(stage.type, stage.value); + stage.label = `Step ${index + 1}`; return stage; }); @@ -70,16 +71,19 @@ export default Record({ return { ...rest, - stages: _stages.length > 0 ? _stages.map(stage => { + stages: _stages.length > 0 ? _stages.map((stage, index) => { if (!stage) return; - stage.label = getRedableName(stage); + // stage.label = getRedableName(stage); + stage.label = `Step ${index + 1}`; return stage; }) : [], affectedUsers, lostConversions, conversionImpact, - firstStage: firstStage && firstStage.label + ' ' + truncate(firstStage.value || '', 10) || '', - lastStage: lastStage && lastStage.label + ' ' + truncate(lastStage.value || '', 10) || '', + // firstStage: firstStage && firstStage.label + ' ' + truncate(firstStage.value || '', 10) || '', + // lastStage: lastStage && lastStage.label + ' ' + truncate(lastStage.value || '', 10) || '', + firstStage: firstStage && firstStage.label || '', + lastStage: lastStage && lastStage.label || '', filter: Filter(filter), sessionsCount: lastStage && lastStage.sessionsCount, stepsCount: stages ? stages.length : 0,