diff --git a/api/auth/auth_project.py b/api/auth/auth_project.py index 98a495bbb..6f842916b 100644 --- a/api/auth/auth_project.py +++ b/api/auth/auth_project.py @@ -15,10 +15,12 @@ class ProjectAuthorizer: if len(request.path_params.keys()) == 0 or request.path_params.get(self.project_identifier) is None: return current_user: schemas.CurrentContext = await OR_context(request) - project_identifier = request.path_params[self.project_identifier] + value = request.path_params[self.project_identifier] if (self.project_identifier == "projectId" \ - and projects.get_project(project_id=project_identifier, tenant_id=current_user.tenant_id) is None) \ - or (self.project_identifier.lower() == "projectKey" \ - and projects.get_internal_project_id(project_key=project_identifier) is None): + and not (isinstance(value, int) or isinstance(value, str) and value.isnumeric()) + and projects.get_project(project_id=value, tenant_id=current_user.tenant_id) is None) \ + or (self.project_identifier == "projectKey" \ + and projects.get_internal_project_id(project_key=value) is None): print("project not found") + print(value) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="project not found.") diff --git a/api/chalicelib/core/autocomplete.py b/api/chalicelib/core/autocomplete.py new file mode 100644 index 000000000..595342a9b --- /dev/null +++ b/api/chalicelib/core/autocomplete.py @@ -0,0 +1,86 @@ +from chalicelib.utils import helper +from chalicelib.utils import pg_client +import schemas +from chalicelib.utils.event_filter_definition import SupportedFilter, Event + + +def __get_autocomplete_table(value, project_id): + autocomplete_events = [schemas.FilterType.rev_id, + schemas.EventType.click, + schemas.FilterType.user_device, + schemas.FilterType.user_id, + schemas.FilterType.user_browser, + schemas.FilterType.user_os, + schemas.EventType.custom, + schemas.FilterType.user_country, + schemas.EventType.location, + schemas.EventType.input] + autocomplete_events.sort() + sub_queries = [] + for e in autocomplete_events: + sub_queries.append(f"""(SELECT type, value + FROM public.autocomplete + WHERE project_id = %(project_id)s + AND type= '{e}' + AND value ILIKE %(svalue)s + LIMIT 5)""") + if len(value) > 2: + sub_queries.append(f"""(SELECT type, value + FROM public.autocomplete + WHERE project_id = %(project_id)s + AND type= '{e}' + AND value ILIKE %(value)s + LIMIT 5)""") + with pg_client.PostgresClient() as cur: + query = cur.mogrify(" UNION DISTINCT ".join(sub_queries) + ";", + {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value)}) + try: + cur.execute(query) + except Exception as err: + print("--------- AUTOCOMPLETE SEARCH QUERY EXCEPTION -----------") + print(query.decode('UTF-8')) + print("--------- VALUE -----------") + print(value) + print("--------------------") + raise err + results = helper.list_to_camel_case(cur.fetchall()) + return results + + +def __generic_query(typename, value_length=None): + if value_length is None or value_length > 2: + return f"""(SELECT DISTINCT value, type + FROM public.autocomplete + WHERE + project_id = %(project_id)s + AND type='{typename}' + AND value ILIKE %(svalue)s + LIMIT 5) + UNION DISTINCT + (SELECT DISTINCT value, type + FROM public.autocomplete + WHERE + project_id = %(project_id)s + AND type='{typename}' + AND value ILIKE %(value)s + LIMIT 5);""" + return f"""SELECT DISTINCT value, type + FROM public.autocomplete + WHERE + project_id = %(project_id)s + AND type='{typename}' + AND value ILIKE %(svalue)s + LIMIT 10;""" + + +def __generic_autocomplete(event: Event): + def f(project_id, value, key=None, source=None): + with pg_client.PostgresClient() as cur: + query = __generic_query(event.ui_type, value_length=len(value)) + params = {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value)} + cur.execute(cur.mogrify(query, params)) + return helper.list_to_camel_case(cur.fetchall()) + + return f diff --git a/api/chalicelib/core/events.py b/api/chalicelib/core/events.py index dd9562de1..e2c979799 100644 --- a/api/chalicelib/core/events.py +++ b/api/chalicelib/core/events.py @@ -1,10 +1,14 @@ import schemas from chalicelib.core import issues -from chalicelib.core import sessions_metas, metadata +from chalicelib.core import metadata +from chalicelib.core import sessions_metas + from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC from chalicelib.utils.event_filter_definition import SupportedFilter, Event +from chalicelib.core import autocomplete + def get_customs_by_sessionId2_pg(session_id, project_id): with pg_client.PostgresClient() as cur: @@ -92,11 +96,6 @@ def get_by_sessionId2_pg(session_id, project_id, group_clickrage=False): return rows -def __get_data_for_extend(data): - if "errors" not in data: - return data["data"] - - def __pg_errors_query(source=None, value_length=None): if value_length is None or value_length > 2: return f"""((SELECT DISTINCT ON(lg.message) @@ -110,7 +109,7 @@ def __pg_errors_query(source=None, value_length=None): AND lg.project_id = %(project_id)s {"AND source = %(source)s" if source is not None else ""} LIMIT 5) - UNION ALL + UNION DISTINCT (SELECT DISTINCT ON(lg.name) lg.name AS value, source, @@ -122,7 +121,7 @@ def __pg_errors_query(source=None, value_length=None): AND lg.project_id = %(project_id)s {"AND source = %(source)s" if source is not None else ""} LIMIT 5) - UNION + UNION DISTINCT (SELECT DISTINCT ON(lg.message) lg.message AS value, source, @@ -134,7 +133,7 @@ def __pg_errors_query(source=None, value_length=None): AND lg.project_id = %(project_id)s {"AND source = %(source)s" if source is not None else ""} LIMIT 5) - UNION ALL + UNION DISTINCT (SELECT DISTINCT ON(lg.name) lg.name AS value, source, @@ -157,7 +156,7 @@ def __pg_errors_query(source=None, value_length=None): AND lg.project_id = %(project_id)s {"AND source = %(source)s" if source is not None else ""} LIMIT 5) - UNION ALL + UNION DISTINCT (SELECT DISTINCT ON(lg.name) lg.name AS value, source, @@ -177,8 +176,7 @@ def __search_pg_errors(project_id, value, key=None, source=None): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify(__pg_errors_query(source, - value_length=len(value) \ - if SUPPORTED_TYPES[event_type.ERROR.ui_type].change_by_length else None), + value_length=len(value)), {"project_id": project_id, "value": helper.string_to_sql_like(value), "svalue": helper.string_to_sql_like("^" + value), "source": source})) @@ -189,7 +187,7 @@ def __search_pg_errors(project_id, value, key=None, source=None): def __search_pg_errors_ios(project_id, value, key=None, source=None): now = TimeUTC.now() - if SUPPORTED_TYPES[event_type.ERROR_IOS.ui_type].change_by_length is False or len(value) > 2: + if len(value) > 2: query = f"""(SELECT DISTINCT ON(lg.reason) lg.reason AS value, '{event_type.ERROR_IOS.ui_type}' AS type @@ -268,7 +266,7 @@ def __search_pg_metadata(project_id, value, key=None, source=None): for k in meta_keys.keys(): colname = metadata.index_to_colname(meta_keys[k]) - if SUPPORTED_TYPES[event_type.METADATA.ui_type].change_by_length is False or len(value) > 2: + if len(value) > 2: sub_from.append(f"""((SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key FROM public.sessions WHERE project_id = %(project_id)s @@ -294,48 +292,6 @@ def __search_pg_metadata(project_id, value, key=None, source=None): return results -def __generic_query(typename, value_length=None): - if value_length is None or value_length > 2: - return f"""(SELECT DISTINCT value, type - FROM public.autocomplete - WHERE - project_id = %(project_id)s - AND type='{typename}' - AND value ILIKE %(svalue)s - LIMIT 5) - UNION - (SELECT DISTINCT value, type - FROM public.autocomplete - WHERE - project_id = %(project_id)s - AND type='{typename}' - AND value ILIKE %(value)s - LIMIT 5);""" - return f"""SELECT DISTINCT value, type - FROM public.autocomplete - WHERE - project_id = %(project_id)s - AND type='{typename}' - AND value ILIKE %(svalue)s - LIMIT 10;""" - - -def __generic_autocomplete(event: Event): - def f(project_id, value, key=None, source=None): - with pg_client.PostgresClient() as cur: - cur.execute( - cur.mogrify( - __generic_query(event.ui_type, - value_length=len(value) \ - if SUPPORTED_TYPES[event.ui_type].change_by_length \ - else None), - {"project_id": project_id, "value": helper.string_to_sql_like(value), - "svalue": helper.string_to_sql_like("^" + value)})) - return helper.list_to_camel_case(cur.fetchall()) - - return f - - class event_type: CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label") INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label") @@ -358,99 +314,65 @@ class event_type: SUPPORTED_TYPES = { - event_type.CLICK.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CLICK), - query=__generic_query(typename=event_type.CLICK.ui_type), - change_by_length=True), - event_type.INPUT.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.INPUT), - query=__generic_query(typename=event_type.INPUT.ui_type), - change_by_length=True), - event_type.LOCATION.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.LOCATION), - query=__generic_query(typename=event_type.LOCATION.ui_type), - change_by_length=True), - event_type.CUSTOM.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CUSTOM), - query=__generic_query(typename=event_type.CUSTOM.ui_type), - change_by_length=True), - event_type.REQUEST.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.REQUEST), - query=__generic_query(typename=event_type.REQUEST.ui_type), - change_by_length=True), - event_type.GRAPHQL.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.GRAPHQL), - query=__generic_query(typename=event_type.GRAPHQL.ui_type), - change_by_length=True), - event_type.STATEACTION.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.STATEACTION), - query=__generic_query(typename=event_type.STATEACTION.ui_type), - change_by_length=True), + event_type.CLICK.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.CLICK), + query=autocomplete.__generic_query(typename=event_type.CLICK.ui_type)), + event_type.INPUT.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.INPUT), + query=autocomplete.__generic_query(typename=event_type.INPUT.ui_type)), + event_type.LOCATION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.LOCATION), + query=autocomplete.__generic_query( + typename=event_type.LOCATION.ui_type)), + event_type.CUSTOM.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.CUSTOM), + query=autocomplete.__generic_query(typename=event_type.CUSTOM.ui_type)), + event_type.REQUEST.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.REQUEST), + query=autocomplete.__generic_query( + typename=event_type.REQUEST.ui_type)), + event_type.GRAPHQL.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.GRAPHQL), + query=autocomplete.__generic_query( + typename=event_type.GRAPHQL.ui_type)), + event_type.STATEACTION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.STATEACTION), + query=autocomplete.__generic_query( + typename=event_type.STATEACTION.ui_type)), event_type.ERROR.ui_type: SupportedFilter(get=__search_pg_errors, - query=None, change_by_length=True), + query=None), event_type.METADATA.ui_type: SupportedFilter(get=__search_pg_metadata, - query=None, change_by_length=True), + query=None), # IOS - event_type.CLICK_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CLICK_IOS), - query=__generic_query(typename=event_type.CLICK_IOS.ui_type), - change_by_length=True), - event_type.INPUT_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.INPUT_IOS), - query=__generic_query(typename=event_type.INPUT_IOS.ui_type), - change_by_length=True), - event_type.VIEW_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.VIEW_IOS), - query=__generic_query(typename=event_type.VIEW_IOS.ui_type), - change_by_length=True), - event_type.CUSTOM_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CUSTOM_IOS), - query=__generic_query(typename=event_type.CUSTOM_IOS.ui_type), - change_by_length=True), - event_type.REQUEST_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.REQUEST_IOS), - query=__generic_query(typename=event_type.REQUEST_IOS.ui_type), - change_by_length=True), + event_type.CLICK_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.CLICK_IOS), + query=autocomplete.__generic_query( + typename=event_type.CLICK_IOS.ui_type)), + event_type.INPUT_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.INPUT_IOS), + query=autocomplete.__generic_query( + typename=event_type.INPUT_IOS.ui_type)), + event_type.VIEW_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.VIEW_IOS), + query=autocomplete.__generic_query( + typename=event_type.VIEW_IOS.ui_type)), + event_type.CUSTOM_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.CUSTOM_IOS), + query=autocomplete.__generic_query( + typename=event_type.CUSTOM_IOS.ui_type)), + event_type.REQUEST_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.REQUEST_IOS), + query=autocomplete.__generic_query( + typename=event_type.REQUEST_IOS.ui_type)), event_type.ERROR_IOS.ui_type: SupportedFilter(get=__search_pg_errors_ios, - query=None, change_by_length=True), + query=None), } -def __get_autocomplete_table(value, project_id): - autocomplete_events = [schemas.FilterType.rev_id, - schemas.EventType.click, - schemas.FilterType.user_device, - schemas.FilterType.user_id, - schemas.FilterType.user_browser, - schemas.FilterType.user_os, - schemas.EventType.custom, - schemas.FilterType.user_country, - schemas.EventType.location, - schemas.EventType.input] - autocomplete_events.sort() - sub_queries = [] - for e in autocomplete_events: - sub_queries.append(f"""(SELECT type, value - FROM public.autocomplete - WHERE project_id = %(project_id)s - AND type= '{e}' - AND value ILIKE %(svalue)s - LIMIT 5)""") - if len(value) > 2: - sub_queries.append(f"""(SELECT type, value - FROM public.autocomplete - WHERE project_id = %(project_id)s - AND type= '{e}' - AND value ILIKE %(value)s - LIMIT 5)""") +def get_errors_by_session_id(session_id, project_id): with pg_client.PostgresClient() as cur: - query = cur.mogrify(" UNION ".join(sub_queries) + ";", - {"project_id": project_id, "value": helper.string_to_sql_like(value), - "svalue": helper.string_to_sql_like("^" + value)}) - try: - cur.execute(query) - except Exception as err: - print("--------- AUTOCOMPLETE SEARCH QUERY EXCEPTION -----------") - print(query.decode('UTF-8')) - print("--------- VALUE -----------") - print(value) - print("--------------------") - raise err - results = helper.list_to_camel_case(cur.fetchall()) - return results + cur.execute(cur.mogrify(f"""\ + SELECT er.*,ur.*, er.timestamp - s.start_ts AS time + FROM {event_type.ERROR.table} AS er INNER JOIN public.errors AS ur USING (error_id) INNER JOIN public.sessions AS s USING (session_id) + WHERE er.session_id = %(session_id)s AND s.project_id=%(project_id)s + ORDER BY timestamp;""", {"session_id": session_id, "project_id": project_id})) + errors = cur.fetchall() + for e in errors: + e["stacktrace_parsed_at"] = TimeUTC.datetime_to_timestamp(e["stacktrace_parsed_at"]) + return helper.list_to_camel_case(errors) def search(text, event_type, project_id, source, key): if not event_type: - return {"data": __get_autocomplete_table(text, project_id)} + return {"data": autocomplete.__get_autocomplete_table(text, project_id)} if event_type in SUPPORTED_TYPES.keys(): rows = SUPPORTED_TYPES[event_type].get(project_id=project_id, value=text, key=key, source=source) @@ -470,16 +392,3 @@ def search(text, event_type, project_id, source, key): return {"errors": ["unsupported event"]} return {"data": rows} - - -def get_errors_by_session_id(session_id, project_id): - with pg_client.PostgresClient() as cur: - cur.execute(cur.mogrify(f"""\ - SELECT er.*,ur.*, er.timestamp - s.start_ts AS time - FROM {event_type.ERROR.table} AS er INNER JOIN public.errors AS ur USING (error_id) INNER JOIN public.sessions AS s USING (session_id) - WHERE er.session_id = %(session_id)s AND s.project_id=%(project_id)s - ORDER BY timestamp;""", {"session_id": session_id, "project_id": project_id})) - errors = cur.fetchall() - for e in errors: - e["stacktrace_parsed_at"] = TimeUTC.datetime_to_timestamp(e["stacktrace_parsed_at"]) - return helper.list_to_camel_case(errors) diff --git a/api/chalicelib/core/sessions_metas.py b/api/chalicelib/core/sessions_metas.py index 07aad2ee4..b956e0f62 100644 --- a/api/chalicelib/core/sessions_metas.py +++ b/api/chalicelib/core/sessions_metas.py @@ -113,8 +113,7 @@ def __generic_autocomplete(typename): def f(project_id, text): with pg_client.PostgresClient() as cur: query = cur.mogrify(__generic_query(typename, - value_length=len(text) \ - if SUPPORTED_TYPES[typename].change_by_length else None), + value_length=len(text)), {"project_id": project_id, "value": helper.string_to_sql_like(text), "svalue": helper.string_to_sql_like("^" + text)}) @@ -128,74 +127,57 @@ def __generic_autocomplete(typename): SUPPORTED_TYPES = { schemas.FilterType.user_os: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.user_os), - query=__generic_query(typename=schemas.FilterType.user_os), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.user_os)), schemas.FilterType.user_browser: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.user_browser), - query=__generic_query(typename=schemas.FilterType.user_browser), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.user_browser)), schemas.FilterType.user_device: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.user_device), - query=__generic_query(typename=schemas.FilterType.user_device), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.user_device)), schemas.FilterType.user_country: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.user_country), - query=__generic_query(typename=schemas.FilterType.user_country), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.user_country)), schemas.FilterType.user_id: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.user_id), - query=__generic_query(typename=schemas.FilterType.user_id), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.user_id)), schemas.FilterType.user_anonymous_id: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.user_anonymous_id), - query=__generic_query(typename=schemas.FilterType.user_anonymous_id), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.user_anonymous_id)), schemas.FilterType.rev_id: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.rev_id), - query=__generic_query(typename=schemas.FilterType.rev_id), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.rev_id)), schemas.FilterType.referrer: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.referrer), - query=__generic_query(typename=schemas.FilterType.referrer), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.referrer)), schemas.FilterType.utm_campaign: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.utm_campaign), - query=__generic_query(typename=schemas.FilterType.utm_campaign), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.utm_campaign)), schemas.FilterType.utm_medium: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.utm_medium), - query=__generic_query(typename=schemas.FilterType.utm_medium), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.utm_medium)), schemas.FilterType.utm_source: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.utm_source), - query=__generic_query(typename=schemas.FilterType.utm_source), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.utm_source)), # IOS schemas.FilterType.user_os_ios: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.user_os_ios), - query=__generic_query(typename=schemas.FilterType.user_os_ios), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.user_os_ios)), schemas.FilterType.user_device_ios: SupportedFilter( get=__generic_autocomplete( typename=schemas.FilterType.user_device_ios), - query=__generic_query(typename=schemas.FilterType.user_device_ios), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.user_device_ios)), schemas.FilterType.user_country_ios: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.user_country_ios), - query=__generic_query(typename=schemas.FilterType.user_country_ios), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.user_country_ios)), schemas.FilterType.user_id_ios: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.user_id_ios), - query=__generic_query(typename=schemas.FilterType.user_id_ios), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.user_id_ios)), schemas.FilterType.user_anonymous_id_ios: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.user_anonymous_id_ios), - query=__generic_query(typename=schemas.FilterType.user_anonymous_id_ios), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.user_anonymous_id_ios)), schemas.FilterType.rev_id_ios: SupportedFilter( get=__generic_autocomplete(typename=schemas.FilterType.rev_id_ios), - query=__generic_query(typename=schemas.FilterType.rev_id_ios), - change_by_length=True), + query=__generic_query(typename=schemas.FilterType.rev_id_ios)), } diff --git a/api/chalicelib/utils/event_filter_definition.py b/api/chalicelib/utils/event_filter_definition.py index b21d49b9c..93b1b9d5f 100644 --- a/api/chalicelib/utils/event_filter_definition.py +++ b/api/chalicelib/utils/event_filter_definition.py @@ -6,7 +6,6 @@ class Event: class SupportedFilter: - def __init__(self, get, query, change_by_length): + def __init__(self, get, query): self.get = get self.query = query - self.change_by_length = change_by_length diff --git a/ee/api/.gitignore b/ee/api/.gitignore index a0ca015ed..1654bb8d7 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -180,6 +180,7 @@ Pipfile /chalicelib/core/alerts.py /chalicelib/core/alerts_processor.py /chalicelib/core/announcements.py +/chalicelib/core/autocomplete.py /chalicelib/core/collaboration_slack.py /chalicelib/core/errors_favorite_viewed.py /chalicelib/core/events.py diff --git a/ee/api/auth/auth_project.py b/ee/api/auth/auth_project.py index c1e1d38cd..2c78041e0 100644 --- a/ee/api/auth/auth_project.py +++ b/ee/api/auth/auth_project.py @@ -15,13 +15,15 @@ class ProjectAuthorizer: if len(request.path_params.keys()) == 0 or request.path_params.get(self.project_identifier) is None: return current_user: schemas.CurrentContext = await OR_context(request) - project_identifier = request.path_params[self.project_identifier] + value = request.path_params[self.project_identifier] user_id = current_user.user_id if request.state.authorizer_identity == "jwt" else None if (self.project_identifier == "projectId" \ - and not projects.is_authorized(project_id=project_identifier, tenant_id=current_user.tenant_id, + and not projects.is_authorized(project_id=value, tenant_id=current_user.tenant_id, user_id=user_id)) \ - or (self.project_identifier.lower() == "projectKey" \ - and not projects.is_authorized(project_id=projects.get_internal_project_id(project_identifier), - tenant_id=current_user.tenant_id, user_id=user_id)): + or (self.project_identifier == "projectKey" \ + and not projects.is_authorized( + project_id=projects.get_internal_project_id(value), + tenant_id=current_user.tenant_id, user_id=user_id)): print("unauthorized project") + print(value) raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="unauthorized project.") diff --git a/ee/api/chalicelib/core/__init__.py b/ee/api/chalicelib/core/__init__.py index e69de29bb..01b1b7ee4 100644 --- a/ee/api/chalicelib/core/__init__.py +++ b/ee/api/chalicelib/core/__init__.py @@ -0,0 +1,6 @@ +from decouple import config + +if config("LEGACY_SEARCH", cast=bool, default=True): + from . import autocomplete as autocomplete +else: + from . import autocomplete_ee as autocomplete diff --git a/ee/api/chalicelib/core/autocomplete_ee.py b/ee/api/chalicelib/core/autocomplete_ee.py new file mode 100644 index 000000000..e96cd819b --- /dev/null +++ b/ee/api/chalicelib/core/autocomplete_ee.py @@ -0,0 +1,88 @@ +import schemas +from chalicelib.utils import ch_client +from chalicelib.utils import helper +from chalicelib.utils.event_filter_definition import Event + + +def __get_autocomplete_table(value, project_id): + autocomplete_events = [schemas.FilterType.rev_id, + schemas.EventType.click, + schemas.FilterType.user_device, + schemas.FilterType.user_id, + schemas.FilterType.user_browser, + schemas.FilterType.user_os, + schemas.EventType.custom, + schemas.FilterType.user_country, + schemas.EventType.location, + schemas.EventType.input] + autocomplete_events.sort() + sub_queries = [] + for e in autocomplete_events: + sub_queries.append(f"""(SELECT type, value + FROM final.autocomplete + WHERE project_id = %(project_id)s + AND type= '{e}' + AND value ILIKE %(svalue)s + LIMIT 5)""") + if len(value) > 2: + sub_queries.append(f"""(SELECT type, value + FROM final.autocomplete + WHERE project_id = %(project_id)s + AND type= '{e}' + AND value ILIKE %(value)s + LIMIT 5)""") + with ch_client.ClickHouseClient() as cur: + query = " UNION DISTINCT ".join(sub_queries) + ";" + params = {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value)} + results = [] + try: + results = cur.execute(query=query, params=params) + except Exception as err: + print("--------- CH AUTOCOMPLETE SEARCH QUERY EXCEPTION -----------") + print(cur.format(query=query, params=params)) + print("--------- PARAMS -----------") + print(params) + print("--------- VALUE -----------") + print(value) + print("--------------------") + raise err + return results + + +def __generic_query(typename, value_length=None): + if value_length is None or value_length > 2: + return f"""(SELECT DISTINCT value, type + FROM final.autocomplete + WHERE + project_id = %(project_id)s + AND type='{typename}' + AND value ILIKE %(svalue)s + LIMIT 5) + UNION DISTINCT + (SELECT DISTINCT value, type + FROM final.autocomplete + WHERE + project_id = %(project_id)s + AND type='{typename}' + AND value ILIKE %(value)s + LIMIT 5);""" + return f"""SELECT DISTINCT value, type + FROM final.autocomplete + WHERE + project_id = %(project_id)s + AND type='{typename}' + AND value ILIKE %(svalue)s + LIMIT 10;""" + + +def __generic_autocomplete(event: Event): + def f(project_id, value, key=None, source=None): + with ch_client.ClickHouseClient() as cur: + query = __generic_query(event.ui_type, value_length=len(value)) + params = {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value)} + results = cur.execute(query=query, params=params) + return helper.list_to_camel_case(results) + + return f diff --git a/ee/api/chalicelib/core/sessions.py b/ee/api/chalicelib/core/sessions.py index 69755ff03..df72b2d86 100644 --- a/ee/api/chalicelib/core/sessions.py +++ b/ee/api/chalicelib/core/sessions.py @@ -1217,10 +1217,10 @@ def __get_event_type(event_type: Union[schemas.EventType, schemas.PerformanceEve defs = { schemas.EventType.click: "CLICK", schemas.EventType.input: "INPUT", - schemas.EventType.location: "PAGE", - schemas.PerformanceEventType.location_dom_complete: "PAGE", - schemas.PerformanceEventType.location_largest_contentful_paint_time: "PAGE", - schemas.PerformanceEventType.location_ttfb: "PAGE", + schemas.EventType.location: "LOCATION", + schemas.PerformanceEventType.location_dom_complete: "LOCATION", + schemas.PerformanceEventType.location_largest_contentful_paint_time: "LOCATION", + schemas.PerformanceEventType.location_ttfb: "LOCATION", schemas.EventType.custom: "CUSTOM", schemas.EventType.request: "REQUEST", schemas.EventType.request_details: "REQUEST", @@ -1385,7 +1385,7 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, referrer_constraint = f"""(SELECT DISTINCT session_id FROM {MAIN_EVENTS_TABLE} AS r WHERE {" AND ".join([f"r.{b}" for b in __events_where_basic])} - AND event_type='PAGE' + AND event_type='{__get_event_type(schemas.EventType.location)}' AND {referrer_constraint})""" # events_conditions_where.append(f"""main.session_id IN {referrer_constraint}""") # extra_constraints.append(f"""s.session_id IN {referrer_constraint}""") diff --git a/ee/api/clean.sh b/ee/api/clean.sh index e7785b22e..9055b36a3 100755 --- a/ee/api/clean.sh +++ b/ee/api/clean.sh @@ -3,6 +3,7 @@ rm -rf ./chalicelib/core/alerts.py rm -rf ./chalicelib/core/alerts_processor.py rm -rf ./chalicelib/core/announcements.py +rm -rf ./chalicelib/core/autocomplete.py rm -rf ./chalicelib/core/collaboration_slack.py rm -rf ./chalicelib/core/errors_favorite_viewed.py rm -rf ./chalicelib/core/events.py diff --git a/ee/api/env.default b/ee/api/env.default index 7687566d7..31cd33ff6 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -57,3 +57,4 @@ sourcemaps_bucket=sourcemaps sourcemaps_reader=http://127.0.0.1:9000/sourcemaps stage=default-ee version_number=1.0.0 +LEGACY_SEARCH=false \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql index af266d4c1..53ec133de 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS events_s ( session_id UInt64, project_id UInt16, - event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8, 'GRAPHQL'=9, 'STATEACTION'=10), + event_type Enum8('CLICK'=0, 'INPUT'=1, 'LOCATION'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8, 'GRAPHQL'=9, 'STATEACTION'=10), datetime DateTime, label Nullable(String), hesitation_time Nullable(UInt32), diff --git a/ee/utilities/.gitignore b/ee/utilities/.gitignore index 0eaed6d80..a11e6be97 100644 --- a/ee/utilities/.gitignore +++ b/ee/utilities/.gitignore @@ -14,3 +14,4 @@ servers/sourcemaps-server.js /utils/HeapSnapshot.js /utils/helper.js /utils/assistHelper.js +.local diff --git a/utilities/.dockerignore b/utilities/.dockerignore index b6aaccd33..1ba4ae6a6 100644 --- a/utilities/.dockerignore +++ b/utilities/.dockerignore @@ -4,3 +4,4 @@ **/build.sh **/build_*.sh **/*deploy.sh +.local \ No newline at end of file diff --git a/utilities/.gitignore b/utilities/.gitignore index fffbe974b..4486f0a7f 100644 --- a/utilities/.gitignore +++ b/utilities/.gitignore @@ -2,4 +2,5 @@ node_modules npm-debug.log .cache -test.html \ No newline at end of file +test.html +.local \ No newline at end of file