diff --git a/api/chalicelib/core/autocomplete.py b/api/chalicelib/core/autocomplete.py index 3ad845a14..d064856f1 100644 --- a/api/chalicelib/core/autocomplete.py +++ b/api/chalicelib/core/autocomplete.py @@ -1,5 +1,5 @@ import schemas -from chalicelib.core import countries +from chalicelib.core import countries, events, metadata from chalicelib.utils import helper from chalicelib.utils import pg_client from chalicelib.utils.event_filter_definition import Event @@ -36,6 +36,7 @@ def __get_autocomplete_table(value, project_id): WHERE project_id = %(project_id)s AND type= '{e}' AND value ILIKE %(svalue)s + ORDER BY value LIMIT 5)""") if len(value) > 2: sub_queries.append(f"""(SELECT type, value @@ -43,6 +44,7 @@ def __get_autocomplete_table(value, project_id): WHERE project_id = %(project_id)s AND type= '{e}' AND value ILIKE %(value)s + ORDER BY value LIMIT 5)""") with pg_client.PostgresClient() as cur: query = cur.mogrify(" UNION DISTINCT ".join(sub_queries) + ";", @@ -122,6 +124,8 @@ def __generic_autocomplete_metas(typename): if typename == schemas.FilterType.user_country: params["value"] = tuple(countries.get_country_code_autocomplete(text)) + if len(params["value"]) == 0: + return [] query = cur.mogrify(__generic_query(typename, value_length=len(text)), params) cur.execute(query) @@ -129,3 +133,194 @@ def __generic_autocomplete_metas(typename): return rows return f + + +def __pg_errors_query(source=None, value_length=None): + if value_length is None or value_length > 2: + return f"""((SELECT DISTINCT ON(lg.message) + lg.message AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {events.event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.message ILIKE %(svalue)s + AND lg.project_id = %(project_id)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5) + UNION DISTINCT + (SELECT DISTINCT ON(lg.name) + lg.name AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {events.event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.name ILIKE %(svalue)s + AND lg.project_id = %(project_id)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5) + UNION DISTINCT + (SELECT DISTINCT ON(lg.message) + lg.message AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {events.event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.message ILIKE %(value)s + AND lg.project_id = %(project_id)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5) + UNION DISTINCT + (SELECT DISTINCT ON(lg.name) + lg.name AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {events.event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.name ILIKE %(value)s + AND lg.project_id = %(project_id)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5));""" + return f"""((SELECT DISTINCT ON(lg.message) + lg.message AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {events.event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.message ILIKE %(svalue)s + AND lg.project_id = %(project_id)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5) + UNION DISTINCT + (SELECT DISTINCT ON(lg.name) + lg.name AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {events.event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.name ILIKE %(svalue)s + AND lg.project_id = %(project_id)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5));""" + + +def __search_pg_errors(project_id, value, key=None, source=None): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(__pg_errors_query(source, + value_length=len(value)), + {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value), + "source": source})) + results = helper.list_to_camel_case(cur.fetchall()) + return results + + +def __search_pg_errors_ios(project_id, value, key=None, source=None): + if len(value) > 2: + query = f"""(SELECT DISTINCT ON(lg.reason) + lg.reason AS value, + '{events.event_type.ERROR_IOS.ui_type}' AS type + FROM {events.event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.project_id = %(project_id)s + AND lg.reason ILIKE %(svalue)s + LIMIT 5) + UNION ALL + (SELECT DISTINCT ON(lg.name) + lg.name AS value, + '{events.event_type.ERROR_IOS.ui_type}' AS type + FROM {events.event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.project_id = %(project_id)s + AND lg.name ILIKE %(svalue)s + LIMIT 5) + UNION ALL + (SELECT DISTINCT ON(lg.reason) + lg.reason AS value, + '{events.event_type.ERROR_IOS.ui_type}' AS type + FROM {events.event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.project_id = %(project_id)s + AND lg.reason ILIKE %(value)s + LIMIT 5) + UNION ALL + (SELECT DISTINCT ON(lg.name) + lg.name AS value, + '{events.event_type.ERROR_IOS.ui_type}' AS type + FROM {events.event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.project_id = %(project_id)s + AND lg.name ILIKE %(value)s + LIMIT 5);""" + else: + query = f"""(SELECT DISTINCT ON(lg.reason) + lg.reason AS value, + '{events.event_type.ERROR_IOS.ui_type}' AS type + FROM {events.event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.project_id = %(project_id)s + AND lg.reason ILIKE %(svalue)s + LIMIT 5) + UNION ALL + (SELECT DISTINCT ON(lg.name) + lg.name AS value, + '{events.event_type.ERROR_IOS.ui_type}' AS type + FROM {events.event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.project_id = %(project_id)s + AND lg.name ILIKE %(svalue)s + LIMIT 5);""" + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify(query, {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value)})) + results = helper.list_to_camel_case(cur.fetchall()) + return results + + +def __search_pg_metadata(project_id, value, key=None, source=None): + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + if len(meta_keys) == 0 or key is not None and key not in meta_keys.keys(): + return [] + sub_from = [] + if key is not None: + meta_keys = {key: meta_keys[key]} + + for k in meta_keys.keys(): + colname = metadata.index_to_colname(meta_keys[k]) + if len(value) > 2: + sub_from.append(f"""((SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key + FROM public.sessions + WHERE project_id = %(project_id)s + AND {colname} ILIKE %(svalue)s LIMIT 5) + UNION + (SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key + FROM public.sessions + WHERE project_id = %(project_id)s + AND {colname} ILIKE %(value)s LIMIT 5)) + """) + else: + sub_from.append(f"""(SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key + FROM public.sessions + WHERE project_id = %(project_id)s + AND {colname} ILIKE %(svalue)s LIMIT 5)""") + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify(f"""\ + SELECT key, value, 'METADATA' AS TYPE + FROM({" UNION ALL ".join(sub_from)}) AS all_metas + LIMIT 5;""", {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value)})) + results = helper.list_to_camel_case(cur.fetchall()) + return results \ No newline at end of file diff --git a/api/chalicelib/core/countries.py b/api/chalicelib/core/countries.py index 2e9212b9f..246e3f8a9 100644 --- a/api/chalicelib/core/countries.py +++ b/api/chalicelib/core/countries.py @@ -257,6 +257,7 @@ COUNTRIES = { "UA": "Ukraine", "UG": "Uganda", "UM": "United States Minor Outlying Islands", + "UN": "United Nations", "US": "United States", "UY": "Uruguay", "UZ": "Uzbekistan", diff --git a/api/chalicelib/core/events.py b/api/chalicelib/core/events.py index e2c979799..4887c94b1 100644 --- a/api/chalicelib/core/events.py +++ b/api/chalicelib/core/events.py @@ -40,7 +40,7 @@ def __get_grouped_clickrage(rows, session_id, project_id): for c in click_rage_issues: merge_count = c.get("payload") if merge_count is not None: - merge_count = merge_count.get("count", 3) + merge_count = merge_count.get("Count", 3) else: merge_count = 3 for i in range(len(rows)): @@ -96,202 +96,6 @@ def get_by_sessionId2_pg(session_id, project_id, group_clickrage=False): return rows -def __pg_errors_query(source=None, value_length=None): - if value_length is None or value_length > 2: - return f"""((SELECT DISTINCT ON(lg.message) - lg.message AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.message ILIKE %(svalue)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5) - UNION DISTINCT - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.name ILIKE %(svalue)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5) - UNION DISTINCT - (SELECT DISTINCT ON(lg.message) - lg.message AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.message ILIKE %(value)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5) - UNION DISTINCT - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.name ILIKE %(value)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5));""" - return f"""((SELECT DISTINCT ON(lg.message) - lg.message AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.message ILIKE %(svalue)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5) - UNION DISTINCT - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.name ILIKE %(svalue)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5));""" - - -def __search_pg_errors(project_id, value, key=None, source=None): - now = TimeUTC.now() - - with pg_client.PostgresClient() as cur: - cur.execute( - cur.mogrify(__pg_errors_query(source, - value_length=len(value)), - {"project_id": project_id, "value": helper.string_to_sql_like(value), - "svalue": helper.string_to_sql_like("^" + value), - "source": source})) - results = helper.list_to_camel_case(cur.fetchall()) - print(f"{TimeUTC.now() - now} : errors") - return results - - -def __search_pg_errors_ios(project_id, value, key=None, source=None): - now = TimeUTC.now() - if len(value) > 2: - query = f"""(SELECT DISTINCT ON(lg.reason) - lg.reason AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.reason ILIKE %(svalue)s - LIMIT 5) - UNION ALL - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.name ILIKE %(svalue)s - LIMIT 5) - UNION ALL - (SELECT DISTINCT ON(lg.reason) - lg.reason AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.reason ILIKE %(value)s - LIMIT 5) - UNION ALL - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.name ILIKE %(value)s - LIMIT 5);""" - else: - query = f"""(SELECT DISTINCT ON(lg.reason) - lg.reason AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.reason ILIKE %(svalue)s - LIMIT 5) - UNION ALL - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.name ILIKE %(svalue)s - LIMIT 5);""" - with pg_client.PostgresClient() as cur: - cur.execute(cur.mogrify(query, {"project_id": project_id, "value": helper.string_to_sql_like(value), - "svalue": helper.string_to_sql_like("^" + value)})) - results = helper.list_to_camel_case(cur.fetchall()) - print(f"{TimeUTC.now() - now} : errors") - return results - - -def __search_pg_metadata(project_id, value, key=None, source=None): - meta_keys = metadata.get(project_id=project_id) - meta_keys = {m["key"]: m["index"] for m in meta_keys} - if len(meta_keys) == 0 or key is not None and key not in meta_keys.keys(): - return [] - sub_from = [] - if key is not None: - meta_keys = {key: meta_keys[key]} - - for k in meta_keys.keys(): - colname = metadata.index_to_colname(meta_keys[k]) - if len(value) > 2: - sub_from.append(f"""((SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key - FROM public.sessions - WHERE project_id = %(project_id)s - AND {colname} ILIKE %(svalue)s LIMIT 5) - UNION - (SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key - FROM public.sessions - WHERE project_id = %(project_id)s - AND {colname} ILIKE %(value)s LIMIT 5)) - """) - else: - sub_from.append(f"""(SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key - FROM public.sessions - WHERE project_id = %(project_id)s - AND {colname} ILIKE %(svalue)s LIMIT 5)""") - with pg_client.PostgresClient() as cur: - cur.execute(cur.mogrify(f"""\ - SELECT key, value, 'METADATA' AS TYPE - FROM({" UNION ALL ".join(sub_from)}) AS all_metas - LIMIT 5;""", {"project_id": project_id, "value": helper.string_to_sql_like(value), - "svalue": helper.string_to_sql_like("^" + value)})) - results = helper.list_to_camel_case(cur.fetchall()) - return results - - class event_type: CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label") INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label") @@ -332,9 +136,9 @@ SUPPORTED_TYPES = { event_type.STATEACTION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.STATEACTION), query=autocomplete.__generic_query( typename=event_type.STATEACTION.ui_type)), - event_type.ERROR.ui_type: SupportedFilter(get=__search_pg_errors, + event_type.ERROR.ui_type: SupportedFilter(get=autocomplete.__search_pg_errors, query=None), - event_type.METADATA.ui_type: SupportedFilter(get=__search_pg_metadata, + event_type.METADATA.ui_type: SupportedFilter(get=autocomplete.__search_pg_metadata, query=None), # IOS event_type.CLICK_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.CLICK_IOS), @@ -352,7 +156,7 @@ SUPPORTED_TYPES = { event_type.REQUEST_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.REQUEST_IOS), query=autocomplete.__generic_query( typename=event_type.REQUEST_IOS.ui_type)), - event_type.ERROR_IOS.ui_type: SupportedFilter(get=__search_pg_errors_ios, + event_type.ERROR_IOS.ui_type: SupportedFilter(get=autocomplete.__search_pg_errors_ios, query=None), } diff --git a/ee/api/.gitignore b/ee/api/.gitignore index ee23eb7ff..7fd0b58d0 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -211,7 +211,7 @@ Pipfile /chalicelib/core/metadata.py /chalicelib/core/mobile.py /chalicelib/core/sessions_assignments.py -/chalicelib/core/sessions_metas.py +#exp /chalicelib/core/sessions_metas.py /chalicelib/core/sessions_mobs.py #exp /chalicelib/core/significance.py /chalicelib/core/slack.py diff --git a/ee/api/chalicelib/core/autocomplete_exp.py b/ee/api/chalicelib/core/autocomplete_exp.py index 2abe97e8e..9ae59b7a2 100644 --- a/ee/api/chalicelib/core/autocomplete_exp.py +++ b/ee/api/chalicelib/core/autocomplete_exp.py @@ -1,6 +1,7 @@ import schemas +from chalicelib.core import countries, events, metadata from chalicelib.utils import ch_client -from chalicelib.utils import helper +from chalicelib.utils import helper, exp_ch_helper from chalicelib.utils.event_filter_definition import Event TABLE = "experimental.autocomplete" @@ -19,13 +20,23 @@ def __get_autocomplete_table(value, project_id): schemas.EventType.input] autocomplete_events.sort() sub_queries = [] + c_list = [] for e in autocomplete_events: + if e == schemas.FilterType.user_country: + c_list = countries.get_country_code_autocomplete(value) + if len(c_list) > 0: + sub_queries.append(f"""(SELECT DISTINCT ON(value) type, value + FROM {TABLE} + WHERE project_id = %(project_id)s + AND type= '{e}' + AND value IN %(c_list)s)""") + continue sub_queries.append(f"""(SELECT type, value FROM {TABLE} WHERE project_id = %(project_id)s AND type= '{e}' AND value ILIKE %(svalue)s - ORDER BY value + ORDER BY value LIMIT 5)""") if len(value) > 2: sub_queries.append(f"""(SELECT type, value @@ -33,12 +44,14 @@ def __get_autocomplete_table(value, project_id): WHERE project_id = %(project_id)s AND type= '{e}' AND value ILIKE %(value)s - ORDER BY value + ORDER BY value LIMIT 5)""") with ch_client.ClickHouseClient() as cur: query = " UNION DISTINCT ".join(sub_queries) + ";" - params = {"project_id": project_id, "value": helper.string_to_sql_like(value), - "svalue": helper.string_to_sql_like("^" + value)} + params = {"project_id": project_id, + "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value), + "c_list": tuple(c_list)} results = [] try: results = cur.execute(query=query, params=params) @@ -55,6 +68,15 @@ def __get_autocomplete_table(value, project_id): def __generic_query(typename, value_length=None): + if typename == schemas.FilterType.user_country: + return f"""SELECT DISTINCT value, type + FROM {TABLE} + WHERE + project_id = %(project_id)s + AND type='{typename}' + AND value IN %(c_list)s + ORDER BY value""" + if value_length is None or value_length > 2: return f"""(SELECT DISTINCT value, type FROM {TABLE} @@ -98,10 +120,138 @@ def __generic_autocomplete(event: Event): def __generic_autocomplete_metas(typename): def f(project_id, text): with ch_client.ClickHouseClient() as cur: + c_list = [] + if typename == schemas.FilterType.user_country: + c_list = countries.get_country_code_autocomplete(text) + if len(c_list) == 0: + return [] + query = __generic_query(typename, value_length=len(text)) params = {"project_id": project_id, "value": helper.string_to_sql_like(text), - "svalue": helper.string_to_sql_like("^" + text)} + "svalue": helper.string_to_sql_like("^" + text), "rvalue": text, + "c_list": tuple(c_list)} results = cur.execute(query=query, params=params) return results return f + + +def __pg_errors_query(source=None, value_length=None): + MAIN_TABLE = exp_ch_helper.get_main_js_errors_sessions_table() + if value_length is None or value_length > 2: + return f"""((SELECT DISTINCT ON(message) + message AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {MAIN_TABLE} + WHERE + project_id = %(project_id)s + AND message ILIKE %(svalue)s + AND event_type = 'ERROR' + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5) + UNION DISTINCT + (SELECT DISTINCT ON(name) + name AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {MAIN_TABLE} + WHERE + project_id = %(project_id)s + AND name ILIKE %(svalue)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5) + UNION DISTINCT + (SELECT DISTINCT ON(message) + message AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {MAIN_TABLE} + WHERE + project_id = %(project_id)s + AND message ILIKE %(value)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5) + UNION DISTINCT + (SELECT DISTINCT ON(name) + name AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {MAIN_TABLE} + WHERE + project_id = %(project_id)s + AND name ILIKE %(value)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5));""" + return f"""((SELECT DISTINCT ON(message) + message AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {MAIN_TABLE} + WHERE + project_id = %(project_id)s + AND message ILIKE %(svalue)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5) + UNION DISTINCT + (SELECT DISTINCT ON(name) + name AS value, + source, + '{events.event_type.ERROR.ui_type}' AS type + FROM {MAIN_TABLE} + WHERE + project_id = %(project_id)s + AND name ILIKE %(svalue)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5));""" + + +def __search_pg_errors(project_id, value, key=None, source=None): + with ch_client.ClickHouseClient() as cur: + query = cur.format(__pg_errors_query(source, value_length=len(value)), + {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value), + "source": source}) + results = cur.execute(query) + return helper.list_to_camel_case(results) + + +def __search_pg_errors_ios(project_id, value, key=None, source=None): + # TODO: define this when ios events are supported in CH + return [] + + +def __search_pg_metadata(project_id, value, key=None, source=None): + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + if len(meta_keys) == 0 or key is not None and key not in meta_keys.keys(): + return [] + sub_from = [] + if key is not None: + meta_keys = {key: meta_keys[key]} + + for k in meta_keys.keys(): + colname = metadata.index_to_colname(meta_keys[k]) + if len(value) > 2: + sub_from.append(f"""((SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key + FROM {exp_ch_helper.get_main_sessions_table(0)} + WHERE project_id = %(project_id)s + AND {colname} ILIKE %(svalue)s LIMIT 5) + UNION DISTINCT + (SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key + FROM {exp_ch_helper.get_main_sessions_table(0)} + WHERE project_id = %(project_id)s + AND {colname} ILIKE %(value)s LIMIT 5)) + """) + else: + sub_from.append(f"""(SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key + FROM {exp_ch_helper.get_main_sessions_table(0)} + WHERE project_id = %(project_id)s + AND {colname} ILIKE %(svalue)s LIMIT 5)""") + with ch_client.ClickHouseClient() as cur: + query = cur.format(f"""SELECT key, value, 'METADATA' AS TYPE + FROM({" UNION ALL ".join(sub_from)}) AS all_metas + LIMIT 5;""", {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value)}) + results = cur.execute(query) + return helper.list_to_camel_case(results) diff --git a/ee/api/chalicelib/core/events.py b/ee/api/chalicelib/core/events.py index 7ad9e830d..f0831864e 100644 --- a/ee/api/chalicelib/core/events.py +++ b/ee/api/chalicelib/core/events.py @@ -45,7 +45,7 @@ def __get_grouped_clickrage(rows, session_id, project_id): for c in click_rage_issues: merge_count = c.get("payload") if merge_count is not None: - merge_count = merge_count.get("count", 3) + merge_count = merge_count.get("Count", 3) else: merge_count = 3 for i in range(len(rows)): @@ -101,202 +101,6 @@ def get_by_sessionId2_pg(session_id, project_id, group_clickrage=False): return rows -def __pg_errors_query(source=None, value_length=None): - if value_length is None or value_length > 2: - return f"""((SELECT DISTINCT ON(lg.message) - lg.message AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.message ILIKE %(svalue)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5) - UNION DISTINCT - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.name ILIKE %(svalue)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5) - UNION DISTINCT - (SELECT DISTINCT ON(lg.message) - lg.message AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.message ILIKE %(value)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5) - UNION DISTINCT - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.name ILIKE %(value)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5));""" - return f"""((SELECT DISTINCT ON(lg.message) - lg.message AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.message ILIKE %(svalue)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5) - UNION DISTINCT - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - source, - '{event_type.ERROR.ui_type}' AS type - FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.name ILIKE %(svalue)s - AND lg.project_id = %(project_id)s - {"AND source = %(source)s" if source is not None else ""} - LIMIT 5));""" - - -def __search_pg_errors(project_id, value, key=None, source=None): - now = TimeUTC.now() - - with pg_client.PostgresClient() as cur: - cur.execute( - cur.mogrify(__pg_errors_query(source, - value_length=len(value)), - {"project_id": project_id, "value": helper.string_to_sql_like(value), - "svalue": helper.string_to_sql_like("^" + value), - "source": source})) - results = helper.list_to_camel_case(cur.fetchall()) - print(f"{TimeUTC.now() - now} : errors") - return results - - -def __search_pg_errors_ios(project_id, value, key=None, source=None): - now = TimeUTC.now() - if len(value) > 2: - query = f"""(SELECT DISTINCT ON(lg.reason) - lg.reason AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.reason ILIKE %(svalue)s - LIMIT 5) - UNION ALL - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.name ILIKE %(svalue)s - LIMIT 5) - UNION ALL - (SELECT DISTINCT ON(lg.reason) - lg.reason AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.reason ILIKE %(value)s - LIMIT 5) - UNION ALL - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.name ILIKE %(value)s - LIMIT 5);""" - else: - query = f"""(SELECT DISTINCT ON(lg.reason) - lg.reason AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.reason ILIKE %(svalue)s - LIMIT 5) - UNION ALL - (SELECT DISTINCT ON(lg.name) - lg.name AS value, - '{event_type.ERROR_IOS.ui_type}' AS type - FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) - WHERE - s.project_id = %(project_id)s - AND lg.project_id = %(project_id)s - AND lg.name ILIKE %(svalue)s - LIMIT 5);""" - with pg_client.PostgresClient() as cur: - cur.execute(cur.mogrify(query, {"project_id": project_id, "value": helper.string_to_sql_like(value), - "svalue": helper.string_to_sql_like("^" + value)})) - results = helper.list_to_camel_case(cur.fetchall()) - print(f"{TimeUTC.now() - now} : errors") - return results - - -def __search_pg_metadata(project_id, value, key=None, source=None): - meta_keys = metadata.get(project_id=project_id) - meta_keys = {m["key"]: m["index"] for m in meta_keys} - if len(meta_keys) == 0 or key is not None and key not in meta_keys.keys(): - return [] - sub_from = [] - if key is not None: - meta_keys = {key: meta_keys[key]} - - for k in meta_keys.keys(): - colname = metadata.index_to_colname(meta_keys[k]) - if len(value) > 2: - sub_from.append(f"""((SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key - FROM public.sessions - WHERE project_id = %(project_id)s - AND {colname} ILIKE %(svalue)s LIMIT 5) - UNION - (SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key - FROM public.sessions - WHERE project_id = %(project_id)s - AND {colname} ILIKE %(value)s LIMIT 5)) - """) - else: - sub_from.append(f"""(SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key - FROM public.sessions - WHERE project_id = %(project_id)s - AND {colname} ILIKE %(svalue)s LIMIT 5)""") - with pg_client.PostgresClient() as cur: - cur.execute(cur.mogrify(f"""\ - SELECT key, value, 'METADATA' AS TYPE - FROM({" UNION ALL ".join(sub_from)}) AS all_metas - LIMIT 5;""", {"project_id": project_id, "value": helper.string_to_sql_like(value), - "svalue": helper.string_to_sql_like("^" + value)})) - results = helper.list_to_camel_case(cur.fetchall()) - return results - - class event_type: CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label") INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label") @@ -337,9 +141,9 @@ SUPPORTED_TYPES = { event_type.STATEACTION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.STATEACTION), query=autocomplete.__generic_query( typename=event_type.STATEACTION.ui_type)), - event_type.ERROR.ui_type: SupportedFilter(get=__search_pg_errors, + event_type.ERROR.ui_type: SupportedFilter(get=autocomplete.__search_pg_errors, query=None), - event_type.METADATA.ui_type: SupportedFilter(get=__search_pg_metadata, + event_type.METADATA.ui_type: SupportedFilter(get=autocomplete.__search_pg_metadata, query=None), # IOS event_type.CLICK_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.CLICK_IOS), @@ -357,7 +161,7 @@ SUPPORTED_TYPES = { event_type.REQUEST_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.REQUEST_IOS), query=autocomplete.__generic_query( typename=event_type.REQUEST_IOS.ui_type)), - event_type.ERROR_IOS.ui_type: SupportedFilter(get=__search_pg_errors_ios, + event_type.ERROR_IOS.ui_type: SupportedFilter(get=autocomplete.__search_pg_errors_ios, query=None), } diff --git a/ee/api/chalicelib/core/sessions_metas.py b/ee/api/chalicelib/core/sessions_metas.py new file mode 100644 index 000000000..65d3eb6db --- /dev/null +++ b/ee/api/chalicelib/core/sessions_metas.py @@ -0,0 +1,76 @@ +import schemas +from chalicelib.utils.event_filter_definition import SupportedFilter +from decouple import config + +if config("EXP_AUTOCOMPLETE", cast=bool, default=False): + from . import autocomplete_exp as autocomplete +else: + from . import autocomplete as autocomplete + +SUPPORTED_TYPES = { + schemas.FilterType.user_os: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_os), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_os)), + schemas.FilterType.user_browser: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_browser), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_browser)), + schemas.FilterType.user_device: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_device), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_device)), + schemas.FilterType.user_country: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_country), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_country)), + schemas.FilterType.user_id: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_id), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_id)), + schemas.FilterType.user_anonymous_id: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_anonymous_id), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_anonymous_id)), + schemas.FilterType.rev_id: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.rev_id), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.rev_id)), + schemas.FilterType.referrer: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.referrer), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.referrer)), + schemas.FilterType.utm_campaign: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_campaign), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_campaign)), + schemas.FilterType.utm_medium: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_medium), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_medium)), + schemas.FilterType.utm_source: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_source), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_source)), + # IOS + schemas.FilterType.user_os_ios: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_os_ios), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_os_ios)), + schemas.FilterType.user_device_ios: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas( + typename=schemas.FilterType.user_device_ios), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_device_ios)), + schemas.FilterType.user_country_ios: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_country_ios), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_country_ios)), + schemas.FilterType.user_id_ios: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_id_ios), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_id_ios)), + schemas.FilterType.user_anonymous_id_ios: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_anonymous_id_ios), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_anonymous_id_ios)), + schemas.FilterType.rev_id_ios: SupportedFilter( + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.rev_id_ios), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.rev_id_ios)), + +} + + +def search(text: str, meta_type: schemas.FilterType, project_id: int): + rows = [] + if meta_type not in list(SUPPORTED_TYPES.keys()): + return {"errors": ["unsupported type"]} + rows += SUPPORTED_TYPES[meta_type].get(project_id=project_id, text=text) + # for IOS events autocomplete + # if meta_type + "_IOS" in list(SUPPORTED_TYPES.keys()): + # rows += SUPPORTED_TYPES[meta_type + "_IOS"].get(project_id=project_id, text=text) + return {"data": rows} diff --git a/ee/api/chalicelib/utils/exp_ch_helper.py b/ee/api/chalicelib/utils/exp_ch_helper.py index 02de9addd..91b3a9c1d 100644 --- a/ee/api/chalicelib/utils/exp_ch_helper.py +++ b/ee/api/chalicelib/utils/exp_ch_helper.py @@ -43,6 +43,8 @@ def get_user_viewed_errors_table(timestamp=0): def get_main_js_errors_sessions_table(timestamp=0): - return "experimental.js_errors_sessions_mv" # \ + return get_main_events_table(timestamp=timestamp) + # enable this when js_errors_sessions_mv is fixed + # return "experimental.js_errors_sessions_mv" # \ # if config("EXP_7D_MV", cast=bool, default=True) \ - # and timestamp >= TimeUTC.now(delta_days=-7) else "experimental.events" + # and timestamp >= TimeUTC.now(delta_days=-7) else "experimental.events" \ No newline at end of file diff --git a/ee/api/clean.sh b/ee/api/clean.sh index 314321b83..602a47222 100755 --- a/ee/api/clean.sh +++ b/ee/api/clean.sh @@ -33,7 +33,7 @@ rm -rf ./chalicelib/core/log_tool_sumologic.py rm -rf ./chalicelib/core/metadata.py rm -rf ./chalicelib/core/mobile.py rm -rf ./chalicelib/core/sessions_assignments.py -rm -rf ./chalicelib/core/sessions_metas.py +#exp rm -rf ./chalicelib/core/sessions_metas.py rm -rf ./chalicelib/core/sessions_mobs.py #exp rm -rf ./chalicelib/core/significance.py rm -rf ./chalicelib/core/slack.py