diff --git a/api/chalicelib/core/autocomplete.py b/api/chalicelib/core/autocomplete.py index 595342a9b..67f178b6d 100644 --- a/api/chalicelib/core/autocomplete.py +++ b/api/chalicelib/core/autocomplete.py @@ -56,6 +56,7 @@ def __generic_query(typename, value_length=None): project_id = %(project_id)s AND type='{typename}' AND value ILIKE %(svalue)s + ORDER BY value LIMIT 5) UNION DISTINCT (SELECT DISTINCT value, type @@ -64,6 +65,7 @@ def __generic_query(typename, value_length=None): project_id = %(project_id)s AND type='{typename}' AND value ILIKE %(value)s + ORDER BY value LIMIT 5);""" return f"""SELECT DISTINCT value, type FROM public.autocomplete @@ -71,6 +73,7 @@ def __generic_query(typename, value_length=None): project_id = %(project_id)s AND type='{typename}' AND value ILIKE %(svalue)s + ORDER BY value LIMIT 10;""" @@ -84,3 +87,16 @@ def __generic_autocomplete(event: Event): return helper.list_to_camel_case(cur.fetchall()) return f + + +def __generic_autocomplete_metas(typename): + def f(project_id, text): + with pg_client.PostgresClient() as cur: + query = cur.mogrify(__generic_query(typename, value_length=len(text)), + {"project_id": project_id, "value": helper.string_to_sql_like(text), + "svalue": helper.string_to_sql_like("^" + text)}) + cur.execute(query) + rows = cur.fetchall() + return rows + + return f diff --git a/api/chalicelib/core/sessions_metas.py b/api/chalicelib/core/sessions_metas.py index b956e0f62..f7e98eb69 100644 --- a/api/chalicelib/core/sessions_metas.py +++ b/api/chalicelib/core/sessions_metas.py @@ -1,188 +1,66 @@ import schemas -from chalicelib.utils import pg_client, helper +from chalicelib.core import autocomplete from chalicelib.utils.event_filter_definition import SupportedFilter - -def get_key_values(project_id): - with pg_client.PostgresClient() as cur: - cur.execute( - cur.mogrify( - f"""\ - SELECT ARRAY_AGG(DISTINCT s.user_os - ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='web') AS {schemas.FilterType.user_os}, - ARRAY_AGG(DISTINCT s.user_browser - ORDER BY s.user_browser) - FILTER ( WHERE s.user_browser IS NOT NULL AND s.platform='web') AS {schemas.FilterType.user_browser}, - ARRAY_AGG(DISTINCT s.user_device - ORDER BY s.user_device) - FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='web') AS {schemas.FilterType.user_device}, - ARRAY_AGG(DISTINCT s.user_country - ORDER BY s.user_country) - FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='web')::text[] AS {schemas.FilterType.user_country}, - ARRAY_AGG(DISTINCT s.user_id - ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='web') AS {schemas.FilterType.user_id}, - ARRAY_AGG(DISTINCT s.user_anonymous_id - ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='web') AS {schemas.FilterType.user_anonymous_id}, - ARRAY_AGG(DISTINCT s.rev_id - ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='web') AS {schemas.FilterType.rev_id}, - ARRAY_AGG(DISTINCT p.referrer - ORDER BY p.referrer) - FILTER ( WHERE p.referrer != '' ) AS {schemas.FilterType.referrer}, - - ARRAY_AGG(DISTINCT s.utm_source - ORDER BY s.utm_source) FILTER ( WHERE s.utm_source IS NOT NULL AND s.utm_source != 'none' AND s.utm_source != '') AS {schemas.FilterType.utm_source}, - ARRAY_AGG(DISTINCT s.utm_medium - ORDER BY s.utm_medium) FILTER ( WHERE s.utm_medium IS NOT NULL AND s.utm_medium != 'none' AND s.utm_medium != '') AS {schemas.FilterType.utm_medium}, - ARRAY_AGG(DISTINCT s.utm_campaign - ORDER BY s.utm_campaign) FILTER ( WHERE s.utm_campaign IS NOT NULL AND s.utm_campaign != 'none' AND s.utm_campaign != '') AS {schemas.FilterType.utm_campaign}, - - ARRAY_AGG(DISTINCT s.user_os - ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='ios' ) AS {schemas.FilterType.user_os_ios}, - ARRAY_AGG(DISTINCT s.user_device - ORDER BY s.user_device) - FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='ios') AS {schemas.FilterType.user_device_ios}, - ARRAY_AGG(DISTINCT s.user_country - ORDER BY s.user_country) - FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='ios')::text[] AS {schemas.FilterType.user_country_ios}, - ARRAY_AGG(DISTINCT s.user_id - ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='ios') AS {schemas.FilterType.user_id_ios}, - ARRAY_AGG(DISTINCT s.user_anonymous_id - ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='ios') AS {schemas.FilterType.user_anonymous_id_ios}, - ARRAY_AGG(DISTINCT s.rev_id - ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='ios') AS {schemas.FilterType.rev_id_ios} - FROM public.sessions AS s - LEFT JOIN events.pages AS p USING (session_id) - WHERE s.project_id = %(site_id)s;""", - {"site_id": project_id} - ) - ) - - row = cur.fetchone() - for k in row.keys(): - if row[k] is None: - row[k] = [] - elif len(row[k]) > 500: - row[k] = row[k][:500] - return helper.dict_to_CAPITAL_keys(row) - - -def get_top_key_values(project_id): - with pg_client.PostgresClient() as cur: - cur.execute( - cur.mogrify( - f"""\ - SELECT {",".join([f"ARRAY((SELECT value FROM public.autocomplete WHERE project_id = %(site_id)s AND type='{k}' GROUP BY value ORDER BY COUNT(*) DESC LIMIT %(limit)s)) AS {k}" for k in SUPPORTED_TYPES.keys()])};""", - {"site_id": project_id, "limit": 5} - ) - ) - - row = cur.fetchone() - return helper.dict_to_CAPITAL_keys(row) - - -def __generic_query(typename, value_length=None): - if value_length is None or value_length > 2: - return f""" (SELECT DISTINCT value, type - FROM public.autocomplete - WHERE - project_id = %(project_id)s - AND type ='{typename}' - AND value ILIKE %(svalue)s - ORDER BY value - LIMIT 5) - UNION - (SELECT DISTINCT value, type - FROM public.autocomplete - WHERE - project_id = %(project_id)s - AND type ='{typename}' - AND value ILIKE %(value)s - ORDER BY value - LIMIT 5);""" - return f""" SELECT DISTINCT value, type - FROM public.autocomplete - WHERE - project_id = %(project_id)s - AND type ='{typename}' - AND value ILIKE %(svalue)s - ORDER BY value - LIMIT 10;""" - - -def __generic_autocomplete(typename): - def f(project_id, text): - with pg_client.PostgresClient() as cur: - query = cur.mogrify(__generic_query(typename, - value_length=len(text)), - {"project_id": project_id, "value": helper.string_to_sql_like(text), - "svalue": helper.string_to_sql_like("^" + text)}) - - cur.execute(query) - rows = cur.fetchall() - return rows - - return f - - SUPPORTED_TYPES = { schemas.FilterType.user_os: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.user_os), - query=__generic_query(typename=schemas.FilterType.user_os)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_os), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_os)), schemas.FilterType.user_browser: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.user_browser), - query=__generic_query(typename=schemas.FilterType.user_browser)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_browser), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_browser)), schemas.FilterType.user_device: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.user_device), - query=__generic_query(typename=schemas.FilterType.user_device)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_device), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_device)), schemas.FilterType.user_country: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.user_country), - query=__generic_query(typename=schemas.FilterType.user_country)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_country), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_country)), schemas.FilterType.user_id: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.user_id), - query=__generic_query(typename=schemas.FilterType.user_id)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_id), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_id)), schemas.FilterType.user_anonymous_id: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.user_anonymous_id), - query=__generic_query(typename=schemas.FilterType.user_anonymous_id)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_anonymous_id), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_anonymous_id)), schemas.FilterType.rev_id: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.rev_id), - query=__generic_query(typename=schemas.FilterType.rev_id)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.rev_id), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.rev_id)), schemas.FilterType.referrer: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.referrer), - query=__generic_query(typename=schemas.FilterType.referrer)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.referrer), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.referrer)), schemas.FilterType.utm_campaign: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.utm_campaign), - query=__generic_query(typename=schemas.FilterType.utm_campaign)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_campaign), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_campaign)), schemas.FilterType.utm_medium: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.utm_medium), - query=__generic_query(typename=schemas.FilterType.utm_medium)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_medium), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_medium)), schemas.FilterType.utm_source: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.utm_source), - query=__generic_query(typename=schemas.FilterType.utm_source)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_source), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.utm_source)), # IOS schemas.FilterType.user_os_ios: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.user_os_ios), - query=__generic_query(typename=schemas.FilterType.user_os_ios)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_os_ios), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_os_ios)), schemas.FilterType.user_device_ios: SupportedFilter( - get=__generic_autocomplete( + get=autocomplete.__generic_autocomplete_metas( typename=schemas.FilterType.user_device_ios), - query=__generic_query(typename=schemas.FilterType.user_device_ios)), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_device_ios)), schemas.FilterType.user_country_ios: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.user_country_ios), - query=__generic_query(typename=schemas.FilterType.user_country_ios)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_country_ios), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_country_ios)), schemas.FilterType.user_id_ios: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.user_id_ios), - query=__generic_query(typename=schemas.FilterType.user_id_ios)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_id_ios), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_id_ios)), schemas.FilterType.user_anonymous_id_ios: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.user_anonymous_id_ios), - query=__generic_query(typename=schemas.FilterType.user_anonymous_id_ios)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_anonymous_id_ios), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.user_anonymous_id_ios)), schemas.FilterType.rev_id_ios: SupportedFilter( - get=__generic_autocomplete(typename=schemas.FilterType.rev_id_ios), - query=__generic_query(typename=schemas.FilterType.rev_id_ios)), + get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.rev_id_ios), + query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.rev_id_ios)), } -def search(text, meta_type, project_id): +def search(text: str, meta_type: schemas.FilterType, project_id: int): rows = [] if meta_type not in list(SUPPORTED_TYPES.keys()): return {"errors": ["unsupported type"]} diff --git a/api/routers/core.py b/api/routers/core.py index 420d44e05..9ff641556 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -163,21 +163,11 @@ def events_search(projectId: int, q: str, return result -@app.get('/{projectId}/sessions/filters', tags=["sessions"]) -def session_filter_values(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {'data': sessions_metas.get_key_values(projectId)} - - -@app.get('/{projectId}/sessions/filters/top', tags=["sessions"]) -def session_top_filter_values(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {'data': sessions_metas.get_top_key_values(projectId)} - - @app.get('/{projectId}/integrations', tags=["integrations"]) def get_integrations_status(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): data = integrations_global.get_global_integrations_status(tenant_id=context.tenant_id, - user_id=context.user_id, - project_id=projectId) + user_id=context.user_id, + project_id=projectId) return {"data": data} diff --git a/ee/api/chalicelib/core/autocomplete_ee.py b/ee/api/chalicelib/core/autocomplete_ee.py index e96cd819b..bf394c199 100644 --- a/ee/api/chalicelib/core/autocomplete_ee.py +++ b/ee/api/chalicelib/core/autocomplete_ee.py @@ -23,6 +23,7 @@ def __get_autocomplete_table(value, project_id): WHERE project_id = %(project_id)s AND type= '{e}' AND value ILIKE %(svalue)s + ORDER BY value LIMIT 5)""") if len(value) > 2: sub_queries.append(f"""(SELECT type, value @@ -30,6 +31,7 @@ def __get_autocomplete_table(value, project_id): WHERE project_id = %(project_id)s AND type= '{e}' AND value ILIKE %(value)s + ORDER BY value LIMIT 5)""") with ch_client.ClickHouseClient() as cur: query = " UNION DISTINCT ".join(sub_queries) + ";" @@ -58,6 +60,7 @@ def __generic_query(typename, value_length=None): project_id = %(project_id)s AND type='{typename}' AND value ILIKE %(svalue)s + ORDER BY value LIMIT 5) UNION DISTINCT (SELECT DISTINCT value, type @@ -66,6 +69,7 @@ def __generic_query(typename, value_length=None): project_id = %(project_id)s AND type='{typename}' AND value ILIKE %(value)s + ORDER BY value LIMIT 5);""" return f"""SELECT DISTINCT value, type FROM final.autocomplete @@ -73,6 +77,7 @@ def __generic_query(typename, value_length=None): project_id = %(project_id)s AND type='{typename}' AND value ILIKE %(svalue)s + ORDER BY value LIMIT 10;""" @@ -86,3 +91,15 @@ def __generic_autocomplete(event: Event): return helper.list_to_camel_case(results) return f + + +def __generic_autocomplete_metas(typename): + def f(project_id, text): + with ch_client.ClickHouseClient() as cur: + query = __generic_query(typename, value_length=len(text)) + params = {"project_id": project_id, "value": helper.string_to_sql_like(text), + "svalue": helper.string_to_sql_like("^" + text)} + results = cur.execute(query=query, params=params) + return results + + return f