jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(chalice): refactored autocomplete search

Browse source
This commit is contained in:
Taha Yassine Kraiem 2023-01-04 18:37:44 +01:00
parent 1349e15fee
commit 01cb5f9bcf
8 changed files with 35 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
api/chalicelib/core/autocomplete.py
View file

@ -135,7 +135,7 @@ def __generic_autocomplete_metas(typename):
return f return f
def __pg_errors_query(source=None, value_length=None): def __errors_query(source=None, value_length=None):
if value_length is None or value_length > 2: if value_length is None or value_length > 2:
return f"""((SELECT DISTINCT ON(lg.message) return f"""((SELECT DISTINCT ON(lg.message)
lg.message AS value, lg.message AS value,
@ -209,11 +209,11 @@ def __pg_errors_query(source=None, value_length=None):
LIMIT 5));""" LIMIT 5));"""
def __search_pg_errors(project_id, value, key=None, source=None): def __search_errors(project_id, value, key=None, source=None):
with pg_client.PostgresClient() as cur: with pg_client.PostgresClient() as cur:
cur.execute( cur.execute(
cur.mogrify(__pg_errors_query(source, cur.mogrify(__errors_query(source,
value_length=len(value)), value_length=len(value)),
{"project_id": project_id, "value": helper.string_to_sql_like(value), {"project_id": project_id, "value": helper.string_to_sql_like(value),
"svalue": helper.string_to_sql_like("^" + value), "svalue": helper.string_to_sql_like("^" + value),
"source": source})) "source": source}))
@ -221,7 +221,7 @@ def __search_pg_errors(project_id, value, key=None, source=None):
return results return results
def __search_pg_errors_ios(project_id, value, key=None, source=None): def __search_errors_ios(project_id, value, key=None, source=None):
if len(value) > 2: if len(value) > 2:
query = f"""(SELECT DISTINCT ON(lg.reason) query = f"""(SELECT DISTINCT ON(lg.reason)
lg.reason AS value, lg.reason AS value,
@ -289,7 +289,7 @@ def __search_pg_errors_ios(project_id, value, key=None, source=None):
return results return results
def __search_pg_metadata(project_id, value, key=None, source=None): def __search_metadata(project_id, value, key=None, source=None):
meta_keys = metadata.get(project_id=project_id) meta_keys = metadata.get(project_id=project_id)
meta_keys = {m["key"]: m["index"] for m in meta_keys} meta_keys = {m["key"]: m["index"] for m in meta_keys}
if len(meta_keys) == 0 or key is not None and key not in meta_keys.keys(): if len(meta_keys) == 0 or key is not None and key not in meta_keys.keys():

4
api/chalicelib/core/click_maps.py
View file

@ -70,7 +70,7 @@ def search_short_session(data: schemas.FlatClickMapSessionsSearch, project_id, u
if session: if session:
session['domURL'] = sessions_mobs.get_urls(session_id=session["session_id"], project_id=project_id) session['domURL'] = sessions_mobs.get_urls(session_id=session["session_id"], project_id=project_id)
session['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session["session_id"]) session['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session["session_id"])
session['events'] = events.get_by_sessionId2_pg(project_id=project_id, session_id=session["session_id"], session['events'] = events.get_by_session_id(project_id=project_id, session_id=session["session_id"],
event_type=schemas.EventType.location) event_type=schemas.EventType.location)
return helper.dict_to_camel_case(session) return helper.dict_to_camel_case(session)

1
api/chalicelib/core/countries.py
View file

@ -257,6 +257,7 @@ COUNTRIES = {
"UA": "Ukraine", "UA": "Ukraine",
"UG": "Uganda", "UG": "Uganda",
"UM": "United States Minor Outlying Islands", "UM": "United States Minor Outlying Islands",
"UN": "United Nations",
"US": "United States", "US": "United States",
"UY": "Uruguay", "UY": "Uruguay",
"UZ": "Uzbekistan", "UZ": "Uzbekistan",

10
api/chalicelib/core/events.py
View file

@ -9,7 +9,7 @@ from chalicelib.utils.TimeUTC import TimeUTC
from chalicelib.utils.event_filter_definition import SupportedFilter, Event from chalicelib.utils.event_filter_definition import SupportedFilter, Event
def get_customs_by_sessionId2_pg(session_id, project_id): def get_customs_by_session_id(session_id, project_id):
with pg_client.PostgresClient() as cur: with pg_client.PostgresClient() as cur:
cur.execute(cur.mogrify("""\ cur.execute(cur.mogrify("""\
SELECT SELECT
@ -52,7 +52,7 @@ def __get_grouped_clickrage(rows, session_id, project_id):
return rows return rows
def get_by_sessionId2_pg(session_id, project_id, group_clickrage=False, event_type: Optional[schemas.EventType] = None): def get_by_session_id(session_id, project_id, group_clickrage=False, event_type: Optional[schemas.EventType] = None):
with pg_client.PostgresClient() as cur: with pg_client.PostgresClient() as cur:
rows = [] rows = []
if event_type is None or event_type == schemas.EventType.click: if event_type is None or event_type == schemas.EventType.click:
@ -138,9 +138,9 @@ SUPPORTED_TYPES = {
EventType.STATEACTION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.STATEACTION), EventType.STATEACTION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.STATEACTION),
query=autocomplete.__generic_query( query=autocomplete.__generic_query(
typename=EventType.STATEACTION.ui_type)), typename=EventType.STATEACTION.ui_type)),
EventType.ERROR.ui_type: SupportedFilter(get=autocomplete.__search_pg_errors, EventType.ERROR.ui_type: SupportedFilter(get=autocomplete.__search_errors,
query=None), query=None),
EventType.METADATA.ui_type: SupportedFilter(get=autocomplete.__search_pg_metadata, EventType.METADATA.ui_type: SupportedFilter(get=autocomplete.__search_metadata,
query=None), query=None),
# IOS # IOS
EventType.CLICK_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.CLICK_IOS), EventType.CLICK_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.CLICK_IOS),
@ -158,7 +158,7 @@ SUPPORTED_TYPES = {
EventType.REQUEST_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.REQUEST_IOS), EventType.REQUEST_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.REQUEST_IOS),
query=autocomplete.__generic_query( query=autocomplete.__generic_query(
typename=EventType.REQUEST_IOS.ui_type)), typename=EventType.REQUEST_IOS.ui_type)),
EventType.ERROR_IOS.ui_type: SupportedFilter(get=autocomplete.__search_pg_errors_ios, EventType.ERROR_IOS.ui_type: SupportedFilter(get=autocomplete.__search_errors_ios,
query=None), query=None),
} }

8
api/chalicelib/core/sessions.py
View file

@ -85,16 +85,16 @@ def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_
session_id=session_id) session_id=session_id)
data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id)
else: else:
data['events'] = events.get_by_sessionId2_pg(project_id=project_id, session_id=session_id, data['events'] = events.get_by_session_id(project_id=project_id, session_id=session_id,
group_clickrage=True) group_clickrage=True)
all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id)
data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"]
# to keep only the first stack # to keep only the first stack
# limit the number of errors to reduce the response-body size # limit the number of errors to reduce the response-body size
data['errors'] = [errors.format_first_stack_frame(e) for e in all_errors data['errors'] = [errors.format_first_stack_frame(e) for e in all_errors
if e['source'] == "js_exception"][:500] if e['source'] == "js_exception"][:500]
data['userEvents'] = events.get_customs_by_sessionId2_pg(project_id=project_id, data['userEvents'] = events.get_customs_by_session_id(project_id=project_id,
session_id=session_id) session_id=session_id)
data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id)
data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id)
data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id) data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id)

2
api/routers/subs/v1_api.py
View file

@ -30,7 +30,7 @@ def get_session_events(projectKey: str, sessionId: int):
if projectId is None: if projectId is None:
return {"errors": ["invalid projectKey"]} return {"errors": ["invalid projectKey"]}
return { return {
'data': events.get_by_sessionId2_pg( 'data': events.get_by_session_id(
project_id=projectId, project_id=projectId,
session_id=sessionId session_id=sessionId
) )

22
ee/api/chalicelib/core/autocomplete_exp.py
View file

@ -36,7 +36,7 @@ def __get_autocomplete_table(value, project_id):
WHERE project_id = %(project_id)s WHERE project_id = %(project_id)s
AND type= '{e}' AND type= '{e}'
AND value ILIKE %(svalue)s AND value ILIKE %(svalue)s
ORDER BY value ORDER BY value
LIMIT 5)""") LIMIT 5)""")
if len(value) > 2: if len(value) > 2:
sub_queries.append(f"""(SELECT type, value sub_queries.append(f"""(SELECT type, value
@ -44,7 +44,7 @@ def __get_autocomplete_table(value, project_id):
WHERE project_id = %(project_id)s WHERE project_id = %(project_id)s
AND type= '{e}' AND type= '{e}'
AND value ILIKE %(value)s AND value ILIKE %(value)s
ORDER BY value ORDER BY value
LIMIT 5)""") LIMIT 5)""")
with ch_client.ClickHouseClient() as cur: with ch_client.ClickHouseClient() as cur:
query = " UNION DISTINCT ".join(sub_queries) + ";" query = " UNION DISTINCT ".join(sub_queries) + ";"
@ -120,11 +120,17 @@ def __generic_autocomplete(event: Event):
def __generic_autocomplete_metas(typename): def __generic_autocomplete_metas(typename):
def f(project_id, text): def f(project_id, text):
with ch_client.ClickHouseClient() as cur: with ch_client.ClickHouseClient() as cur:
query = __generic_query(typename, value_length=len(text))
params = {"project_id": project_id, "value": helper.string_to_sql_like(text), params = {"project_id": project_id, "value": helper.string_to_sql_like(text),
"svalue": helper.string_to_sql_like("^" + text)} "svalue": helper.string_to_sql_like("^" + text)}
results = cur.execute(query=query, params=params)
return results if typename == schemas.FilterType.user_country:
params["value"] = tuple(countries.get_country_code_autocomplete(text))
if len(params["value"]) == 0:
return []
query = __generic_query(typename, value_length=len(text))
rows = cur.execute(query=query, params=params)
return rows
return f return f
@ -199,7 +205,7 @@ def __pg_errors_query(source=None, value_length=None):
LIMIT 5));""" LIMIT 5));"""
def __search_pg_errors(project_id, value, key=None, source=None): def __search_errors(project_id, value, key=None, source=None):
with ch_client.ClickHouseClient() as cur: with ch_client.ClickHouseClient() as cur:
query = cur.format(__pg_errors_query(source, value_length=len(value)), query = cur.format(__pg_errors_query(source, value_length=len(value)),
{"project_id": project_id, "value": helper.string_to_sql_like(value), {"project_id": project_id, "value": helper.string_to_sql_like(value),
@ -209,12 +215,12 @@ def __search_pg_errors(project_id, value, key=None, source=None):
return helper.list_to_camel_case(results) return helper.list_to_camel_case(results)
def __search_pg_errors_ios(project_id, value, key=None, source=None): def __search_errors_ios(project_id, value, key=None, source=None):
# TODO: define this when ios events are supported in CH # TODO: define this when ios events are supported in CH
return [] return []
def __search_pg_metadata(project_id, value, key=None, source=None): def __search_metadata(project_id, value, key=None, source=None):
meta_keys = metadata.get(project_id=project_id) meta_keys = metadata.get(project_id=project_id)
meta_keys = {m["key"]: m["index"] for m in meta_keys} meta_keys = {m["key"]: m["index"] for m in meta_keys}
if len(meta_keys) == 0 or key is not None and key not in meta_keys.keys(): if len(meta_keys) == 0 or key is not None and key not in meta_keys.keys():

5
ee/api/chalicelib/core/events.py
View file

@ -14,8 +14,7 @@ if config("EXP_AUTOCOMPLETE", cast=bool, default=False):
else: else:
from . import autocomplete as autocomplete from . import autocomplete as autocomplete
def get_customs_by_session_id(session_id, project_id):
def get_customs_by_sessionId2_pg(session_id, project_id):
with pg_client.PostgresClient() as cur: with pg_client.PostgresClient() as cur:
cur.execute(cur.mogrify("""\ cur.execute(cur.mogrify("""\
SELECT SELECT
@ -58,7 +57,7 @@ def __get_grouped_clickrage(rows, session_id, project_id):
return rows return rows
def get_by_sessionId2_pg(session_id, project_id, group_clickrage=False, event_type: Optional[schemas.EventType] = None): def get_by_session_id(session_id, project_id, group_clickrage=False, event_type: Optional[schemas.EventType] = None):
with pg_client.PostgresClient() as cur: with pg_client.PostgresClient() as cur:
rows = [] rows = []
if event_type is None or event_type == schemas.EventType.click: if event_type is None or event_type == schemas.EventType.click: