dd5ff6bad8
* refactor(chalice): upgraded dependencies * refactor(chalice): upgraded dependencies feat(chalice): support heatmaps * feat(chalice): support table-of-browsers showing user-count * feat(chalice): support table-of-devices showing user-count * feat(chalice): support table-of-URLs showing user-count * fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * refactor(chalice): upgraded dependencies * refactor(chalice): upgraded dependencies feat(chalice): support heatmaps * feat(chalice): support table-of-browsers showing user-count * feat(chalice): support table-of-devices showing user-count * feat(chalice): support table-of-URLs showing user-count * fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * refactor(chalice): refactored search sessions hooks * refactor(DB): DB delta * refactor(DB): DB delta * refactor(DB): DB delta * refactor(chalice): refactored schemas * refactor(chalice): refactored schemas refactor(chalice): cleaned scripts feat(chalice): search sessions by CSS selector (PG)
261 lines
11 KiB
Python
261 lines
11 KiB
Python
import schemas
|
|
from chalicelib.core import countries, events, metadata
|
|
from chalicelib.utils import ch_client
|
|
from chalicelib.utils import helper, exp_ch_helper
|
|
from chalicelib.utils.event_filter_definition import Event
|
|
|
|
TABLE = "experimental.autocomplete"
|
|
|
|
|
|
def __get_autocomplete_table(value, project_id):
|
|
autocomplete_events = [schemas.FilterType.REV_ID,
|
|
schemas.EventType.CLICK,
|
|
schemas.FilterType.USER_DEVICE,
|
|
schemas.FilterType.USER_ID,
|
|
schemas.FilterType.USER_BROWSER,
|
|
schemas.FilterType.USER_OS,
|
|
schemas.EventType.CUSTOM,
|
|
schemas.FilterType.USER_COUNTRY,
|
|
schemas.FilterType.USER_CITY,
|
|
schemas.FilterType.USER_STATE,
|
|
schemas.EventType.LOCATION,
|
|
schemas.EventType.INPUT]
|
|
autocomplete_events.sort()
|
|
sub_queries = []
|
|
c_list = []
|
|
for e in autocomplete_events:
|
|
if e == schemas.FilterType.USER_COUNTRY:
|
|
c_list = countries.get_country_code_autocomplete(value)
|
|
if len(c_list) > 0:
|
|
sub_queries.append(f"""(SELECT DISTINCT ON(value) '{e.value}' AS _type, value
|
|
FROM {TABLE}
|
|
WHERE project_id = %(project_id)s
|
|
AND type= '{e.value.upper()}'
|
|
AND value IN %(c_list)s)""")
|
|
continue
|
|
sub_queries.append(f"""(SELECT '{e.value}' AS _type, value
|
|
FROM {TABLE}
|
|
WHERE project_id = %(project_id)s
|
|
AND type= '{e.value.upper()}'
|
|
AND value ILIKE %(svalue)s
|
|
ORDER BY value
|
|
LIMIT 5)""")
|
|
if len(value) > 2:
|
|
sub_queries.append(f"""(SELECT '{e.value}' AS _type, value
|
|
FROM {TABLE}
|
|
WHERE project_id = %(project_id)s
|
|
AND type= '{e.value.upper()}'
|
|
AND value ILIKE %(value)s
|
|
ORDER BY value
|
|
LIMIT 5)""")
|
|
with ch_client.ClickHouseClient() as cur:
|
|
query = " UNION DISTINCT ".join(sub_queries) + ";"
|
|
params = {"project_id": project_id,
|
|
"value": helper.string_to_sql_like(value),
|
|
"svalue": helper.string_to_sql_like("^" + value),
|
|
"c_list": tuple(c_list)}
|
|
results = []
|
|
try:
|
|
results = cur.execute(query=query, params=params)
|
|
except Exception as err:
|
|
print("--------- CH AUTOCOMPLETE SEARCH QUERY EXCEPTION -----------")
|
|
print(cur.format(query=query, params=params))
|
|
print("--------- PARAMS -----------")
|
|
print(params)
|
|
print("--------- VALUE -----------")
|
|
print(value)
|
|
print("--------------------")
|
|
raise err
|
|
for r in results:
|
|
r["type"] = r.pop("_type")
|
|
results = helper.list_to_camel_case(results)
|
|
return results
|
|
|
|
|
|
def __generic_query(typename, value_length=None):
|
|
if typename == schemas.FilterType.USER_COUNTRY:
|
|
return f"""SELECT DISTINCT value, type
|
|
FROM {TABLE}
|
|
WHERE
|
|
project_id = %(project_id)s
|
|
AND type='{typename.upper()}'
|
|
AND value IN %(value)s
|
|
ORDER BY value"""
|
|
|
|
if value_length is None or value_length > 2:
|
|
return f"""(SELECT DISTINCT value, type
|
|
FROM {TABLE}
|
|
WHERE
|
|
project_id = %(project_id)s
|
|
AND type='{typename.upper()}'
|
|
AND value ILIKE %(svalue)s
|
|
ORDER BY value
|
|
LIMIT 5)
|
|
UNION DISTINCT
|
|
(SELECT DISTINCT value, type
|
|
FROM {TABLE}
|
|
WHERE
|
|
project_id = %(project_id)s
|
|
AND type='{typename.upper()}'
|
|
AND value ILIKE %(value)s
|
|
ORDER BY value
|
|
LIMIT 5);"""
|
|
return f"""SELECT DISTINCT value, type
|
|
FROM {TABLE}
|
|
WHERE
|
|
project_id = %(project_id)s
|
|
AND type='{typename.upper()}'
|
|
AND value ILIKE %(svalue)s
|
|
ORDER BY value
|
|
LIMIT 10;"""
|
|
|
|
|
|
def __generic_autocomplete(event: Event):
|
|
def f(project_id, value, key=None, source=None):
|
|
with ch_client.ClickHouseClient() as cur:
|
|
query = __generic_query(event.ui_type, value_length=len(value))
|
|
params = {"project_id": project_id, "value": helper.string_to_sql_like(value),
|
|
"svalue": helper.string_to_sql_like("^" + value)}
|
|
results = cur.execute(query=query, params=params)
|
|
return helper.list_to_camel_case(results)
|
|
|
|
return f
|
|
|
|
|
|
def __generic_autocomplete_metas(typename):
|
|
def f(project_id, text):
|
|
with ch_client.ClickHouseClient() as cur:
|
|
params = {"project_id": project_id, "value": helper.string_to_sql_like(text),
|
|
"svalue": helper.string_to_sql_like("^" + text)}
|
|
|
|
if typename == schemas.FilterType.USER_COUNTRY:
|
|
params["value"] = tuple(countries.get_country_code_autocomplete(text))
|
|
if len(params["value"]) == 0:
|
|
return []
|
|
|
|
query = __generic_query(typename, value_length=len(text))
|
|
rows = cur.execute(query=query, params=params)
|
|
return rows
|
|
|
|
return f
|
|
|
|
|
|
def __pg_errors_query(source=None, value_length=None):
|
|
MAIN_TABLE = exp_ch_helper.get_main_js_errors_sessions_table()
|
|
if value_length is None or value_length > 2:
|
|
return f"""((SELECT DISTINCT ON(message)
|
|
message AS value,
|
|
source,
|
|
'{events.EventType.ERROR.ui_type}' AS type
|
|
FROM {MAIN_TABLE}
|
|
WHERE
|
|
project_id = %(project_id)s
|
|
AND message ILIKE %(svalue)s
|
|
AND event_type = 'ERROR'
|
|
{"AND source = %(source)s" if source is not None else ""}
|
|
LIMIT 5)
|
|
UNION DISTINCT
|
|
(SELECT DISTINCT ON(name)
|
|
name AS value,
|
|
source,
|
|
'{events.EventType.ERROR.ui_type}' AS type
|
|
FROM {MAIN_TABLE}
|
|
WHERE
|
|
project_id = %(project_id)s
|
|
AND name ILIKE %(svalue)s
|
|
{"AND source = %(source)s" if source is not None else ""}
|
|
LIMIT 5)
|
|
UNION DISTINCT
|
|
(SELECT DISTINCT ON(message)
|
|
message AS value,
|
|
source,
|
|
'{events.EventType.ERROR.ui_type}' AS type
|
|
FROM {MAIN_TABLE}
|
|
WHERE
|
|
project_id = %(project_id)s
|
|
AND message ILIKE %(value)s
|
|
{"AND source = %(source)s" if source is not None else ""}
|
|
LIMIT 5)
|
|
UNION DISTINCT
|
|
(SELECT DISTINCT ON(name)
|
|
name AS value,
|
|
source,
|
|
'{events.EventType.ERROR.ui_type}' AS type
|
|
FROM {MAIN_TABLE}
|
|
WHERE
|
|
project_id = %(project_id)s
|
|
AND name ILIKE %(value)s
|
|
{"AND source = %(source)s" if source is not None else ""}
|
|
LIMIT 5));"""
|
|
return f"""((SELECT DISTINCT ON(message)
|
|
message AS value,
|
|
source,
|
|
'{events.EventType.ERROR.ui_type}' AS type
|
|
FROM {MAIN_TABLE}
|
|
WHERE
|
|
project_id = %(project_id)s
|
|
AND message ILIKE %(svalue)s
|
|
{"AND source = %(source)s" if source is not None else ""}
|
|
LIMIT 5)
|
|
UNION DISTINCT
|
|
(SELECT DISTINCT ON(name)
|
|
name AS value,
|
|
source,
|
|
'{events.EventType.ERROR.ui_type}' AS type
|
|
FROM {MAIN_TABLE}
|
|
WHERE
|
|
project_id = %(project_id)s
|
|
AND name ILIKE %(svalue)s
|
|
{"AND source = %(source)s" if source is not None else ""}
|
|
LIMIT 5));"""
|
|
|
|
|
|
def __search_errors(project_id, value, key=None, source=None):
|
|
with ch_client.ClickHouseClient() as cur:
|
|
query = cur.format(__pg_errors_query(source, value_length=len(value)),
|
|
{"project_id": project_id, "value": helper.string_to_sql_like(value),
|
|
"svalue": helper.string_to_sql_like("^" + value),
|
|
"source": source})
|
|
results = cur.execute(query)
|
|
return helper.list_to_camel_case(results)
|
|
|
|
|
|
def __search_errors_mobile(project_id, value, key=None, source=None):
|
|
# TODO: define this when ios events are supported in CH
|
|
return []
|
|
|
|
|
|
def __search_metadata(project_id, value, key=None, source=None):
|
|
meta_keys = metadata.get(project_id=project_id)
|
|
meta_keys = {m["key"]: m["index"] for m in meta_keys}
|
|
if len(meta_keys) == 0 or key is not None and key not in meta_keys.keys():
|
|
return []
|
|
sub_from = []
|
|
if key is not None:
|
|
meta_keys = {key: meta_keys[key]}
|
|
|
|
for k in meta_keys.keys():
|
|
colname = metadata.index_to_colname(meta_keys[k])
|
|
if len(value) > 2:
|
|
sub_from.append(f"""((SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key
|
|
FROM {exp_ch_helper.get_main_sessions_table()}
|
|
WHERE project_id = %(project_id)s
|
|
AND {colname} ILIKE %(svalue)s LIMIT 5)
|
|
UNION DISTINCT
|
|
(SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key
|
|
FROM {exp_ch_helper.get_main_sessions_table()}
|
|
WHERE project_id = %(project_id)s
|
|
AND {colname} ILIKE %(value)s LIMIT 5))
|
|
""")
|
|
else:
|
|
sub_from.append(f"""(SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key
|
|
FROM {exp_ch_helper.get_main_sessions_table()}
|
|
WHERE project_id = %(project_id)s
|
|
AND {colname} ILIKE %(svalue)s LIMIT 5)""")
|
|
with ch_client.ClickHouseClient() as cur:
|
|
query = cur.format(f"""SELECT key, value, 'METADATA' AS TYPE
|
|
FROM({" UNION ALL ".join(sub_from)}) AS all_metas
|
|
LIMIT 5;""", {"project_id": project_id, "value": helper.string_to_sql_like(value),
|
|
"svalue": helper.string_to_sql_like("^" + value)})
|
|
results = cur.execute(query)
|
|
return helper.list_to_camel_case(results)
|