diff --git a/api/chalicelib/core/assist.py b/api/chalicelib/core/assist.py index 8242e69ff..cd76d0be4 100644 --- a/api/chalicelib/core/assist.py +++ b/api/chalicelib/core/assist.py @@ -1,3 +1,4 @@ +import schemas from chalicelib.utils import pg_client, helper from chalicelib.core import projects, sessions, sessions_metas import requests @@ -44,7 +45,7 @@ def get_live_sessions(project_id, filters=None): continue filter_type = f["type"].upper() f["value"] = sessions.__get_sql_value_multiple(f["value"]) - if filter_type == sessions_metas.meta_type.USERID: + if filter_type == schemas.FilterType.user_id: op = sessions.__get_sql_operator(f["operator"]) extra_constraints.append(f"user_id {op} %(value_{i})s") extra_params[f"value_{i}"] = helper.string_to_sql_like_with_op(f["value"][0], op) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index 919d6aa5a..9cd88eb6a 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -1,3 +1,4 @@ +import schemas from chalicelib.core import metadata from chalicelib.utils import args_transformer from chalicelib.utils import helper, dev @@ -94,25 +95,25 @@ def __get_meta_constraint(project_id, data): else: filter_type = f["key"].upper() filter_type = [filter_type, "USER" + filter_type, filter_type[4:]] - if any(item in [sessions_metas.meta_type.USERBROWSER] \ + if any(item in [schemas.FilterType.user_browser] \ for item in filter_type): constraints.append(f"sessions.user_browser = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS] \ + elif any(item in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios] \ for item in filter_type): constraints.append(f"sessions.user_os = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS] \ + elif any(item in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios] \ for item in filter_type): constraints.append(f"sessions.user_device = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS] \ + elif any(item in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios] \ for item in filter_type): constraints.append(f"sessions.user_country = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS] \ + elif any(item in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios] \ for item in filter_type): constraints.append(f"sessions.user_id = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.USERANONYMOUSID, sessions_metas.meta_type.USERANONYMOUSID_IOS] \ + elif any(item in [schemas.FilterType.user_anonymous_id, schemas.FilterType.user_anonymous_id_ios] \ for item in filter_type): constraints.append(f"sessions.user_anonymous_id = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS] \ + elif any(item in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios] \ for item in filter_type): constraints.append(f"sessions.rev_id = %({f['key']}_{i})s") return constraints diff --git a/api/chalicelib/core/events.py b/api/chalicelib/core/events.py index deb16cb8b..0a330d625 100644 --- a/api/chalicelib/core/events.py +++ b/api/chalicelib/core/events.py @@ -390,18 +390,18 @@ def search_pg2(text, event_type, project_id, source, key): if not event_type: return {"data": __get_autocomplete_table(text, project_id)} - if event_type.upper() in SUPPORTED_TYPES.keys(): - rows = SUPPORTED_TYPES[event_type.upper()].get(project_id=project_id, value=text, key=key, source=source) - if event_type.upper() + "_IOS" in SUPPORTED_TYPES.keys(): - rows += SUPPORTED_TYPES[event_type.upper() + "_IOS"].get(project_id=project_id, value=text, key=key, - source=source) - elif event_type.upper() + "_IOS" in SUPPORTED_TYPES.keys(): - rows = SUPPORTED_TYPES[event_type.upper() + "_IOS"].get(project_id=project_id, value=text, key=key, - source=source) - elif event_type.upper() in sessions_metas.SUPPORTED_TYPES.keys(): + if event_type in SUPPORTED_TYPES.keys(): + rows = SUPPORTED_TYPES[event_type].get(project_id=project_id, value=text, key=key, source=source) + if event_type + "_IOS" in SUPPORTED_TYPES.keys(): + rows += SUPPORTED_TYPES[event_type + "_IOS"].get(project_id=project_id, value=text, key=key, + source=source) + elif event_type + "_IOS" in SUPPORTED_TYPES.keys(): + rows = SUPPORTED_TYPES[event_type + "_IOS"].get(project_id=project_id, value=text, key=key, + source=source) + elif event_type in sessions_metas.SUPPORTED_TYPES.keys(): return sessions_metas.search(text, event_type, project_id) - elif event_type.upper().endswith("_IOS") \ - and event_type.upper()[:-len("_IOS")] in sessions_metas.SUPPORTED_TYPES.keys(): + elif event_type.endswith("_IOS") \ + and event_type[:-len("_IOS")] in sessions_metas.SUPPORTED_TYPES.keys(): return sessions_metas.search(text, event_type, project_id) else: return {"errors": ["unsupported event"]} diff --git a/api/chalicelib/core/insights.py b/api/chalicelib/core/insights.py index 79b32a4b1..08adfd3ca 100644 --- a/api/chalicelib/core/insights.py +++ b/api/chalicelib/core/insights.py @@ -1,3 +1,4 @@ +import schemas from chalicelib.core import sessions_metas from chalicelib.utils import helper, dev from chalicelib.utils import pg_client @@ -45,7 +46,7 @@ def journey(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp= elif f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query_subset.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] @@ -300,7 +301,7 @@ def feature_retention(project_id, startTimestamp=TimeUTC.now(delta_days=-70), en elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] @@ -390,7 +391,7 @@ def feature_acquisition(project_id, startTimestamp=TimeUTC.now(delta_days=-70), elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] @@ -477,7 +478,7 @@ def feature_popularity_frequency(project_id, startTimestamp=TimeUTC.now(delta_da if f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] @@ -543,7 +544,7 @@ def feature_adoption(project_id, startTimestamp=TimeUTC.now(delta_days=-70), end elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] @@ -613,7 +614,7 @@ def feature_adoption_top_users(project_id, startTimestamp=TimeUTC.now(delta_days elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] @@ -674,7 +675,7 @@ def feature_adoption_daily_usage(project_id, startTimestamp=TimeUTC.now(delta_da elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query_chart.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] @@ -737,7 +738,7 @@ def feature_intensity(project_id, startTimestamp=TimeUTC.now(delta_days=-70), en if f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] pg_sub_query.append(f"length({event_column})>2") @@ -772,7 +773,7 @@ def users_active(project_id, startTimestamp=TimeUTC.now(delta_days=-70), endTime for f in filters: if f["type"] == "PERIOD" and f["value"] in ["DAY", "WEEK"]: period = f["value"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query_chart.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] @@ -844,7 +845,7 @@ def users_slipping(project_id, startTimestamp=TimeUTC.now(delta_days=-70), endTi elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index d0c674007..0067b7c29 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1,5 +1,5 @@ import schemas -from chalicelib.core import events, sessions_metas, metadata, events_ios, \ +from chalicelib.core import events, metadata, events_ios, \ sessions_mobs, issues, projects, errors, resources, assist, performance_event from chalicelib.utils import pg_client, helper, dev @@ -194,37 +194,71 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} op = __get_sql_operator(f.operator) \ if filter_type not in [schemas.FilterType.events_count] else f.operator + is_any = _isAny_opreator(f.operator) is_not = False if __is_negation_operator(f.operator): is_not = True # op = __reverse_sql_operator(op) - if filter_type == sessions_metas.meta_type.USERBROWSER: + if filter_type == schemas.FilterType.user_browser: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: + elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: + elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: + elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.utm_source]: + if is_any: + extra_constraints.append('s.utm_source IS NOT NULL') + ss_constraints.append('ms.utm_source IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_source {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_source {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_medium]: + if is_any: + extra_constraints.append('s.utm_medium IS NOT NULL') + ss_constraints.append('ms.utm_medium IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_medium {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_medium {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_campaign]: + if is_any: + extra_constraints.append('s.utm_campaign IS NOT NULL') + ss_constraints.append('ms.utm_campaign IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.duration: if len(f.value) > 0 and f.value[0] is not None: extra_constraints.append("s.duration >= %(minDuration)s") @@ -234,7 +268,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f extra_constraints.append("s.duration <= %(maxDuration)s") ss_constraints.append("ms.duration <= %(maxDuration)s") full_args["maxDuration"] = f.value[1] - elif filter_type == sessions_metas.meta_type.REFERRER: + elif filter_type == schemas.FilterType.referrer: # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" # op = __get_sql_operator_multiple(f.operator) @@ -253,14 +287,14 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f ss_constraints.append( _multiple_conditions(f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: # op = __get_sql_operator(f.operator) extra_constraints.append( _multiple_conditions(f"s.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( _multiple_conditions(f"ms.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, - sessions_metas.meta_type.USERANONYMOUSID_IOS]: + elif filter_type in [schemas.FilterType.user_anonymous_id, + schemas.FilterType.user_anonymous_id_ios]: # op = __get_sql_operator(f.operator) extra_constraints.append( _multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, @@ -268,7 +302,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f ss_constraints.append( _multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: + elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: # op = __get_sql_operator(f.operator) extra_constraints.append( _multiple_conditions(f"s.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) @@ -695,8 +729,8 @@ def search_by_metadata(tenant_id, user_id, m_key, m_value, project_id=None): available_keys = metadata.get_keys_by_projects(project_ids) for i in available_keys: - available_keys[i]["user_id"] = sessions_metas.meta_type.USERID - available_keys[i]["user_anonymous_id"] = sessions_metas.meta_type.USERANONYMOUSID + available_keys[i]["user_id"] = schemas.FilterType.user_id + available_keys[i]["user_anonymous_id"] = schemas.FilterType.user_anonymous_id results = {} for i in project_ids: if m_key not in available_keys[i].values(): diff --git a/api/chalicelib/core/sessions_metas.py b/api/chalicelib/core/sessions_metas.py index 1e55720a9..1d342d03f 100644 --- a/api/chalicelib/core/sessions_metas.py +++ b/api/chalicelib/core/sessions_metas.py @@ -9,40 +9,47 @@ def get_key_values(project_id): cur.mogrify( f"""\ SELECT ARRAY_AGG(DISTINCT s.user_os - ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='web') AS {meta_type.USEROS}, + ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='web') AS {schemas.FilterType.user_os}, ARRAY_AGG(DISTINCT s.user_browser ORDER BY s.user_browser) - FILTER ( WHERE s.user_browser IS NOT NULL AND s.platform='web') AS {meta_type.USERBROWSER}, + FILTER ( WHERE s.user_browser IS NOT NULL AND s.platform='web') AS {schemas.FilterType.user_browser}, ARRAY_AGG(DISTINCT s.user_device ORDER BY s.user_device) - FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='web') AS {meta_type.USERDEVICE}, + FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='web') AS {schemas.FilterType.user_device}, ARRAY_AGG(DISTINCT s.user_country ORDER BY s.user_country) - FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='web')::text[] AS {meta_type.USERCOUNTRY}, + FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='web')::text[] AS {schemas.FilterType.user_country}, ARRAY_AGG(DISTINCT s.user_id - ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='web') AS {meta_type.USERID}, + ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='web') AS {schemas.FilterType.user_id}, ARRAY_AGG(DISTINCT s.user_anonymous_id - ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='web') AS {meta_type.USERANONYMOUSID}, + ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='web') AS {schemas.FilterType.user_anonymous_id}, ARRAY_AGG(DISTINCT s.rev_id - ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='web') AS {meta_type.REVID}, + ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='web') AS {schemas.FilterType.rev_id}, ARRAY_AGG(DISTINCT p.referrer ORDER BY p.referrer) - FILTER ( WHERE p.referrer != '' ) AS {meta_type.REFERRER}, + FILTER ( WHERE p.referrer != '' ) AS {schemas.FilterType.referrer}, + + ARRAY_AGG(DISTINCT s.utm_source + ORDER BY s.utm_source) FILTER ( WHERE s.utm_source IS NOT NULL AND s.utm_source != 'none' AND s.utm_source != '') AS {schemas.FilterType.utm_source}, + ARRAY_AGG(DISTINCT s.utm_medium + ORDER BY s.utm_medium) FILTER ( WHERE s.utm_medium IS NOT NULL AND s.utm_medium != 'none' AND s.utm_medium != '') AS {schemas.FilterType.utm_medium}, + ARRAY_AGG(DISTINCT s.utm_campaign + ORDER BY s.utm_campaign) FILTER ( WHERE s.utm_campaign IS NOT NULL AND s.utm_campaign != 'none' AND s.utm_campaign != '') AS {schemas.FilterType.utm_campaign}, ARRAY_AGG(DISTINCT s.user_os - ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='ios' ) AS {meta_type.USEROS_IOS}, + ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='ios' ) AS {schemas.FilterType.user_os_ios}, ARRAY_AGG(DISTINCT s.user_device ORDER BY s.user_device) - FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='ios') AS {meta_type.USERDEVICE}, + FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='ios') AS {schemas.FilterType.user_device_ios}, ARRAY_AGG(DISTINCT s.user_country ORDER BY s.user_country) - FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='ios')::text[] AS {meta_type.USERCOUNTRY_IOS}, + FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='ios')::text[] AS {schemas.FilterType.user_country_ios}, ARRAY_AGG(DISTINCT s.user_id - ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='ios') AS {meta_type.USERID_IOS}, + ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='ios') AS {schemas.FilterType.user_id_ios}, ARRAY_AGG(DISTINCT s.user_anonymous_id - ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='ios') AS {meta_type.USERANONYMOUSID_IOS}, + ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='ios') AS {schemas.FilterType.user_anonymous_id_ios}, ARRAY_AGG(DISTINCT s.rev_id - ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='ios') AS {meta_type.REVID_IOS} + ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='ios') AS {schemas.FilterType.rev_id_ios} FROM public.sessions AS s LEFT JOIN events.pages AS p USING (session_id) WHERE s.project_id = %(site_id)s;""", @@ -109,119 +116,137 @@ def __generic_autocomplete(typename): return f -class meta_type: - USEROS = schemas.FilterType.user_os - USERBROWSER = schemas.FilterType.user_browser - USERDEVICE = schemas.FilterType.user_device - USERCOUNTRY = schemas.FilterType.user_country - USERID = schemas.FilterType.user_id - USERANONYMOUSID = schemas.FilterType.user_anonymous_id - REFERRER = schemas.FilterType.referrer - REVID = schemas.FilterType.rev_id - # IOS - USEROS_IOS = schemas.FilterType.user_os_ios - USERDEVICE_IOS = schemas.FilterType.user_device_ios - USERCOUNTRY_IOS = schemas.FilterType.user_country_ios - USERID_IOS = schemas.FilterType.user_id_ios - USERANONYMOUSID_IOS = schemas.FilterType.user_anonymous_id_ios - REVID_IOS = schemas.FilterType.rev_id_ios - - SUPPORTED_TYPES = { - meta_type.USEROS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USEROS), - query=__generic_query(typename=meta_type.USEROS), - value_limit=0, - starts_with="", - starts_limit=0, - ignore_if_starts_with=["/"]), - meta_type.USERBROWSER: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERBROWSER), - query=__generic_query(typename=meta_type.USERBROWSER), - value_limit=0, - starts_with="", - starts_limit=0, - ignore_if_starts_with=["/"]), - meta_type.USERDEVICE: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERDEVICE), - query=__generic_query(typename=meta_type.USERDEVICE), - value_limit=3, - starts_with="", - starts_limit=3, - ignore_if_starts_with=["/"]), - meta_type.USERCOUNTRY: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERCOUNTRY), - query=__generic_query(typename=meta_type.USERCOUNTRY), - value_limit=2, - starts_with="", - starts_limit=2, - ignore_if_starts_with=["/"]), - meta_type.USERID: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERID), - query=__generic_query(typename=meta_type.USERID), - value_limit=2, - starts_with="", - starts_limit=2, - ignore_if_starts_with=["/"]), - meta_type.USERANONYMOUSID: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERANONYMOUSID), - query=__generic_query(typename=meta_type.USERANONYMOUSID), - value_limit=3, - starts_with="", - starts_limit=3, - ignore_if_starts_with=["/"]), - meta_type.REVID: SupportedFilter(get=__generic_autocomplete(typename=meta_type.REVID), - query=__generic_query(typename=meta_type.REVID), - value_limit=0, - starts_with="", - starts_limit=0, - ignore_if_starts_with=["/"]), - meta_type.REFERRER: SupportedFilter(get=__generic_autocomplete(typename=meta_type.REFERRER), - query=__generic_query(typename=meta_type.REFERRER), - value_limit=5, - starts_with="/", - starts_limit=5, - ignore_if_starts_with=[]), + schemas.FilterType.user_os: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_os), + query=__generic_query(typename=schemas.FilterType.user_os), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_browser: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_browser), + query=__generic_query(typename=schemas.FilterType.user_browser), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_device: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_device), + query=__generic_query(typename=schemas.FilterType.user_device), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_country: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_country), + query=__generic_query(typename=schemas.FilterType.user_country), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_id: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_id), + query=__generic_query(typename=schemas.FilterType.user_id), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_anonymous_id: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_anonymous_id), + query=__generic_query(typename=schemas.FilterType.user_anonymous_id), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + schemas.FilterType.rev_id: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.rev_id), + query=__generic_query(typename=schemas.FilterType.rev_id), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.referrer: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.referrer), + query=__generic_query(typename=schemas.FilterType.referrer), + value_limit=5, + starts_with="/", + starts_limit=5, + ignore_if_starts_with=[]), + schemas.FilterType.utm_campaign: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.utm_campaign), + query=__generic_query(typename=schemas.FilterType.utm_campaign), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.utm_medium: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.utm_medium), + query=__generic_query(typename=schemas.FilterType.utm_medium), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.utm_source: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.utm_source), + query=__generic_query(typename=schemas.FilterType.utm_source), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), # IOS - meta_type.USEROS_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USEROS_IOS), - query=__generic_query(typename=meta_type.USEROS_IOS), - value_limit=0, - starts_with="", - starts_limit=0, - ignore_if_starts_with=["/"]), - meta_type.USERDEVICE_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERDEVICE_IOS), - query=__generic_query(typename=meta_type.USERDEVICE_IOS), - value_limit=3, - starts_with="", - starts_limit=3, - ignore_if_starts_with=["/"]), - meta_type.USERCOUNTRY_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERCOUNTRY_IOS), - query=__generic_query(typename=meta_type.USERCOUNTRY_IOS), - value_limit=2, - starts_with="", - starts_limit=2, - ignore_if_starts_with=["/"]), - meta_type.USERID_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERID_IOS), - query=__generic_query(typename=meta_type.USERID_IOS), - value_limit=2, - starts_with="", - starts_limit=2, - ignore_if_starts_with=["/"]), - meta_type.USERANONYMOUSID_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERANONYMOUSID_IOS), - query=__generic_query(typename=meta_type.USERANONYMOUSID_IOS), - value_limit=3, - starts_with="", - starts_limit=3, - ignore_if_starts_with=["/"]), - meta_type.REVID_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.REVID_IOS), - query=__generic_query(typename=meta_type.REVID_IOS), - value_limit=0, - starts_with="", - starts_limit=0, - ignore_if_starts_with=["/"]), + schemas.FilterType.user_os_ios: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_os_ios), + query=__generic_query(typename=schemas.FilterType.user_os_ios), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_device_ios: SupportedFilter( + get=__generic_autocomplete( + typename=schemas.FilterType.user_device_ios), + query=__generic_query(typename=schemas.FilterType.user_device_ios), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_country_ios: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_country_ios), + query=__generic_query(typename=schemas.FilterType.user_country_ios), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_id_ios: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_id_ios), + query=__generic_query(typename=schemas.FilterType.user_id_ios), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_anonymous_id_ios: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_anonymous_id_ios), + query=__generic_query(typename=schemas.FilterType.user_anonymous_id_ios), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + schemas.FilterType.rev_id_ios: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.rev_id_ios), + query=__generic_query(typename=schemas.FilterType.rev_id_ios), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), } def search(text, meta_type, project_id): rows = [] - if meta_type.upper() not in list(SUPPORTED_TYPES.keys()): + if meta_type not in list(SUPPORTED_TYPES.keys()): return {"errors": ["unsupported type"]} - rows += SUPPORTED_TYPES[meta_type.upper()].get(project_id=project_id, text=text) - if meta_type.upper() + "_IOS" in list(SUPPORTED_TYPES.keys()): - rows += SUPPORTED_TYPES[meta_type.upper() + "_IOS"].get(project_id=project_id, text=text) + rows += SUPPORTED_TYPES[meta_type].get(project_id=project_id, text=text) + if meta_type + "_IOS" in list(SUPPORTED_TYPES.keys()): + rows += SUPPORTED_TYPES[meta_type + "_IOS"].get(project_id=project_id, text=text) return {"data": rows} diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index 3f991b895..9fc55065f 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -59,22 +59,22 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: values = {**values, **sessions._multiple_values(helper.values_for_operator(value=f["value"], op=f["operator"]), value_key=f_k)} - if filter_type == sessions_metas.meta_type.USERBROWSER: + if filter_type == schemas.FilterType.user_browser: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_browser {op} %({f_k})s', f["value"], value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: + elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_os {op} %({f_k})s', f["value"], value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: + elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_device {op} %({f_k})s', f["value"], value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: + elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_country {op} %({f_k})s', f["value"], value_key=f_k)) @@ -85,7 +85,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: if len(f["value"]) > 1 and f["value"][1] is not None and int(f["value"][1]) > 0: first_stage_extra_constraints.append('s.duration <= %(maxDuration)s') values["maxDuration"] = f["value"][1] - elif filter_type == sessions_metas.meta_type.REFERRER: + elif filter_type == schemas.FilterType.referrer: # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" filter_extra_from = [f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)"] # op = sessions.__get_sql_operator_multiple(f["operator"]) @@ -102,18 +102,18 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: f's.{metadata.index_to_colname(meta_keys[f["key"]])} {op} %({f_k})s', f["value"], value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_id {op} %({f_k})s', f["value"], value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, - sessions_metas.meta_type.USERANONYMOUSID_IOS]: + elif filter_type in [schemas.FilterType.user_anonymous_id, + schemas.FilterType.user_anonymous_id_ios]: # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_anonymous_id {op} %({f_k})s', f["value"], value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: + elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.rev_id {op} %({f_k})s', f["value"], value_key=f_k)) @@ -561,7 +561,8 @@ def get_top_insights(filter_d, project_id): "dropDueToIssues": 0 }] - counts = sessions.search2_pg(data=schemas.SessionsSearchCountSchema.parse_obj(filter_d), project_id=project_id, user_id=None, count_only=True) + counts = sessions.search2_pg(data=schemas.SessionsSearchCountSchema.parse_obj(filter_d), project_id=project_id, + user_id=None, count_only=True) output[0]["sessionsCount"] = counts["countSessions"] output[0]["usersCount"] = counts["countUsers"] return output, 0 diff --git a/api/routers/core.py b/api/routers/core.py index 800a59765..90f6bf3e1 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -98,12 +98,11 @@ def comment_assignment(projectId: int, sessionId: int, issueId: str, data: schem @app.get('/{projectId}/events/search', tags=["events"]) -def events_search(projectId: int, q: str, type: str = None, key: str = None, source: str = None, - context: schemas.CurrentContext = Depends(OR_context)): +def events_search(projectId: int, q: str, type: Union[schemas.FilterType, schemas.EventType] = None, key: str = None, + source: str = None, context: schemas.CurrentContext = Depends(OR_context)): if len(q) == 0: return {"data": []} - result = events.search_pg2(text=q, event_type=type, project_id=projectId, source=source, - key=key) + result = events.search_pg2(text=q, event_type=type, project_id=projectId, source=source, key=key) return result diff --git a/api/schemas.py b/api/schemas.py index de69f413c..37118bd1d 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -399,6 +399,9 @@ class FilterType(str, Enum): metadata = "METADATA" issue = "ISSUE" events_count = "EVENTS_COUNT" + utm_source = "UTM_SOURCE" + utm_medium = "UTM_MEDIUM" + utm_campaign = "UTM_CAMPAIGN" class SearchEventOperator(str, Enum): diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.9.9/1.9.9.sql new file mode 100644 index 000000000..d427688f5 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.9.9/1.9.9.sql @@ -0,0 +1,4 @@ +ALTER TABLE sessions + ADD COLUMN utm_source Nullable(String), + ADD COLUMN utm_medium Nullable(String), + ADD COLUMN utm_campaign Nullable(String); diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql index 77d430d85..54901bdc4 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql @@ -1,22 +1,26 @@ -CREATE TABLE sessions ( - session_id UInt64, - project_id UInt32, - tracker_version String, - rev_id Nullable(String), - user_uuid UUID, - user_os String, - user_os_version Nullable(String), - user_browser String, - user_browser_version Nullable(String), - user_device Nullable(String), - user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), - user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), - datetime DateTime, - duration UInt32, - pages_count UInt16, - events_count UInt16, - errors_count UInt16 -) ENGINE = ReplacingMergeTree( duration ) -PARTITION BY toDate(datetime) -ORDER BY (project_id, datetime, session_id) -TTL datetime + INTERVAL 1 MONTH; +CREATE TABLE sessions +( + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + duration UInt32, + pages_count UInt16, + events_count UInt16, + errors_count UInt16, + utm_source Nullable(String), + utm_medium Nullable(String), + utm_campaign Nullable(String) +) ENGINE = ReplacingMergeTree(duration) + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 1 MONTH; diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 5f5b8f913..bac24d114 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -7,5 +7,13 @@ CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_idx ON events.pages (sessi CREATE INDEX ON events.errors (timestamp); CREATE INDEX ON public.projects (project_key); +ALTER TABLE sessions + ADD COLUMN utm_source text NULL DEFAULT NULL, + ADD COLUMN utm_medium text NULL DEFAULT NULL, + ADD COLUMN utm_campaign text NULL DEFAULT NULL; + +CREATE INDEX sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); +CREATE INDEX sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); +CREATE INDEX sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index e2e0af48f..516caa6ec 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -514,6 +514,9 @@ $$ watchdogs_score bigint NOT NULL DEFAULT 0, issue_score bigint NOT NULL DEFAULT 0, issue_types issue_type[] NOT NULL DEFAULT '{}'::issue_type[], + utm_source text NULL DEFAULT NULL, + utm_medium text NULL DEFAULT NULL, + utm_campaign text NULL DEFAULT NULL, metadata_1 text DEFAULT NULL, metadata_2 text DEFAULT NULL, metadata_3 text DEFAULT NULL, @@ -568,6 +571,9 @@ $$ CREATE INDEX sessions_session_id_project_id_start_ts_durationNN_idx ON sessions (session_id, project_id, start_ts) WHERE duration IS NOT NULL; CREATE INDEX sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; CREATE INDEX sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; + CREATE INDEX sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); + CREATE INDEX sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); + CREATE INDEX sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); ALTER TABLE public.sessions ADD CONSTRAINT web_browser_constraint CHECK (