From ae53f2b139b125a26dabc172cc52d5d800f988b1 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 19 Jul 2022 13:48:55 +0200 Subject: [PATCH] feat(chalice): ch sessions search --- ee/api/.gitignore | 1 - ee/api/chalicelib/core/sessions.py | 2105 ++++++++++++ ee/api/clean.sh | 1 - ee/api/routers/core_dynamic.py | 10 + .../db/init_dbs/clickhouse/1.8.0/1.8.0.sql | 258 ++ .../clickhouse/1.8.0/__clickhouse.csv | 68 + .../db/init_dbs/clickhouse/1.8.0/fill.sql | 2878 +++++++++++++++++ .../db/init_dbs/clickhouse/1.8.0/queries.sql | 983 ++++++ 8 files changed, 6302 insertions(+), 2 deletions(-) create mode 100644 ee/api/chalicelib/core/sessions.py create mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql create mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/__clickhouse.csv create mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql create mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/queries.sql diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 12a468ef1..a0ca015ed 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -205,7 +205,6 @@ Pipfile /chalicelib/core/log_tool_sumologic.py /chalicelib/core/metadata.py /chalicelib/core/mobile.py -/chalicelib/core/sessions.py /chalicelib/core/sessions_assignments.py /chalicelib/core/sessions_metas.py /chalicelib/core/sessions_mobs.py diff --git a/ee/api/chalicelib/core/sessions.py b/ee/api/chalicelib/core/sessions.py new file mode 100644 index 000000000..f2725188d --- /dev/null +++ b/ee/api/chalicelib/core/sessions.py @@ -0,0 +1,2105 @@ +from typing import List + +import schemas +import schemas_ee +from chalicelib.core import events, metadata, events_ios, \ + sessions_mobs, issues, projects, errors, resources, assist, performance_event +from chalicelib.utils import pg_client, helper, metrics_helper, ch_client + +SESSION_PROJECTION_COLS = """\ +s.project_id, +s.session_id::text AS session_id, +s.user_uuid, +s.user_id, +s.user_os, +s.user_browser, +s.user_device, +s.user_device_type, +s.user_country, +s.start_ts, +s.duration, +s.events_count, +s.pages_count, +s.errors_count, +s.user_anonymous_id, +s.platform, +s.issue_score, +to_jsonb(s.issue_types) AS issue_types, +favorite_sessions.session_id NOTNULL AS favorite, +COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + """ + +SESSION_PROJECTION_COLS_CH = """\ +s.project_id, +s.session_id AS session_id, +s.user_uuid AS user_uuid, +s.user_id AS user_id, +s.user_os AS user_os, +s.user_browser AS user_browser, +s.user_device AS user_device, +s.user_device_type AS user_device_type, +s.user_country AS user_country, +toUnixTimestamp(s.datetime)*1000 AS start_ts, +s.duration AS duration, +s.events_count AS events_count, +s.pages_count AS pages_count, +s.errors_count AS errors_count, +s.user_anonymous_id AS user_anonymous_id, +s.platform AS platform, +0 AS issue_score, +-- , +-- to_jsonb(s.issue_types) AS issue_types, +isNotNull(favorite_sessions.session_id) AS favorite, +-- COALESCE((SELECT TRUE +-- FROM public.user_viewed_sessions AS fs +-- WHERE s.session_id = fs.session_id +-- AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + """ + +SESSION_PROJECTION_COLS_CH_MAP = """\ +'project_id', toString(%(project_id)s), +'session_id', toString(s.session_id), +'user_uuid', toString(s.user_uuid), +'user_id', toString(s.user_id), +'user_os', toString(s.user_os), +'user_browser', toString(s.user_browser), +'user_device', toString(s.user_device), +'user_device_type', toString(s.user_device_type), +'user_country', toString(s.user_country), +'start_ts', toString(toUnixTimestamp(s.datetime)*1000), +'duration', toString(s.duration), +'events_count', toString(s.events_count), +'pages_count', toString(s.pages_count), +'errors_count', toString(s.errors_count), +'user_anonymous_id', toString(s.user_anonymous_id), +'platform', toString(s.platform), +'issue_score', '0', +'favorite', toString(isNotNull(favorite_sessions.session_id)) +""" + + +def __group_metadata(session, project_metadata): + meta = {} + for m in project_metadata.keys(): + if project_metadata[m] is not None and session.get(m) is not None: + meta[project_metadata[m]] = session[m] + session.pop(m) + return meta + + +def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_viewed=False, group_metadata=False, + live=True): + with pg_client.PostgresClient() as cur: + extra_query = [] + if include_fav_viewed: + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite""") + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS viewed""") + query = cur.mogrify( + f"""\ + SELECT + s.*, + s.session_id::text AS session_id, + (SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key + {"," if len(extra_query) > 0 else ""}{",".join(extra_query)} + {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata._get_column_names()]) + ") AS project_metadata") if group_metadata else ''} + FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""} + WHERE s.project_id = %(project_id)s + AND s.session_id = %(session_id)s;""", + {"project_id": project_id, "session_id": session_id, "userId": user_id} + ) + # print("===============") + # print(query) + cur.execute(query=query) + + data = cur.fetchone() + if data is not None: + data = helper.dict_to_camel_case(data) + if full_data: + if data["platform"] == 'ios': + data['events'] = events_ios.get_by_sessionId(project_id=project_id, session_id=session_id) + for e in data['events']: + if e["type"].endswith("_IOS"): + e["type"] = e["type"][:-len("_IOS")] + data['crashes'] = events_ios.get_crashes_by_session_id(session_id=session_id) + data['userEvents'] = events_ios.get_customs_by_sessionId(project_id=project_id, + session_id=session_id) + data['mobsUrl'] = sessions_mobs.get_ios(sessionId=session_id) + else: + data['events'] = events.get_by_sessionId2_pg(project_id=project_id, session_id=session_id, + group_clickrage=True) + all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) + data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] + # to keep only the first stack + data['errors'] = [errors.format_first_stack_frame(e) for e in all_errors if + e['source'] == "js_exception"][ + :500] # limit the number of errors to reduce the response-body size + data['userEvents'] = events.get_customs_by_sessionId2_pg(project_id=project_id, + session_id=session_id) + data['mobsUrl'] = sessions_mobs.get_web(sessionId=session_id) + data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, + start_ts=data["startTs"], + duration=data["duration"]) + + data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) + data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) + data['live'] = live and assist.is_live(project_id=project_id, + session_id=session_id, + project_key=data["projectKey"]) + data["inDB"] = True + return data + elif live: + return assist.get_live_session_by_id(project_id=project_id, session_id=session_id) + else: + return None + + +def __get_sql_operator(op: schemas.SearchEventOperator): + return { + schemas.SearchEventOperator._is: "=", + schemas.SearchEventOperator._is_any: "IN", + schemas.SearchEventOperator._on: "=", + schemas.SearchEventOperator._on_any: "IN", + schemas.SearchEventOperator._is_not: "!=", + schemas.SearchEventOperator._not_on: "!=", + schemas.SearchEventOperator._contains: "ILIKE", + schemas.SearchEventOperator._not_contains: "NOT ILIKE", + schemas.SearchEventOperator._starts_with: "ILIKE", + schemas.SearchEventOperator._ends_with: "ILIKE", + }.get(op, "=") + + +def __is_negation_operator(op: schemas.SearchEventOperator): + return op in [schemas.SearchEventOperator._is_not, + schemas.SearchEventOperator._not_on, + schemas.SearchEventOperator._not_contains] + + +def __reverse_sql_operator(op): + return "=" if op == "!=" else "!=" if op == "=" else "ILIKE" if op == "NOT ILIKE" else "NOT ILIKE" + + +def __get_sql_operator_multiple(op: schemas.SearchEventOperator): + return " IN " if op not in [schemas.SearchEventOperator._is_not, schemas.SearchEventOperator._not_on, + schemas.SearchEventOperator._not_contains] else " NOT IN " + + +def __get_sql_value_multiple(values): + if isinstance(values, tuple): + return values + return tuple(values) if isinstance(values, list) else (values,) + + +def _multiple_conditions(condition, values, value_key="value", is_not=False): + query = [] + for i in range(len(values)): + k = f"{value_key}_{i}" + query.append(condition.replace(value_key, k)) + return "(" + (" AND " if is_not else " OR ").join(query) + ")" + + +def _multiple_values(values, value_key="value"): + query_values = {} + if values is not None and isinstance(values, list): + for i in range(len(values)): + k = f"{value_key}_{i}" + query_values[k] = values[i] + return query_values + + +def _isAny_opreator(op: schemas.SearchEventOperator): + return op in [schemas.SearchEventOperator._on_any, schemas.SearchEventOperator._is_any] + + +def _isUndefined_operator(op: schemas.SearchEventOperator): + return op in [schemas.SearchEventOperator._is_undefined] + + +def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, + error_status=schemas.ErrorStatus.all, count_only=False, issue=None): + full_args, query_part = search_query_parts(data=data, error_status=error_status, errors_only=errors_only, + favorite_only=data.bookmarked, issue=issue, project_id=project_id, + user_id=user_id) + if data.limit is not None and data.page is not None: + full_args["sessions_limit_s"] = (data.page - 1) * data.limit + full_args["sessions_limit_e"] = data.page * data.limit + else: + full_args["sessions_limit_s"] = 1 + full_args["sessions_limit_e"] = 200 + + meta_keys = [] + with pg_client.PostgresClient() as cur: + if errors_only: + main_query = cur.mogrify(f"""SELECT DISTINCT er.error_id, ser.status, ser.parent_error_id, ser.payload, + COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite, + COALESCE((SELECT TRUE + FROM public.user_viewed_errors AS ve + WHERE er.error_id = ve.error_id + AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + {query_part};""", full_args) + + elif count_only: + main_query = cur.mogrify(f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, + COUNT(DISTINCT s.user_uuid) AS count_users + {query_part};""", full_args) + elif data.group_by_user: + g_sort = "count(full_sessions)" + if data.order is None: + data.order = schemas.SortOrderType.desc + else: + data.order = data.order.upper() + if data.sort is not None and data.sort != 'sessionsCount': + sort = helper.key_to_snake_case(data.sort) + g_sort = f"{'MIN' if data.order == schemas.SortOrderType.desc else 'MAX'}({sort})" + else: + sort = 'start_ts' + + meta_keys = metadata.get(project_id=project_id) + main_query = cur.mogrify(f"""SELECT COUNT(*) AS count, + COALESCE(JSONB_AGG(users_sessions) + FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions + FROM (SELECT user_id, + count(full_sessions) AS user_sessions_count, + jsonb_agg(full_sessions) FILTER (WHERE rn <= 1) AS last_session, + MIN(full_sessions.start_ts) AS first_session_ts, + ROW_NUMBER() OVER (ORDER BY {g_sort} {data.order}) AS rn + FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY {sort} {data.order}) AS rn + FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} + {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} + {query_part} + ) AS filtred_sessions + ) AS full_sessions + GROUP BY user_id + ) AS users_sessions;""", + full_args) + else: + if data.order is None: + data.order = schemas.SortOrderType.desc + sort = 'session_id' + if data.sort is not None and data.sort != "session_id": + # sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) + sort = helper.key_to_snake_case(data.sort) + + meta_keys = metadata.get(project_id=project_id) + main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, + COALESCE(JSONB_AGG(full_sessions) + FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions + FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY {sort} {data.order}, issue_score DESC) AS rn + FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} + {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} + {query_part} + ORDER BY s.session_id desc) AS filtred_sessions + ORDER BY {sort} {data.order}, issue_score DESC) AS full_sessions;""", + full_args) + # print("--------------------") + # print(main_query) + # print("--------------------") + try: + cur.execute(main_query) + except Exception as err: + print("--------- SESSIONS SEARCH QUERY EXCEPTION -----------") + print(main_query.decode('UTF-8')) + print("--------- PAYLOAD -----------") + print(data.json()) + print("--------------------") + raise err + if errors_only: + return helper.list_to_camel_case(cur.fetchall()) + + sessions = cur.fetchone() + if count_only: + return helper.dict_to_camel_case(sessions) + + total = sessions["count"] + sessions = sessions["sessions"] + + if data.group_by_user: + for i, s in enumerate(sessions): + sessions[i] = {**s.pop("last_session")[0], **s} + sessions[i].pop("rn") + sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ + if sessions[i][f'metadata_{k["index"]}'] is not None} + else: + for i, s in enumerate(sessions): + sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ + if sessions[i][f'metadata_{k["index"]}'] is not None} + # if not data.group_by_user and data.sort is not None and data.sort != "session_id": + # sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data.sort)], + # reverse=data.order.upper() == "DESC") + return { + 'total': total, + 'sessions': helper.list_to_camel_case(sessions) + } + + +def search2_ch(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, + error_status=schemas.ErrorStatus.all, count_only=False, issue=None): + full_args, query_part = search_query_parts_ch(data=data, error_status=error_status, errors_only=errors_only, + favorite_only=data.bookmarked, issue=issue, project_id=project_id, + user_id=user_id) + if data.sort == "startTs": + data.sort = "datetime" + if data.limit is not None and data.page is not None: + full_args["sessions_limit_s"] = (data.page - 1) * data.limit + full_args["sessions_limit_e"] = data.page * data.limit + full_args["sessions_limit"] = data.limit + else: + full_args["sessions_limit_s"] = 1 + full_args["sessions_limit_e"] = 200 + full_args["sessions_limit"] = 200 + + meta_keys = [] + with ch_client.ClickHouseClient() as cur: + if errors_only: + main_query = cur.mogrify(f"""SELECT DISTINCT er.error_id, ser.status, ser.parent_error_id, ser.payload, + COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite, + COALESCE((SELECT TRUE + FROM public.user_viewed_errors AS ve + WHERE er.error_id = ve.error_id + AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + {query_part};""", full_args) + + elif count_only: + main_query = cur.mogrify(f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, + COUNT(DISTINCT s.user_uuid) AS count_users + {query_part};""", full_args) + elif data.group_by_user: + g_sort = "count(full_sessions)" + if data.order is None: + data.order = schemas.SortOrderType.desc + else: + data.order = data.order.upper() + if data.sort is not None and data.sort != 'sessionsCount': + sort = helper.key_to_snake_case(data.sort) + g_sort = f"{'MIN' if data.order == schemas.SortOrderType.desc else 'MAX'}({sort})" + else: + sort = 'start_ts' + + meta_keys = metadata.get(project_id=project_id) + main_query = cur.mogrify(f"""SELECT COUNT(*) AS count, + COALESCE(JSONB_AGG(users_sessions) + FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions + FROM (SELECT user_id, + count(full_sessions) AS user_sessions_count, + jsonb_agg(full_sessions) FILTER (WHERE rn <= 1) AS last_session, + MIN(full_sessions.start_ts) AS first_session_ts, + ROW_NUMBER() OVER (ORDER BY {g_sort} {data.order}) AS rn + FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY {sort} {data.order}) AS rn + FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} + {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} + {query_part} + ) AS filtred_sessions + ) AS full_sessions + GROUP BY user_id + ) AS users_sessions;""", + full_args) + else: + if data.order is None: + data.order = schemas.SortOrderType.desc + sort = 'session_id' + if data.sort is not None and data.sort != "session_id": + # sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) + sort = helper.key_to_snake_case(data.sort) + + meta_keys = metadata.get(project_id=project_id) + main_query = cur.format(f"""SELECT any(total) AS count, groupArray(%(sessions_limit)s)(details) AS sessions + FROM (SELECT COUNT() OVER () AS total, + rowNumberInAllBlocks() AS rn, + map({SESSION_PROJECTION_COLS_CH_MAP}) AS details + {query_part} +-- ORDER BY {sort} {data.order} + ) AS raw + WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s;""", full_args) + print("--------------------") + print(main_query) + print("--------------------") + try: + sessions = cur.execute(main_query) + except Exception as err: + print("--------- SESSIONS SEARCH QUERY EXCEPTION -----------") + print(main_query) + print("--------- PAYLOAD -----------") + print(data.json()) + print("--------------------") + raise err + if errors_only: + return helper.list_to_camel_case(cur.fetchall()) + + if len(sessions) > 0: + sessions = sessions[0] + # if count_only: + # return helper.dict_to_camel_case(sessions) + # for s in sessions: + # print(s) + # s["session_id"] = str(s["session_id"]) + total = sessions["count"] + sessions = sessions["sessions"] + + if data.group_by_user: + for i, s in enumerate(sessions): + sessions[i] = {**s.pop("last_session")[0], **s} + sessions[i].pop("rn") + sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ + if sessions[i][f'metadata_{k["index"]}'] is not None} + else: + for i in range(len(sessions)): + sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ + if sessions[i].get(f'metadata_{k["index"]}') is not None} + sessions[i] = schemas_ee.SessionModel.parse_obj(helper.dict_to_camel_case(sessions[i])) + + # if not data.group_by_user and data.sort is not None and data.sort != "session_id": + # sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data.sort)], + # reverse=data.order.upper() == "DESC") + return { + 'total': total, + 'sessions': sessions + } + + +def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, density: int, + view_type: schemas.MetricTimeseriesViewType, metric_type: schemas.MetricType, + metric_of: schemas.TableMetricOfType, metric_value: List): + step_size = int(metrics_helper.__get_step_size(endTimestamp=data.endDate, startTimestamp=data.startDate, + density=density, factor=1, decimal=True)) + extra_event = None + if metric_of == schemas.TableMetricOfType.visited_url: + extra_event = "events.pages" + elif metric_of == schemas.TableMetricOfType.issues and len(metric_value) > 0: + data.filters.append(schemas.SessionSearchFilterSchema(value=metric_value, type=schemas.FilterType.issue, + operator=schemas.SearchEventOperator._is)) + full_args, query_part = search_query_parts(data=data, error_status=None, errors_only=False, + favorite_only=False, issue=None, project_id=project_id, + user_id=None, extra_event=extra_event) + full_args["step_size"] = step_size + sessions = [] + with pg_client.PostgresClient() as cur: + if metric_type == schemas.MetricType.timeseries: + if view_type == schemas.MetricTimeseriesViewType.line_chart: + main_query = cur.mogrify(f"""WITH full_sessions AS (SELECT DISTINCT ON(s.session_id) s.session_id, s.start_ts + {query_part}) + SELECT generated_timestamp AS timestamp, + COUNT(s) AS count + FROM generate_series(%(startDate)s, %(endDate)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT 1 AS s + FROM full_sessions + WHERE start_ts >= generated_timestamp + AND start_ts <= generated_timestamp + %(step_size)s) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""", full_args) + else: + main_query = cur.mogrify(f"""SELECT count(DISTINCT s.session_id) AS count + {query_part};""", full_args) + + # print("--------------------") + # print(main_query) + # print("--------------------") + cur.execute(main_query) + if view_type == schemas.MetricTimeseriesViewType.line_chart: + sessions = cur.fetchall() + else: + sessions = cur.fetchone()["count"] + elif metric_type == schemas.MetricType.table: + if isinstance(metric_of, schemas.TableMetricOfType): + main_col = "user_id" + extra_col = "" + extra_where = "" + pre_query = "" + if metric_of == schemas.TableMetricOfType.user_country: + main_col = "user_country" + elif metric_of == schemas.TableMetricOfType.user_device: + main_col = "user_device" + elif metric_of == schemas.TableMetricOfType.user_browser: + main_col = "user_browser" + elif metric_of == schemas.TableMetricOfType.issues: + main_col = "issue" + extra_col = f", UNNEST(s.issue_types) AS {main_col}" + if len(metric_value) > 0: + extra_where = [] + for i in range(len(metric_value)): + arg_name = f"selected_issue_{i}" + extra_where.append(f"{main_col} = %({arg_name})s") + full_args[arg_name] = metric_value[i] + extra_where = f"WHERE ({' OR '.join(extra_where)})" + elif metric_of == schemas.TableMetricOfType.visited_url: + main_col = "path" + extra_col = ", path" + main_query = cur.mogrify(f"""{pre_query} + SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS values + FROM (SELECT {main_col} AS name, + count(full_sessions) AS session_count, + ROW_NUMBER() OVER (ORDER BY count(full_sessions) DESC) AS rn + FROM (SELECT * + FROM (SELECT DISTINCT ON(s.session_id) s.session_id, s.user_uuid, + s.user_id, s.user_os, + s.user_browser, s.user_device, + s.user_device_type, s.user_country, s.issue_types{extra_col} + {query_part} + ORDER BY s.session_id desc) AS filtred_sessions + ) AS full_sessions + {extra_where} + GROUP BY {main_col} + ORDER BY session_count DESC) AS users_sessions;""", + full_args) + # print("--------------------") + # print(main_query) + # print("--------------------") + cur.execute(main_query) + sessions = cur.fetchone() + for s in sessions["values"]: + s.pop("rn") + sessions["values"] = helper.list_to_camel_case(sessions["values"]) + + return sessions + + +def __is_valid_event(is_any: bool, event: schemas._SessionSearchEventSchema): + return not (not is_any and len(event.value) == 0 and event.type not in [schemas.EventType.request_details, + schemas.EventType.graphql_details] \ + or event.type in [schemas.PerformanceEventType.location_dom_complete, + schemas.PerformanceEventType.location_largest_contentful_paint_time, + schemas.PerformanceEventType.location_ttfb, + schemas.PerformanceEventType.location_avg_cpu_load, + schemas.PerformanceEventType.location_avg_memory_usage + ] and (event.source is None or len(event.source) == 0) \ + or event.type in [schemas.EventType.request_details, schemas.EventType.graphql_details] and ( + event.filters is None or len(event.filters) == 0)) + + +def search_query_parts(data, error_status, errors_only, favorite_only, issue, project_id, user_id, extra_event=None): + ss_constraints = [] + full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, + "projectId": project_id, "userId": user_id} + extra_constraints = [ + "s.project_id = %(project_id)s", + "s.duration IS NOT NULL" + ] + extra_from = "" + events_query_part = "" + if len(data.filters) > 0: + meta_keys = None + for i, f in enumerate(data.filters): + if not isinstance(f.value, list): + f.value = [f.value] + filter_type = f.type + f.value = helper.values_for_operator(value=f.value, op=f.operator) + f_k = f"f_value{i}" + full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} + op = __get_sql_operator(f.operator) \ + if filter_type not in [schemas.FilterType.events_count] else f.operator + is_any = _isAny_opreator(f.operator) + is_undefined = _isUndefined_operator(f.operator) + if not is_any and not is_undefined and len(f.value) == 0: + continue + is_not = False + if __is_negation_operator(f.operator): + is_not = True + if filter_type == schemas.FilterType.user_browser: + if is_any: + extra_constraints.append('s.user_browser IS NOT NULL') + ss_constraints.append('ms.user_browser IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: + if is_any: + extra_constraints.append('s.user_os IS NOT NULL') + ss_constraints.append('ms.user_os IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: + if is_any: + extra_constraints.append('s.user_device IS NOT NULL') + ss_constraints.append('ms.user_device IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: + if is_any: + extra_constraints.append('s.user_country IS NOT NULL') + ss_constraints.append('ms.user_country IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.utm_source]: + if is_any: + extra_constraints.append('s.utm_source IS NOT NULL') + ss_constraints.append('ms.utm_source IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.utm_source IS NULL') + ss_constraints.append('ms.utm_source IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_medium]: + if is_any: + extra_constraints.append('s.utm_medium IS NOT NULL') + ss_constraints.append('ms.utm_medium IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.utm_medium IS NULL') + ss_constraints.append('ms.utm_medium IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_campaign]: + if is_any: + extra_constraints.append('s.utm_campaign IS NOT NULL') + ss_constraints.append('ms.utm_campaign IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.utm_campaign IS NULL') + ss_constraints.append('ms.utm_campaign IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + + elif filter_type == schemas.FilterType.duration: + if len(f.value) > 0 and f.value[0] is not None: + extra_constraints.append("s.duration >= %(minDuration)s") + ss_constraints.append("ms.duration >= %(minDuration)s") + full_args["minDuration"] = f.value[0] + if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: + extra_constraints.append("s.duration <= %(maxDuration)s") + ss_constraints.append("ms.duration <= %(maxDuration)s") + full_args["maxDuration"] = f.value[1] + elif filter_type == schemas.FilterType.referrer: + extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" + if is_any: + extra_constraints.append('p.base_referrer IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f"p.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + elif filter_type == events.event_type.METADATA.ui_type: + # get metadata list only if you need it + if meta_keys is None: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + if f.source in meta_keys.keys(): + if is_any: + extra_constraints.append(f"s.{metadata.index_to_colname(meta_keys[f.source])} IS NOT NULL") + ss_constraints.append(f"ms.{metadata.index_to_colname(meta_keys[f.source])} IS NOT NULL") + elif is_undefined: + extra_constraints.append(f"s.{metadata.index_to_colname(meta_keys[f.source])} IS NULL") + ss_constraints.append(f"ms.{metadata.index_to_colname(meta_keys[f.source])} IS NULL") + else: + extra_constraints.append( + _multiple_conditions( + f"s.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text", + f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions( + f"ms.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text", + f.value, is_not=is_not, value_key=f_k)) + elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: + if is_any: + extra_constraints.append('s.user_id IS NOT NULL') + ss_constraints.append('ms.user_id IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.user_id IS NULL') + ss_constraints.append('ms.user_id IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f"s.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + elif filter_type in [schemas.FilterType.user_anonymous_id, + schemas.FilterType.user_anonymous_id_ios]: + if is_any: + extra_constraints.append('s.user_anonymous_id IS NOT NULL') + ss_constraints.append('ms.user_anonymous_id IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.user_anonymous_id IS NULL') + ss_constraints.append('ms.user_anonymous_id IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: + if is_any: + extra_constraints.append('s.rev_id IS NOT NULL') + ss_constraints.append('ms.rev_id IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.rev_id IS NULL') + ss_constraints.append('ms.rev_id IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f"s.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + elif filter_type == schemas.FilterType.platform: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.issue: + if is_any: + extra_constraints.append("array_length(s.issue_types, 1) > 0") + ss_constraints.append("array_length(ms.issue_types, 1) > 0") + else: + extra_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.events_count: + extra_constraints.append( + _multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + # --------------------------------------------------------------------------- + if len(data.events) > 0: + valid_events_count = 0 + for event in data.events: + is_any = _isAny_opreator(event.operator) + if not isinstance(event.value, list): + event.value = [event.value] + if __is_valid_event(is_any=is_any, event=event): + valid_events_count += 1 + events_query_from = [] + event_index = 0 + or_events = data.events_order == schemas.SearchEventOrder._or + # events_joiner = " FULL JOIN " if or_events else " INNER JOIN LATERAL " + events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " + for i, event in enumerate(data.events): + event_type = event.type + is_any = _isAny_opreator(event.operator) + if not isinstance(event.value, list): + event.value = [event.value] + if not __is_valid_event(is_any=is_any, event=event): + continue + op = __get_sql_operator(event.operator) + is_not = False + if __is_negation_operator(event.operator): + is_not = True + op = __reverse_sql_operator(op) + if event_index == 0 or or_events: + event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" + event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", + "main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s", + "ms.start_ts <= %(endDate)s", "ms.duration IS NOT NULL"] + if favorite_only and not errors_only: + event_from += "INNER JOIN public.user_favorite_sessions AS fs USING(session_id)" + event_where.append("fs.user_id = %(userId)s") + else: + event_from = "%s" + event_where = ["main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", + "main.session_id=event_0.session_id"] + if data.events_order == schemas.SearchEventOrder._then: + event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") + e_k = f"e_value{i}" + s_k = e_k + "_source" + if event.type != schemas.PerformanceEventType.time_between_events: + event.value = helper.values_for_operator(value=event.value, op=event.operator) + full_args = {**full_args, + **_multiple_values(event.value, value_key=e_k), + **_multiple_values(event.source, value_key=s_k)} + + if event_type == events.event_type.CLICK.ui_type: + event_from = event_from % f"{events.event_type.CLICK.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, + value_key=e_k)) + + elif event_type == events.event_type.INPUT.ui_type: + event_from = event_from % f"{events.event_type.INPUT.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, + value_key=e_k)) + if event.source is not None and len(event.source) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, + value_key=f"custom{i}")) + full_args = {**full_args, **_multiple_values(event.source, value_key=f"custom{i}")} + + elif event_type == events.event_type.LOCATION.ui_type: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.CUSTOM.ui_type: + event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.REQUEST.ui_type: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.GRAPHQL.ui_type: + event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.STATEACTION.ui_type: + event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.ERROR.ui_type: + event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" + event.source = tuple(event.source) + if not is_any and event.value not in [None, "*", ""]: + event_where.append( + _multiple_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) + if event.source[0] not in [None, "*", ""]: + event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.value, value_key=s_k)) + + + # ----- IOS + elif event_type == events.event_type.CLICK_IOS.ui_type: + event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + + elif event_type == events.event_type.INPUT_IOS.ui_type: + event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + if event.source is not None and len(event.source) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, + value_key="custom{i}")) + full_args = {**full_args, **_multiple_values(event.source, f"custom{i}")} + elif event_type == events.event_type.VIEW_IOS.ui_type: + event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.CUSTOM_IOS.ui_type: + event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.REQUEST_IOS.ui_type: + event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.ERROR_IOS.ui_type: + event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" + if not is_any and event.value not in [None, "*", ""]: + event_where.append( + _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.fetch_failed: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + event.value, value_key=e_k)) + col = performance_event.get_col(event_type) + colname = col["column"] + event_where.append(f"main.{colname} = FALSE") + # elif event_type == schemas.PerformanceEventType.fetch_duration: + # event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + # if not is_any: + # event_where.append( + # _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + # event.value, value_key=e_k)) + # col = performance_event.get_col(event_type) + # colname = col["column"] + # tname = "main" + # e_k += "_custom" + # full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + # _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + # event.source, value_key=e_k)) + elif event_type in [schemas.PerformanceEventType.location_dom_complete, + schemas.PerformanceEventType.location_largest_contentful_paint_time, + schemas.PerformanceEventType.location_ttfb, + schemas.PerformanceEventType.location_avg_cpu_load, + schemas.PerformanceEventType.location_avg_memory_usage + ]: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + col = performance_event.get_col(event_type) + colname = col["column"] + tname = "main" + if col.get("extraJoin") is not None: + tname = "ej" + event_from += f" INNER JOIN {col['extraJoin']} AS {tname} USING(session_id)" + event_where += [f"{tname}.timestamp >= main.timestamp", f"{tname}.timestamp >= %(startDate)s", + f"{tname}.timestamp <= %(endDate)s"] + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + + event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.time_between_events: + event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " + if not isinstance(event.value[0].value, list): + event.value[0].value = [event.value[0].value] + if not isinstance(event.value[1].value, list): + event.value[1].value = [event.value[1].value] + event.value[0].value = helper.values_for_operator(value=event.value[0].value, + op=event.value[0].operator) + event.value[1].value = helper.values_for_operator(value=event.value[1].value, + op=event.value[0].operator) + e_k1 = e_k + "_e1" + e_k2 = e_k + "_e2" + full_args = {**full_args, + **_multiple_values(event.value[0].value, value_key=e_k1), + **_multiple_values(event.value[1].value, value_key=e_k2)} + s_op = __get_sql_operator(event.value[0].operator) + event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] + if event_index > 0 and not or_events: + event_where.append("main2.session_id=event_0.session_id") + is_any = _isAny_opreator(event.value[0].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", + event.value[0].value, value_key=e_k1)) + s_op = __get_sql_operator(event.value[1].operator) + is_any = _isAny_opreator(event.value[1].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", + event.value[1].value, value_key=e_k2)) + + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + event_where.append( + _multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) + + elif event_type == schemas.EventType.request_details: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + apply = False + for j, f in enumerate(event.filters): + is_any = _isAny_opreator(f.operator) + if is_any or len(f.value) == 0: + continue + f.value = helper.values_for_operator(value=f.value, op=f.operator) + op = __get_sql_operator(f.operator) + e_k_f = e_k + f"_fetch{j}" + full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + if f.type == schemas.FetchFilterType._url: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s::text", + f.value, value_key=e_k_f)) + apply = True + elif f.type == schemas.FetchFilterType._status_code: + event_where.append( + _multiple_conditions(f"main.status_code {f.operator} %({e_k_f})s::integer", f.value, + value_key=e_k_f)) + apply = True + elif f.type == schemas.FetchFilterType._method: + event_where.append( + _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + apply = True + elif f.type == schemas.FetchFilterType._duration: + event_where.append( + _multiple_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value, + value_key=e_k_f)) + apply = True + elif f.type == schemas.FetchFilterType._request_body: + event_where.append( + _multiple_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value, value_key=e_k_f)) + apply = True + elif f.type == schemas.FetchFilterType._response_body: + event_where.append( + _multiple_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value, + value_key=e_k_f)) + apply = True + else: + print(f"undefined FETCH filter: {f.type}") + if not apply: + continue + elif event_type == schemas.EventType.graphql_details: + event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " + for j, f in enumerate(event.filters): + is_any = _isAny_opreator(f.operator) + if is_any or len(f.value) == 0: + continue + f.value = helper.values_for_operator(value=f.value, op=f.operator) + op = __get_sql_operator(f.operator) + e_k_f = e_k + f"_graphql{j}" + full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + if f.type == schemas.GraphqlFilterType._name: + event_where.append( + _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value, + value_key=e_k_f)) + elif f.type == schemas.GraphqlFilterType._method: + event_where.append( + _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + elif f.type == schemas.GraphqlFilterType._request_body: + event_where.append( + _multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + elif f.type == schemas.GraphqlFilterType._response_body: + event_where.append( + _multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + else: + print(f"undefined GRAPHQL filter: {f.type}") + else: + continue + if event_index == 0 or or_events: + event_where += ss_constraints + if is_not: + if event_index == 0 or or_events: + events_query_from.append(f"""\ + (SELECT + session_id, + 0 AS timestamp + FROM sessions + WHERE EXISTS(SELECT session_id + FROM {event_from} + WHERE {" AND ".join(event_where)} + AND sessions.session_id=ms.session_id) IS FALSE + AND project_id = %(projectId)s + AND start_ts >= %(startDate)s + AND start_ts <= %(endDate)s + AND duration IS NOT NULL + ) {"" if or_events else (f"AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + else: + events_query_from.append(f"""\ + (SELECT + event_0.session_id, + event_{event_index - 1}.timestamp AS timestamp + WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE + ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ + """) + else: + events_query_from.append(f"""\ + (SELECT main.session_id, {"MIN" if event_index < (valid_events_count - 1) else "MAX"}(main.timestamp) AS timestamp + FROM {event_from} + WHERE {" AND ".join(event_where)} + GROUP BY 1 + ) {"" if or_events else (f"AS event_{event_index} " + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + event_index += 1 + if event_index > 0: + if or_events: + events_query_part = f"""SELECT + session_id, + MIN(timestamp) AS first_event_ts, + MAX(timestamp) AS last_event_ts + FROM ({events_joiner.join(events_query_from)}) AS u + GROUP BY 1""" + else: + events_query_part = f"""SELECT + event_0.session_id, + MIN(event_0.timestamp) AS first_event_ts, + MAX(event_{event_index - 1}.timestamp) AS last_event_ts + FROM {events_joiner.join(events_query_from)} + GROUP BY 1""" + else: + data.events = [] + # --------------------------------------------------------------------------- + if data.startDate is not None: + extra_constraints.append("s.start_ts >= %(startDate)s") + if data.endDate is not None: + extra_constraints.append("s.start_ts <= %(endDate)s") + # if data.platform is not None: + # if data.platform == schemas.PlatformType.mobile: + # extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") + # elif data.platform == schemas.PlatformType.desktop: + # extra_constraints.append( + # b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") + + if errors_only: + extra_from += f" INNER JOIN {events.event_type.ERROR.table} AS er USING (session_id) INNER JOIN public.errors AS ser USING (error_id)" + extra_constraints.append("ser.source = 'js_exception'") + extra_constraints.append("ser.project_id = %(project_id)s") + if error_status != schemas.ErrorStatus.all: + extra_constraints.append("ser.status = %(error_status)s") + full_args["error_status"] = error_status + if favorite_only: + extra_from += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" + extra_constraints.append("ufe.user_id = %(userId)s") + # extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] + if favorite_only and not errors_only and user_id is not None: + extra_from += """INNER JOIN (SELECT user_id, session_id + FROM public.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + USING (session_id)""" + elif not favorite_only and not errors_only and user_id is not None: + extra_from += """LEFT JOIN (SELECT user_id, session_id + FROM public.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + USING (session_id)""" + extra_join = "" + if issue is not None: + extra_join = """ + INNER JOIN LATERAL(SELECT TRUE FROM events_common.issues INNER JOIN public.issues AS p_issues USING (issue_id) + WHERE issues.session_id=f.session_id + AND p_issues.type=%(issue_type)s + AND p_issues.context_string=%(issue_contextString)s + AND timestamp >= f.first_event_ts + AND timestamp <= f.last_event_ts) AS issues ON(TRUE) + """ + full_args["issue_contextString"] = issue["contextString"] + full_args["issue_type"] = issue["type"] + if extra_event: + extra_join += f"""INNER JOIN {extra_event} AS ev USING(session_id)""" + extra_constraints.append("ev.timestamp>=%(startDate)s") + extra_constraints.append("ev.timestamp<=%(endDate)s") + query_part = f"""\ + FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else "public.sessions AS s"} + {extra_join} + {"INNER JOIN public.sessions AS s USING(session_id)" if len(events_query_part) > 0 else ""} + {extra_from} + WHERE + {" AND ".join(extra_constraints)}""" + return full_args, query_part + + +def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, project_id, user_id, extra_event=None): + print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>") + ss_constraints = [] + full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, + "projectId": project_id, "userId": user_id} + extra_constraints = [ + "s.project_id = %(project_id)s", + "isNotNull(s.duration)" + ] + if favorite_only: + extra_constraints.append("""s.session_id IN (SELECT session_id + FROM final.user_favorite_sessions + WHERE user_id = %(userId)s)""") + extra_from = "" + events_query_part = "" + __events_where_basic = ["project_id = %(projectId)s", + "datetime >= toDateTime(%(startDate)s/1000)", + "datetime <= toDateTime(%(endDate)s/1000)"] + events_conditions_where = ["main.project_id = %(projectId)s", + "main.datetime >= toDateTime(%(startDate)s/1000)", + "main.datetime <= toDateTime(%(endDate)s/1000)"] + if len(data.filters) > 0: + meta_keys = None + # to reduce include a sub-query of sessions inside events query, in order to reduce the selected data + include_in_events = False + for i, f in enumerate(data.filters): + if not isinstance(f.value, list): + f.value = [f.value] + filter_type = f.type + f.value = helper.values_for_operator(value=f.value, op=f.operator) + f_k = f"f_value{i}" + full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} + op = __get_sql_operator(f.operator) \ + if filter_type not in [schemas.FilterType.events_count] else f.operator + is_any = _isAny_opreator(f.operator) + is_undefined = _isUndefined_operator(f.operator) + if not is_any and not is_undefined and len(f.value) == 0: + continue + is_not = False + if __is_negation_operator(f.operator): + is_not = True + if filter_type == schemas.FilterType.user_browser: + if is_any: + extra_constraints.append('isNotNull(s.user_browser)') + ss_constraints.append('isNotNull(ms.user_browser)') + else: + extra_constraints.append( + _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: + if is_any: + extra_constraints.append('isNotNull(s.user_os)') + ss_constraints.append('isNotNull(ms.user_os)') + else: + extra_constraints.append( + _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: + if is_any: + extra_constraints.append('isNotNull(s.user_device)') + ss_constraints.append('isNotNull(ms.user_device)') + else: + extra_constraints.append( + _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: + if is_any: + extra_constraints.append('isNotNull(s.user_country)') + ss_constraints.append('isNotNull(ms.user_country)') + else: + extra_constraints.append( + _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.utm_source]: + if is_any: + extra_constraints.append('isNotNull(s.utm_source)') + ss_constraints.append('isNotNull(ms.utm_source)') + elif is_undefined: + extra_constraints.append('isNull(s.utm_source)') + ss_constraints.append('isNull(ms.utm_source)') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_source {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_source {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_medium]: + if is_any: + extra_constraints.append('isNotNull(s.utm_medium)') + ss_constraints.append('isNotNull(ms.utm_medium)') + elif is_undefined: + extra_constraints.append('isNull(s.utm_medium)') + ss_constraints.append('isNull(ms.utm_medium') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_medium {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_medium {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_campaign]: + if is_any: + extra_constraints.append('isNotNull(s.utm_campaign)') + ss_constraints.append('isNotNull(ms.utm_campaign)') + elif is_undefined: + extra_constraints.append('isNull(s.utm_campaign)') + ss_constraints.append('isNull(ms.utm_campaign)') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_campaign {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_campaign {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + + elif filter_type == schemas.FilterType.duration: + if len(f.value) > 0 and f.value[0] is not None: + extra_constraints.append("s.duration >= %(minDuration)s") + ss_constraints.append("ms.duration >= %(minDuration)s") + full_args["minDuration"] = f.value[0] + if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: + extra_constraints.append("s.duration <= %(maxDuration)s") + ss_constraints.append("ms.duration <= %(maxDuration)s") + full_args["maxDuration"] = f.value[1] + elif filter_type == schemas.FilterType.referrer: + # extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" + if is_any: + referrer_constraint = 'isNotNull(r.base_referrer)' + else: + referrer_constraint = _multiple_conditions(f"r.base_referrer {op} %({f_k})s", f.value, + is_not=is_not, value_key=f_k) + referrer_constraint = f"""(SELECT DISTINCT session_id + FROM final.events AS r + WHERE {" AND ".join([f"r.{b}" for b in __events_where_basic])} + AND event_type='PAGE' + AND {referrer_constraint})""" + # events_conditions_where.append(f"""main.session_id IN {referrer_constraint}""") + # extra_constraints.append(f"""s.session_id IN {referrer_constraint}""") + extra_from += f"\nINNER JOIN {referrer_constraint} AS referred ON(referred.session_id=s.session_id)" + elif filter_type == events.event_type.METADATA.ui_type: + # get metadata list only if you need it + if meta_keys is None: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + if f.source in meta_keys.keys(): + if is_any: + extra_constraints.append(f"isNotNull(s.{metadata.index_to_colname(meta_keys[f.source])})") + ss_constraints.append(f"isNotNull(ms.{metadata.index_to_colname(meta_keys[f.source])})") + elif is_undefined: + extra_constraints.append(f"isNull(s.{metadata.index_to_colname(meta_keys[f.source])})") + ss_constraints.append(f"isNull(ms.{metadata.index_to_colname(meta_keys[f.source])})") + else: + extra_constraints.append( + _multiple_conditions( + f"s.{metadata.index_to_colname(meta_keys[f.source])} {op} toString(%({f_k})s)", + f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions( + f"ms.{metadata.index_to_colname(meta_keys[f.source])} {op} toString(%({f_k})s)", + f.value, is_not=is_not, value_key=f_k)) + elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: + if is_any: + extra_constraints.append('isNotNull(s.user_id)') + ss_constraints.append('isNotNull(ms.user_id)') + elif is_undefined: + extra_constraints.append('isNull(s.user_id)') + ss_constraints.append('isNull(ms.user_id)') + else: + extra_constraints.append( + _multiple_conditions(f"s.user_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.user_anonymous_id, + schemas.FilterType.user_anonymous_id_ios]: + if is_any: + extra_constraints.append('isNotNull(s.user_anonymous_id)') + ss_constraints.append('isNotNull(ms.user_anonymous_id)') + elif is_undefined: + extra_constraints.append('isNull(s.user_anonymous_id)') + ss_constraints.append('isNull(ms.user_anonymous_id)') + else: + extra_constraints.append( + _multiple_conditions(f"s.user_anonymous_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_anonymous_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: + if is_any: + extra_constraints.append('isNotNull(s.rev_id)') + ss_constraints.append('isNotNull(ms.rev_id)') + elif is_undefined: + extra_constraints.append('isNull(s.rev_id)') + ss_constraints.append('isNull(ms.rev_id)') + else: + extra_constraints.append( + _multiple_conditions(f"s.rev_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.rev_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.platform: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.issue: + if is_any: + extra_constraints.append("array_length(s.issue_types, 1) > 0") + ss_constraints.append("array_length(ms.issue_types, 1) > 0") + else: + extra_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.events_count: + extra_constraints.append( + _multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + else: + continue + include_in_events = True + + if include_in_events: + events_conditions_where.append(f"""main.session_id IN (SELECT s.session_id + FROM final.sessions AS s + WHERE {" AND ".join(extra_constraints)})""") + # --------------------------------------------------------------------------- + if len(data.events) > 0: + valid_events_count = 0 + for event in data.events: + is_any = _isAny_opreator(event.operator) + if not isinstance(event.value, list): + event.value = [event.value] + if __is_valid_event(is_any=is_any, event=event): + valid_events_count += 1 + events_query_from = [] + events_conditions = [] + event_index = 0 + or_events = data.events_order == schemas.SearchEventOrder._or + # events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " + for i, event in enumerate(data.events): + event_type = event.type + print(f">>>>>>>>>>>>>{event_type}") + is_any = _isAny_opreator(event.operator) + if not isinstance(event.value, list): + event.value = [event.value] + if not __is_valid_event(is_any=is_any, event=event): + continue + op = __get_sql_operator(event.operator) + is_not = False + if __is_negation_operator(event.operator): + is_not = True + op = __reverse_sql_operator(op) + # if event_index == 0 or or_events: + # event_from = "%s INNER JOIN final.sessions AS ms USING (session_id)" + event_from = "%s" + event_where = ["main.project_id = %(projectId)s", + "main.datetime >= toDateTime(%(startDate)s/1000)", + "main.datetime <= toDateTime(%(endDate)s/1000)"] + if favorite_only and not errors_only: + event_from += "INNER JOIN final.user_favorite_sessions AS fs USING(session_id)" + event_where.append("fs.user_id = %(userId)s") + # else: + # event_from = "%s" + # event_where = ["main.datetime >= toDateTime(%(startDate)s/1000)", + # "main.datetime <= toDateTime(%(endDate)s/1000)", + # "main.session_id=event_0.session_id"] + # if data.events_order == schemas.SearchEventOrder._then: + # event_where.append(f"event_{event_index - 1}.datetime <= main.datetime") + e_k = f"e_value{i}" + s_k = e_k + "_source" + if event.type != schemas.PerformanceEventType.time_between_events: + event.value = helper.values_for_operator(value=event.value, op=event.operator) + full_args = {**full_args, + **_multiple_values(event.value, value_key=e_k), + **_multiple_values(event.source, value_key=s_k)} + + if event_type == events.event_type.CLICK.ui_type: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='CLICK'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, + value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + + elif event_type == events.event_type.INPUT.ui_type: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='INPUT'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, + value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + if event.source is not None and len(event.source) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, + value_key=f"custom{i}")) + full_args = {**full_args, **_multiple_values(event.source, value_key=f"custom{i}")} + + elif event_type == events.event_type.LOCATION.ui_type: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='PAGE'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + elif event_type == events.event_type.CUSTOM.ui_type: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='CUSTOM'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, + value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + elif event_type == events.event_type.REQUEST.ui_type: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='REQUEST'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, + value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + elif event_type == events.event_type.GRAPHQL.ui_type: + event_from = event_from % f"final.events AS main" + event_where.append(f"main.event_type='GRAPHQL'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, + value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + elif event_type == events.event_type.STATEACTION.ui_type: + event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.ERROR.ui_type: + event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" + event.source = tuple(event.source) + if not is_any and event.value not in [None, "*", ""]: + event_where.append( + _multiple_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) + if event.source[0] not in [None, "*", ""]: + event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.value, value_key=s_k)) + + elif event_type == schemas.PerformanceEventType.fetch_failed: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + event.value, value_key=e_k)) + col = performance_event.get_col(event_type) + colname = col["column"] + event_where.append(f"main.{colname} = FALSE") + # elif event_type == schemas.PerformanceEventType.fetch_duration: + # event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + # if not is_any: + # event_where.append( + # _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + # event.value, value_key=e_k)) + # col = performance_event.get_col(event_type) + # colname = col["column"] + # tname = "main" + # e_k += "_custom" + # full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + # _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + # event.source, value_key=e_k)) + elif event_type in [schemas.PerformanceEventType.location_dom_complete, + schemas.PerformanceEventType.location_largest_contentful_paint_time, + schemas.PerformanceEventType.location_ttfb, + schemas.PerformanceEventType.location_avg_cpu_load, + schemas.PerformanceEventType.location_avg_memory_usage + ]: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + col = performance_event.get_col(event_type) + colname = col["column"] + tname = "main" + if col.get("extraJoin") is not None: + tname = "ej" + event_from += f" INNER JOIN {col['extraJoin']} AS {tname} USING(session_id)" + event_where += [f"{tname}.timestamp >= main.timestamp", f"{tname}.timestamp >= %(startDate)s", + f"{tname}.timestamp <= %(endDate)s"] + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + + event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.time_between_events: + event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " + if not isinstance(event.value[0].value, list): + event.value[0].value = [event.value[0].value] + if not isinstance(event.value[1].value, list): + event.value[1].value = [event.value[1].value] + event.value[0].value = helper.values_for_operator(value=event.value[0].value, + op=event.value[0].operator) + event.value[1].value = helper.values_for_operator(value=event.value[1].value, + op=event.value[0].operator) + e_k1 = e_k + "_e1" + e_k2 = e_k + "_e2" + full_args = {**full_args, + **_multiple_values(event.value[0].value, value_key=e_k1), + **_multiple_values(event.value[1].value, value_key=e_k2)} + s_op = __get_sql_operator(event.value[0].operator) + event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] + if event_index > 0 and not or_events: + event_where.append("main2.session_id=event_0.session_id") + is_any = _isAny_opreator(event.value[0].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", + event.value[0].value, value_key=e_k1)) + s_op = __get_sql_operator(event.value[1].operator) + is_any = _isAny_opreator(event.value[1].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", + event.value[1].value, value_key=e_k2)) + + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + event_where.append( + _multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) + + elif event_type == schemas.EventType.request_details: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='REQUEST'") + events_conditions.append({"type": event_where[-1]}) + apply = False + events_conditions[-1]["condition"] = [] + for j, f in enumerate(event.filters): + is_any = _isAny_opreator(f.operator) + if is_any or len(f.value) == 0: + continue + f.value = helper.values_for_operator(value=f.value, op=f.operator) + op = __get_sql_operator(f.operator) + e_k_f = e_k + f"_fetch{j}" + full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + if f.type == schemas.FetchFilterType._url: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s", f.value, + value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + elif f.type == schemas.FetchFilterType._status_code: + event_where.append( + _multiple_conditions(f"main.status {f.operator} %({e_k_f})s", f.value, + value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + elif f.type == schemas.FetchFilterType._method: + event_where.append( + _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + elif f.type == schemas.FetchFilterType._duration: + event_where.append( + _multiple_conditions(f"main.duration {f.operator} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + elif f.type == schemas.FetchFilterType._request_body: + event_where.append( + _multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + elif f.type == schemas.FetchFilterType._response_body: + event_where.append( + _multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + else: + print(f"undefined FETCH filter: {f.type}") + if not apply: + continue + else: + events_conditions[-1]["condition"] = " AND ".join(events_conditions[-1]["condition"]) + + elif event_type == schemas.EventType.graphql_details: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='REQUEST'") + events_conditions.append({"type": event_where[-1]}) + events_conditions[-1]["condition"] = [] + for j, f in enumerate(event.filters): + is_any = _isAny_opreator(f.operator) + if is_any or len(f.value) == 0: + continue + f.value = helper.values_for_operator(value=f.value, op=f.operator) + op = __get_sql_operator(f.operator) + e_k_f = e_k + f"_graphql{j}" + full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + if f.type == schemas.GraphqlFilterType._name: + event_where.append( + _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value, + value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + elif f.type == schemas.GraphqlFilterType._method: + event_where.append( + _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + elif f.type == schemas.GraphqlFilterType._request_body: + event_where.append( + _multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + elif f.type == schemas.GraphqlFilterType._response_body: + event_where.append( + _multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + else: + print(f"undefined GRAPHQL filter: {f.type}") + events_conditions[-1]["condition"] = " AND ".join(events_conditions[-1]["condition"]) + else: + continue + if event_index == 0 or or_events: + event_where += ss_constraints + if is_not: + if event_index == 0 or or_events: + events_query_from.append(f"""\ + (SELECT + session_id, + 0 AS timestamp + FROM sessions + WHERE EXISTS(SELECT session_id + FROM {event_from} + WHERE {" AND ".join(event_where)} + AND sessions.session_id=ms.session_id) IS FALSE + AND project_id = %(projectId)s + AND start_ts >= %(startDate)s + AND start_ts <= %(endDate)s + AND duration IS NOT NULL + ) {"" if or_events else (f"AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + else: + events_query_from.append(f"""\ + (SELECT + event_0.session_id, + event_{event_index - 1}.timestamp AS timestamp + WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE + ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ + """) + else: + if data.events_order == schemas.SearchEventOrder._then: + pass + else: + events_query_from.append(f"""\ + (SELECT main.session_id, {"MIN" if event_index < (valid_events_count - 1) else "MAX"}(main.datetime) AS datetime + FROM {event_from} + WHERE {" AND ".join(event_where)} + GROUP BY session_id + ) {"" if or_events else (f"AS event_{event_index} " + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + event_index += 1 + + if event_index < 2: + data.events_order = schemas.SearchEventOrder._or + if favorite_only and user_id is not None: + events_conditions_where.append("""main.session_id IN (SELECT session_id + FROM final.user_favorite_sessions + WHERE user_id = %(userId)s)""") + for e in events_conditions: + print(e) + print("---") + if data.events_order in [schemas.SearchEventOrder._then, schemas.SearchEventOrder._and]: + events_conditions_where.append(f"({' OR '.join([c['type'] for c in events_conditions])})") + events_conditions_where.append(f"({' OR '.join([c['condition'] for c in events_conditions])})") + events_conditions = [c['type'] + ' AND ' + c['condition'] for c in events_conditions] + if data.events_order == schemas.SearchEventOrder._then: + having = f"""HAVING sequenceMatch('{''.join([f'(?{i + 1})' for i in range(len(events_conditions))])}')\ + (main.datetime,{','.join(events_conditions)})""" + else: + having = f"""HAVING {" AND ".join([f"countIf({c})>0" for c in events_conditions])}""" + + events_query_part = f"""SELECT main.session_id, + MIN(main.datetime) AS first_event_ts, + MAX(main.datetime) AS last_event_ts + FROM final.events AS main + WHERE {" AND ".join(events_conditions_where)} + GROUP BY session_id + {having}""" + else: + events_conditions_where.append(f"({' OR '.join([c['type'] for c in events_conditions])})") + events_conditions = [c['type'] + ' AND ' + c['condition'] for c in events_conditions] + events_conditions_where.append(f"({' OR '.join(events_conditions)})") + events_query_part = f"""SELECT main.session_id, + MIN(main.datetime) AS first_event_ts, + MAX(main.datetime) AS last_event_ts + FROM final.events AS main + WHERE {" AND ".join(events_conditions_where)} + GROUP BY session_id""" + else: + data.events = [] + # --------------------------------------------------------------------------- + if data.startDate is not None: + extra_constraints.append("s.datetime >= toDateTime(%(startDate)s/1000)") + if data.endDate is not None: + extra_constraints.append("s.datetime <= toDateTime(%(endDate)s/1000)") + # if data.platform is not None: + # if data.platform == schemas.PlatformType.mobile: + # extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") + # elif data.platform == schemas.PlatformType.desktop: + # extra_constraints.append( + # b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") + + if errors_only: + extra_from += f" INNER JOIN {events.event_type.ERROR.table} AS er USING (session_id) INNER JOIN public.errors AS ser USING (error_id)" + extra_constraints.append("ser.source = 'js_exception'") + extra_constraints.append("ser.project_id = %(project_id)s") + if error_status != schemas.ErrorStatus.all: + extra_constraints.append("ser.status = %(error_status)s") + full_args["error_status"] = error_status + if favorite_only: + extra_from += " INNER JOIN final.user_favorite_errors AS ufe USING (error_id)" + extra_constraints.append("ufe.user_id = %(userId)s") + # extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] + if favorite_only and not errors_only and user_id is not None: + extra_from += """INNER JOIN (SELECT 1 AS session_id) AS favorite_sessions + ON (TRUE)""" + elif not favorite_only and not errors_only and user_id is not None: + extra_from += """LEFT JOIN (SELECT session_id + FROM final.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + ON (s.session_id=favorite_sessions.session_id)""" + extra_join = "" + if issue is not None: + extra_join = """ + INNER JOIN LATERAL(SELECT TRUE FROM events_common.issues INNER JOIN public.issues AS p_issues USING (issue_id) + WHERE issues.session_id=f.session_id + AND p_issues.type=%(issue_type)s + AND p_issues.context_string=%(issue_contextString)s + AND timestamp >= f.first_event_ts + AND timestamp <= f.last_event_ts) AS issues ON(TRUE) + """ + full_args["issue_contextString"] = issue["contextString"] + full_args["issue_type"] = issue["type"] + if extra_event: + extra_join += f"""INNER JOIN {extra_event} AS ev USING(session_id)""" + extra_constraints.append("ev.timestamp>=%(startDate)s") + extra_constraints.append("ev.timestamp<=%(endDate)s") + if len(events_query_part) > 0: + extra_join += f"""INNER JOIN (SELECT * + FROM final.sessions AS s + WHERE {" AND ".join(extra_constraints)}) AS s ON(s.session_id=f.session_id)""" + else: + extra_join += f"""(SELECT * + FROM final.sessions AS s + WHERE {" AND ".join(extra_constraints)}) AS s""" + query_part = f"""\ + FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else ""} + {extra_join} + {extra_from} + """ + return full_args, query_part + + +def search_by_metadata(tenant_id, user_id, m_key, m_value, project_id=None): + if project_id is None: + all_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) + else: + all_projects = [ + projects.get_project(tenant_id=tenant_id, project_id=int(project_id), include_last_session=False, + include_gdpr=False)] + + all_projects = {int(p["projectId"]): p["name"] for p in all_projects} + project_ids = list(all_projects.keys()) + + available_keys = metadata.get_keys_by_projects(project_ids) + for i in available_keys: + available_keys[i]["user_id"] = schemas.FilterType.user_id + available_keys[i]["user_anonymous_id"] = schemas.FilterType.user_anonymous_id + results = {} + for i in project_ids: + if m_key not in available_keys[i].values(): + available_keys.pop(i) + results[i] = {"total": 0, "sessions": [], "missingMetadata": True} + project_ids = list(available_keys.keys()) + if len(project_ids) > 0: + with pg_client.PostgresClient() as cur: + sub_queries = [] + for i in project_ids: + col_name = list(available_keys[i].keys())[list(available_keys[i].values()).index(m_key)] + sub_queries.append(cur.mogrify( + f"(SELECT COALESCE(COUNT(s.*)) AS count FROM public.sessions AS s WHERE s.project_id = %(id)s AND s.{col_name} = %(value)s) AS \"{i}\"", + {"id": i, "value": m_value}).decode('UTF-8')) + query = f"""SELECT {", ".join(sub_queries)};""" + cur.execute(query=query) + + rows = cur.fetchone() + + sub_queries = [] + for i in rows.keys(): + results[i] = {"total": rows[i], "sessions": [], "missingMetadata": False, "name": all_projects[int(i)]} + if rows[i] > 0: + col_name = list(available_keys[int(i)].keys())[list(available_keys[int(i)].values()).index(m_key)] + sub_queries.append( + cur.mogrify( + f"""( + SELECT * + FROM ( + SELECT DISTINCT ON(favorite_sessions.session_id, s.session_id) {SESSION_PROJECTION_COLS} + FROM public.sessions AS s LEFT JOIN (SELECT session_id + FROM public.user_favorite_sessions + WHERE user_favorite_sessions.user_id = %(userId)s + ) AS favorite_sessions USING (session_id) + WHERE s.project_id = %(id)s AND s.duration IS NOT NULL AND s.{col_name} = %(value)s + ) AS full_sessions + ORDER BY favorite DESC, issue_score DESC + LIMIT 10 + )""", + {"id": i, "value": m_value, "userId": user_id}).decode('UTF-8')) + if len(sub_queries) > 0: + cur.execute("\nUNION\n".join(sub_queries)) + rows = cur.fetchall() + for i in rows: + results[str(i["project_id"])]["sessions"].append(helper.dict_to_camel_case(i)) + return results + + +def search_by_issue(user_id, issue, project_id, start_date, end_date): + constraints = ["s.project_id = %(projectId)s", + "p_issues.context_string = %(issueContextString)s", + "p_issues.type = %(issueType)s"] + if start_date is not None: + constraints.append("start_ts >= %(startDate)s") + if end_date is not None: + constraints.append("start_ts <= %(endDate)s") + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""SELECT DISTINCT ON(favorite_sessions.session_id, s.session_id) {SESSION_PROJECTION_COLS} + FROM public.sessions AS s + INNER JOIN events_common.issues USING (session_id) + INNER JOIN public.issues AS p_issues USING (issue_id) + LEFT JOIN (SELECT user_id, session_id + FROM public.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + USING (session_id) + WHERE {" AND ".join(constraints)} + ORDER BY s.session_id DESC;""", + { + "issueContextString": issue["contextString"], + "issueType": issue["type"], "userId": user_id, + "projectId": project_id, + "startDate": start_date, + "endDate": end_date + })) + + rows = cur.fetchall() + return helper.list_to_camel_case(rows) + + +def get_user_sessions(project_id, user_id, start_date, end_date): + with pg_client.PostgresClient() as cur: + constraints = ["s.project_id = %(projectId)s", "s.user_id = %(userId)s"] + if start_date is not None: + constraints.append("s.start_ts >= %(startDate)s") + if end_date is not None: + constraints.append("s.start_ts <= %(endDate)s") + + query_part = f"""\ + FROM public.sessions AS s + WHERE {" AND ".join(constraints)}""" + + cur.execute(cur.mogrify(f"""\ + SELECT s.project_id, + s.session_id::text AS session_id, + s.user_uuid, + s.user_id, + s.user_os, + s.user_browser, + s.user_device, + s.user_country, + s.start_ts, + s.duration, + s.events_count, + s.pages_count, + s.errors_count + {query_part} + ORDER BY s.session_id + LIMIT 50;""", { + "projectId": project_id, + "userId": user_id, + "startDate": start_date, + "endDate": end_date + })) + + sessions = cur.fetchall() + return helper.list_to_camel_case(sessions) + + +def get_session_user(project_id, user_id): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """\ + SELECT + user_id, + count(*) as session_count, + max(start_ts) as last_seen, + min(start_ts) as first_seen + FROM + "public".sessions + WHERE + project_id = %(project_id)s + AND user_id = %(userId)s + AND duration is not null + GROUP BY user_id; + """, + {"project_id": project_id, "userId": user_id} + ) + cur.execute(query=query) + data = cur.fetchone() + return helper.dict_to_camel_case(data) + + +def get_session_ids_by_user_ids(project_id, user_ids): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """\ + SELECT session_id FROM public.sessions + WHERE + project_id = %(project_id)s AND user_id IN %(userId)s;""", + {"project_id": project_id, "userId": tuple(user_ids)} + ) + ids = cur.execute(query=query) + return ids + + +def delete_sessions_by_session_ids(session_ids): + with pg_client.PostgresClient(unlimited_query=True) as cur: + query = cur.mogrify( + """\ + DELETE FROM public.sessions + WHERE + session_id IN %(session_ids)s;""", + {"session_ids": tuple(session_ids)} + ) + cur.execute(query=query) + + return True + + +def delete_sessions_by_user_ids(project_id, user_ids): + with pg_client.PostgresClient(unlimited_query=True) as cur: + query = cur.mogrify( + """\ + DELETE FROM public.sessions + WHERE + project_id = %(project_id)s AND user_id IN %(userId)s;""", + {"project_id": project_id, "userId": tuple(user_ids)} + ) + cur.execute(query=query) + + return True + + +def count_all(): + with pg_client.PostgresClient(unlimited_query=True) as cur: + row = cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions") + return row.get("count", 0) diff --git a/ee/api/clean.sh b/ee/api/clean.sh index fa1ab8cb5..e7785b22e 100755 --- a/ee/api/clean.sh +++ b/ee/api/clean.sh @@ -29,7 +29,6 @@ rm -rf ./chalicelib/core/log_tool_stackdriver.py rm -rf ./chalicelib/core/log_tool_sumologic.py rm -rf ./chalicelib/core/metadata.py rm -rf ./chalicelib/core/mobile.py -rm -rf ./chalicelib/core/sessions.py rm -rf ./chalicelib/core/sessions_assignments.py rm -rf ./chalicelib/core/sessions_metas.py rm -rf ./chalicelib/core/sessions_mobs.py diff --git a/ee/api/routers/core_dynamic.py b/ee/api/routers/core_dynamic.py index 3c5c21905..e5ba6c6c4 100644 --- a/ee/api/routers/core_dynamic.py +++ b/ee/api/routers/core_dynamic.py @@ -176,3 +176,13 @@ def get_general_stats(): def get_projects(context: schemas.CurrentContext = Depends(OR_context)): return {"data": projects.get_projects(tenant_id=context.tenant_id, recording_state=True, gdpr=True, recorded=True, stack_integrations=True, user_id=context.user_id)} + + +@app.post('/{projectId}/sessions/search2', tags=["sessions"]) +def sessions_search2(projectId: int, data: schemas.FlatSessionsSearchPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + if config("LEGACY_SEARCH", cast=bool, default=False): + data = sessions.search2_pg(data=data, project_id=projectId, user_id=context.user_id) + else: + data = sessions.search2_ch(data=data, project_id=projectId, user_id=context.user_id) + return {'data': data} diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql new file mode 100644 index 000000000..f6ba9d751 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql @@ -0,0 +1,258 @@ +ALTER TABLE sessions + DROP COLUMN pages_count; + + +CREATE TABLE projects_metadata +( + project_id UInt32, + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(_timestamp) + ORDER BY (project_id) + SETTINGS index_granularity = 512; + +CREATE TABLE IF NOT EXISTS events_s +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(UInt8), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16), + _timestamp DateTime DEFAULT now() +) ENGINE = MergeTree + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, event_type, session_id) + TTL datetime + INTERVAL 1 MONTH; + +CREATE TABLE IF NOT EXISTS sessions +( + session_id UInt64, + project_id UInt32, + tracker_version LowCardinality(String), + rev_id LowCardinality(Nullable(String)), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + duration UInt32, + pages_count UInt16, + events_count UInt16, + errors_count UInt16, + utm_source Nullable(String), + utm_medium Nullable(String), + utm_campaign Nullable(String), + user_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 1 MONTH + SETTINGS index_granularity = 512; + +CREATE TABLE IF NOT EXISTS autocomplete +( + project_id UInt32 NOT NULL, + type LowCardinality(String) NOT NULL, + value String NOT NULL, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(_timestamp) + ORDER BY (project_id, type) + TTL _timestamp + INTERVAL 1 MONTH; + + +CREATE MATERIALIZED VIEW sessions_l7d_mv + ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 7 DAY + SETTINGS index_granularity = 512 + POPULATE +AS +SELECT * +FROM massive_split.sessions_s +WHERE datetime >= now() - INTERVAL 7 DAY + AND isNotNull(duration) + AND duration > 0; + +CREATE MATERIALIZED VIEW events_l7d_mv + ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 7 DAY + POPULATE +AS +SELECT * +FROM massive_split.events_s +WHERE datetime >= now() - INTERVAL 7 DAY; + + +CREATE MATERIALIZED VIEW sessions_info_l1m_mv + ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 1 MONTH + SETTINGS index_granularity = 512 + POPULATE +AS +SELECT project_id, + session_id, + datetime, + now() AS _timestamp, + toJSONString(map('project_id', toString(project_id), + 'session_id', toString(session_id), + 'user_uuid', toString(user_uuid), + 'user_id', user_id, + 'user_os', user_os, + 'user_browser', user_browser, + 'user_device', user_device, + --'user_device_type', user_device_type, +--'user_country', user_country, + 'start_ts', toString(datetime), + 'duration', toString(duration), + 'events_count', toString(events_count), + 'pages_count', toString(pages_count), + 'errors_count', toString(errors_count), + -- 'user_anonymous_id', user_anonymous_id, +-- 'platform', platform, +-- 'issue_score', issue_score, +-- issue_types, +-- favorite, +-- viewed, + 'metadata', CAST((arrayFilter(x->isNotNull(x), + arrayMap( + x->if(isNotNull(x[1]) AND isNotNull(x[2]), toString(x[1]), + NULL), + [ + [projects_meta.metadata_1,sessions.metadata_1], + [projects_meta.metadata_2,sessions.metadata_2], + [projects_meta.metadata_3,sessions.metadata_3], + [projects_meta.metadata_4,sessions.metadata_4], + [projects_meta.metadata_5,sessions.metadata_5], + [projects_meta.metadata_6,sessions.metadata_6], + [projects_meta.metadata_7,sessions.metadata_7], + [projects_meta.metadata_8,sessions.metadata_8], + [projects_meta.metadata_9,sessions.metadata_9], + [projects_meta.metadata_10,sessions.metadata_10] + ])), + arrayFilter(x->isNotNull(x), + arrayMap( + x->if(isNotNull(x[1]) AND isNotNull(x[2]), toString(x[2]), + NULL), + [ + [projects_meta.metadata_1,sessions.metadata_1], + [projects_meta.metadata_2,sessions.metadata_2], + [projects_meta.metadata_3,sessions.metadata_3], + [projects_meta.metadata_4,sessions.metadata_4], + [projects_meta.metadata_5,sessions.metadata_5], + [projects_meta.metadata_6,sessions.metadata_6], + [projects_meta.metadata_7,sessions.metadata_7], + [projects_meta.metadata_8,sessions.metadata_8], + [projects_meta.metadata_9,sessions.metadata_9], + [projects_meta.metadata_10,sessions.metadata_10] + ]))), 'Map(String,String)') + )) AS info +FROM massive_split.sessions + INNER JOIN projects_metadata USING (project_id) +WHERE datetime >= now() - INTERVAL 1 MONTH + AND isNotNull(duration) + AND duration > 0; + +CREATE MATERIALIZED VIEW sessions_info_l7d_mv + ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 7 DAY + SETTINGS index_granularity = 512 + POPULATE +AS +SELECT * +FROM sessions_info_l1m_mv +WHERE datetime >= now() - INTERVAL 7 DAY; diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/__clickhouse.csv b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/__clickhouse.csv new file mode 100644 index 000000000..7c2f7cab1 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/__clickhouse.csv @@ -0,0 +1,68 @@ +MACHINE ,QUERY ,TABLE ,TIME(max_threads=auto(2)),TIME (max_threads=4),TIME (max_threads=4) 1W,TIME (max_threads=4) 1W_mv,TIME 1d (max_threads=4) 1W_mv,TIME 1d (max_threads=4) 1d_mv,REMARK +r5.large (16Gb) ,Q1 ,massive2.events7 ,timeout ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,Q1 ,massive2.events7 ,12s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),Q1 ,massive2.events7 ,7s ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,Q1 ,massive2.events7 ,24s ,19s ,17s ,- ,- ,- , +r5.xlarge (32Gb) ,Q1.1 ,massive2.events7 ,16s ,14s ,12s ,- ,- ,- ,WHERE user_id +r5.xlarge (32Gb) ,Q1 ,massive_split.events_s,23s ,18s ,16s ,3s ,1s ,0.8s , +r5.xlarge (32Gb) ,Q1.2 ,massive_split.events_s,timeout ,27s ,23s ,2s ,0.8s ,0.7s ,events INNER JOIN meta WHERE user_id +r5.xlarge (32Gb) ,Q1.2.1 ,massive_split.events_s,timeout ,timeout ,27s ,4s ,1.6s ,0.8s ,events INNER JOIN meta WHERE ALL AND user_id +r5.xlarge (32Gb) ,Q1.3 ,massive_split.events_s,15s ,13s ,11s ,2s ,0.7s ,0.6s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,Q1.4 ,massive_split.events_s,memory ,memory ,memory ,4s ,0.7s ,0.6s ,SUBQUERY WHERE user_id INNER JOIN events +r5.xlarge (32Gb) ,Q1.5 ,massive_split.events_s,11s ,10s ,8s ,1s ,0.6s ,0.6s ,WHERE session_id IN SUBQUERY WHERE user_id +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== +r5.large (16Gb) ,Q2 ,massive2.events7 ,timeout ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,Q2 ,massive2.events7 ,12s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),Q2 ,massive2.events7 ,7s ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,Q2 ,massive2.events7 ,24s ,20s ,17s ,- ,- ,- , +r5.xlarge (32Gb) ,Q2.1 ,massive2.events7 ,21s ,14s ,12s ,- ,- ,- ,WHERE user_id +r5.xlarge (32Gb) ,Q2 ,massive_split.events_s,22s ,19s ,16s ,3s ,0.7s ,0.6s , +r5.xlarge (32Gb) ,Q2.2 ,massive_split.events_s,timeout ,27s ,23s ,2s ,0.6s ,0.6s ,events INNER JOIN meta WHERE user_id +r5.xlarge (32Gb) ,Q2.2.1 ,massive_split.events_s,timeout ,timeout ,27s ,4s ,1.3s ,0.7s ,events INNER JOIN meta WHERE ALL AND user_id +r5.xlarge (32Gb) ,Q2.3 ,massive_split.events_s,15s ,13s ,11s ,2s ,0.6s ,0.6s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,Q2.4 ,massive_split.events_s,memory ,memory ,memory ,4s ,0.6s ,0.6s ,SUBQUERY WHERE user_id INNER JOIN events +r5.xlarge (32Gb) ,Q2.5 ,massive_split.events_s,11s ,10s ,8s ,1s ,0.6s ,0.6s ,WHERE session_id IN SUBQUERY WHERE user_id +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== +r5.large (16Gb) ,Q3 ,massive2.events7 ,timeout ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,Q3 ,massive2.events7 ,11s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),Q3 ,massive2.events7 ,6s ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,Q3 ,massive2.events7 ,22s ,19s ,15s ,- ,- ,- , +r5.xlarge (32Gb) ,Q3.1 ,massive2.events7 ,18s ,15s ,12s ,- ,- ,- ,WHERE user_id +r5.xlarge (32Gb) ,Q3 ,massive_split.events_s,21s ,18s ,15s ,3s ,0.7s ,0.7s , +r5.xlarge (32Gb) ,Q3.2 ,massive_split.events_s,24s ,20s ,17s ,2s ,0.7s ,0.6s ,events INNER JOIN meta WHERE user_id +r5.xlarge (32Gb) ,Q3.2.1 ,massive_split.events_s,26s ,22s ,19s ,4s ,1.4s ,0.7s ,events INNER JOIN meta WHERE ALL AND user_id +r5.xlarge (32Gb) ,Q3.3 ,massive_split.events_s,15s ,13s ,11s ,2s ,0.6s ,0.6s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,Q3.4 ,massive_split.events_s,memory ,memory ,memroy ,3s ,0.7s ,0.7s ,SUBQUERY WHERE user_id INNER JOIN events +r5.xlarge (32Gb) ,Q3.5 ,massive_split.events_s,13s ,11s ,9s ,1s ,0.7s ,0.6s ,WHERE session_id IN SUBQUERY WHERE user_id +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== +r5.large (16Gb) ,Q4 ,massive2.events7 ,timeout ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,Q4 ,massive2.events7 ,11s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),Q4 ,massive2.events7 ,6s ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,Q4 ,massive2.events7 ,22s ,18s ,15s ,- ,- ,- , +r5.xlarge (32Gb) ,Q4.1 ,massive2.events7 ,18s ,15s ,13s ,- ,- ,- ,WHERE user_id +r5.xlarge (32Gb) ,Q4 ,massive_split.events_s,21s ,18s ,15s ,3s ,0.7s ,0.6s , +r5.xlarge (32Gb) ,Q4.2 ,massive_split.events_s,24s ,20s ,17s ,2s ,0.7s ,0.6s ,events INNER JOIN meta WHERE user_id +r5.xlarge (32Gb) ,Q4.2.1 ,massive_split.events_s,27s ,22s ,19s ,4s ,1.4s ,0.7s ,events INNER JOIN meta WHERE ALL AND user_id +r5.xlarge (32Gb) ,Q4.3 ,massive_split.events_s,15s ,13s ,11s ,2s ,0.6s ,0.7s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,Q4.4 ,massive_split.events_s,memory ,memory ,memroy ,3s ,1.2s ,0.7s ,SUBQUERY WHERE user_id INNER JOIN events +r5.xlarge (32Gb) ,Q4.5 ,massive_split.events_s,14s ,12s ,9s ,2s ,0.6s ,0.6s ,WHERE session_id IN SUBQUERY WHERE user_id +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== +r5.large (16Gb) ,QU1 ,massive2.events7 ,timeout ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,QU1 ,massive2.events7 ,18s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),QU1 ,massive2.events7 ,10s ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,QU1 ,massive2.events7 ,timeout ,28s ,24s ,- ,- ,- , +r5.xlarge (32Gb) ,QU1.1 ,massive2.events7 ,17s ,14s ,12s ,- ,- ,- ,WHERE user_id +r5.xlarge (32Gb) ,QU1.2 ,massive_split.events_s,timeout ,timeout ,timeout ,6s ,1.5s ,0.7s ,events INNER JOIN meta +r5.xlarge (32Gb) ,QU1.3 ,massive_split.events_s,timeout ,timeout ,23s ,3s ,1.4s ,0.7s ,events INNER JOIN meta WHERE user_id +r5.xlarge (32Gb) ,QU1.4 ,massive_split.events_s,15s ,13s ,11s ,2s ,0.7s ,0.6s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,QU1.4-A,massive_split.events_s,timeout ,timeout ,timeout ,6s ,0.7s ,0.7s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,QU1.5 ,massive_split.events_s,memory ,memory ,memory ,memory ,memory ,2s ,SUBQUERY WHERE user_id INNER JOIN events +r5.xlarge (32Gb) ,QU1.6 ,massive_split.events_s,11s ,10s ,8s ,1s ,0.7s ,0.7s ,events INNER JOIN SUBQUERY WHERE user_id WHERE session_id IN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,QU1.6-A,massive_split.events_s,timeout ,timeout ,timeout ,6s ,0.7s ,0.7s ,events INNER JOIN SUBQUERY WHERE session_id IN SUBQUERY +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== +r5.large (16Gb) ,QM4 ,massive2.events7 ,- ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,QM4 ,massive2.events7 ,3s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),QM4 ,massive2.events7 ,- ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,QM4 ,massive2.events7 ,7s ,5s ,4s ,- ,- ,- , +r5.xlarge (32Gb) ,QM4 ,massive_split.events_s,6s ,5s ,4s ,1s ,0.7s ,0.6s , +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql new file mode 100644 index 000000000..e22b73848 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql @@ -0,0 +1,2878 @@ +-- CREATE TABLE IF NOT EXISTS single_t.events +-- ( +-- session_id UInt64, +-- project_id UInt32, +-- event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), +-- tracker_version LowCardinality(String), +-- rev_id Nullable(String), +-- user_uuid UUID, +-- user_os LowCardinality(String), +-- user_os_version LowCardinality(Nullable(String)), +-- user_browser LowCardinality(String), +-- user_browser_version LowCardinality(Nullable(String)), +-- user_device Nullable(String), +-- user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), +-- user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), +-- datetime DateTime, +-- label Nullable(String), +-- hesitation_time Nullable(UInt32), +-- name Nullable(String), +-- payload Nullable(String), +-- level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), +-- source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), +-- message Nullable(String), +-- error_id Nullable(String), +-- duration Nullable(UInt16), +-- context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), +-- container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), +-- container_id Nullable(String), +-- container_name Nullable(String), +-- container_src Nullable(String), +-- url Nullable(String), +-- url_host Nullable(String) MATERIALIZED lower(domain(url)), +-- url_path Nullable(String) MATERIALIZED lower(pathFull(url)), +-- request_start Nullable(UInt16), +-- response_start Nullable(UInt16), +-- response_end Nullable(UInt16), +-- dom_content_loaded_event_start Nullable(UInt16), +-- dom_content_loaded_event_end Nullable(UInt16), +-- load_event_start Nullable(UInt16), +-- load_event_end Nullable(UInt16), +-- first_paint Nullable(UInt16), +-- first_contentful_paint Nullable(UInt16), +-- speed_index Nullable(UInt16), +-- visually_complete Nullable(UInt16), +-- time_to_interactive Nullable(UInt16), +-- ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), +-- minus(response_start, request_start), Null), +-- ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), +-- minus(response_end, request_start), Null), +-- response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), +-- minus(response_end, response_start), Null), +-- dom_building_time Nullable(UInt16) MATERIALIZED if( +-- greaterOrEquals(dom_content_loaded_event_start, response_end), +-- minus(dom_content_loaded_event_start, response_end), Null), +-- dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( +-- greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), +-- minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), +-- load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), +-- minus(load_event_end, load_event_start), Null), +-- min_fps Nullable(UInt8), +-- avg_fps Nullable(UInt8), +-- max_fps Nullable(UInt8), +-- min_cpu Nullable(UInt8), +-- avg_cpu Nullable(UInt8), +-- max_cpu Nullable(UInt8), +-- min_total_js_heap_size Nullable(UInt64), +-- avg_total_js_heap_size Nullable(UInt64), +-- max_total_js_heap_size Nullable(UInt64), +-- min_used_js_heap_size Nullable(UInt64), +-- avg_used_js_heap_size Nullable(UInt64), +-- max_used_js_heap_size Nullable(UInt64), +-- type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), +-- header_size Nullable(UInt16), +-- encoded_body_size Nullable(UInt32), +-- decoded_body_size Nullable(UInt32), +-- compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), +-- success Nullable(UInt8), +-- method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), +-- status Nullable(UInt16) +-- ) ENGINE = MergeTree +-- PARTITION BY toDate(datetime) +-- ORDER BY (project_id, datetime); +-- -- TTL datetime + INTERVAL 1 MONTH; +-- DROP TABLE single_t.events; +-- +-- INSERT INTO eng_t.events4(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, label, hesitation_time, name, payload, level, source, message, +-- error_id, duration, context, container_type, container_id, container_name, container_src, +-- url, request_start, response_start, response_end, dom_content_loaded_event_start, +-- dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, +-- first_contentful_paint, speed_index, visually_complete, time_to_interactive, min_fps, +-- avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, +-- max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, +-- type, header_size, encoded_body_size, decoded_body_size, success, method, status) +-- SELECT session_id, +-- project_id, +-- event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- label, +-- hesitation_time, +-- name, +-- payload, +-- level, +-- source, +-- message, +-- error_id, +-- duration, +-- context, +-- container_type, +-- container_id, +-- container_name, +-- container_src, +-- url, +-- request_start, +-- response_start, +-- response_end, +-- dom_content_loaded_event_start, +-- dom_content_loaded_event_end, +-- load_event_start, +-- load_event_end, +-- first_paint, +-- first_contentful_paint, +-- speed_index, +-- visually_complete, +-- time_to_interactive, +-- min_fps, +-- avg_fps, +-- max_fps, +-- min_cpu, +-- avg_cpu, +-- max_cpu, +-- min_total_js_heap_size, +-- avg_total_js_heap_size, +-- max_total_js_heap_size, +-- min_used_js_heap_size, +-- avg_used_js_heap_size, +-- max_used_js_heap_size, +-- type, +-- header_size, +-- encoded_body_size, +-- decoded_body_size, +-- success, +-- method, +-- status +-- FROM ( +-- SELECT session_id, +-- project_id, +-- 'CLICK' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- label, +-- hesitation_time, +-- null AS name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- null AS message, +-- null AS error_id, +-- null AS duration, +-- null AS context, +-- null AS container_type, +-- null AS container_id, +-- null AS container_name, +-- null AS container_src, +-- null AS url, +-- null AS request_start, +-- null AS response_start, +-- null AS response_end, +-- null AS dom_content_loaded_event_start, +-- null AS dom_content_loaded_event_end, +-- null AS load_event_start, +-- null AS load_event_end, +-- null AS first_paint, +-- null AS first_contentful_paint, +-- null AS speed_index, +-- null AS visually_complete, +-- null AS time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- null AS type, +-- null AS header_size, +-- null AS encoded_body_size, +-- null AS decoded_body_size, +-- null AS success, +-- null AS method, +-- null AS status +-- FROM clicks +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'ERROR' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- null AS label, +-- null AS hesitation_time, +-- name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- message, +-- error_id, +-- null AS duration, +-- null AS context, +-- null AS container_type, +-- null AS container_id, +-- null AS container_name, +-- null AS container_src, +-- null AS url, +-- null AS request_start, +-- null AS response_start, +-- null AS response_end, +-- null AS dom_content_loaded_event_start, +-- null AS dom_content_loaded_event_end, +-- null AS load_event_start, +-- null AS load_event_end, +-- null AS first_paint, +-- null AS first_contentful_paint, +-- null AS speed_index, +-- null AS visually_complete, +-- null AS time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- null AS type, +-- null AS header_size, +-- null AS encoded_body_size, +-- null AS decoded_body_size, +-- null AS success, +-- null AS method, +-- null AS status +-- FROM errors +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'INPUT' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- label, +-- null AS hesitation_time, +-- null AS name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- null AS message, +-- null AS error_id, +-- null AS duration, +-- null AS context, +-- null AS container_type, +-- null AS container_id, +-- null AS container_name, +-- null AS container_src, +-- null AS url, +-- null AS request_start, +-- null AS response_start, +-- null AS response_end, +-- null AS dom_content_loaded_event_start, +-- null AS dom_content_loaded_event_end, +-- null AS load_event_start, +-- null AS load_event_end, +-- null AS first_paint, +-- null AS first_contentful_paint, +-- null AS speed_index, +-- null AS visually_complete, +-- null AS time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- null AS type, +-- null AS header_size, +-- null AS encoded_body_size, +-- null AS decoded_body_size, +-- null AS success, +-- null AS method, +-- null AS status +-- FROM inputs +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'LONGTASK' event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- null AS label, +-- null AS hesitation_time, +-- null AS name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- null AS message, +-- null AS error_id, +-- duration, +-- context, +-- container_type, +-- container_id, +-- container_name, +-- container_src, +-- null AS url, +-- null AS request_start, +-- null AS response_start, +-- null AS response_end, +-- null AS dom_content_loaded_event_start, +-- null AS dom_content_loaded_event_end, +-- null AS load_event_start, +-- null AS load_event_end, +-- null AS first_paint, +-- null AS first_contentful_paint, +-- null AS speed_index, +-- null AS visually_complete, +-- null AS time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- null AS type, +-- null AS header_size, +-- null AS encoded_body_size, +-- null AS decoded_body_size, +-- null AS success, +-- null AS method, +-- null AS status +-- FROM longtasks +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'PAGE' event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- null AS label, +-- null AS hesitation_time, +-- null AS name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- null AS message, +-- null AS error_id, +-- null AS duration, +-- null AS context, +-- null AS container_type, +-- null AS container_id, +-- null AS container_name, +-- null AS container_src, +-- url, +-- request_start, +-- response_start, +-- response_end, +-- dom_content_loaded_event_start, +-- dom_content_loaded_event_end, +-- load_event_start, +-- load_event_end, +-- first_paint, +-- first_contentful_paint, +-- speed_index, +-- visually_complete, +-- time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- null AS type, +-- null AS header_size, +-- null AS encoded_body_size, +-- null AS decoded_body_size, +-- null AS success, +-- null AS method, +-- null AS status +-- FROM pages +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'PERFORMANCE' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- null AS label label, +-- null AS label hesitation_time, +-- null AS label name, +-- null AS label payload, +-- null AS label level, +-- null AS label source, +-- null AS label message, +-- null AS label error_id, +-- null AS label duration, +-- null AS label context, +-- null AS label container_type, +-- null AS label container_id, +-- null AS label container_name, +-- null AS label container_src, +-- null AS labelurl, +-- null AS label request_start, +-- null AS label response_start, +-- null AS label response_end, +-- null AS label dom_content_loaded_event_start, +-- null AS label dom_content_loaded_event_end, +-- null AS label load_event_startnull, +-- null AS label load_event_end, +-- null AS label first_paint, +-- null AS label first_contentful_paint, +-- null AS label speed_index, +-- null AS label visually_complete, +-- null AS label time_to_interactive, +-- min_fps, +-- avg_fps, +-- max_fps, +-- min_cpu, +-- avg_cpu, +-- max_cpu, +-- min_total_js_heap_size, +-- avg_total_js_heap_size, +-- max_total_js_heap_size, +-- min_used_js_heap_size, +-- avg_used_js_heap_size, +-- max_used_js_heap_size, +-- null AS label type, +-- null AS label header_size, +-- null AS label encoded_body_size, +-- null AS label decoded_body_size, +-- null AS label success, +-- null AS label method, +-- null AS label status +-- FROM performance +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'RESOURCE' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- null AS label, +-- null AS hesitation_time, +-- null AS name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- null AS message, +-- null AS error_id, +-- duration, +-- null AS context, +-- null AS container_type, +-- null AS container_id, +-- null AS container_name, +-- null AS container_src, +-- url, +-- null AS request_start, +-- null AS response_start, +-- null AS response_end, +-- null AS dom_content_loaded_event_start, +-- null AS dom_content_loaded_event_end, +-- null AS load_event_start, +-- null AS load_event_end, +-- null AS first_paint, +-- null AS first_contentful_paint, +-- null AS speed_index, +-- null AS visually_complete, +-- null AS time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- type, +-- header_size, +-- encoded_body_size, +-- decoded_body_size, +-- success, +-- method, +-- status +-- FROM resources); +-- +-- +-- INSERT INTO eng_t.events4(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, label, hesitation_time) +-- SELECT session_id, +-- project_id, +-- 'CLICK' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- label, +-- hesitation_time +-- FROM clicks; +-- +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, +-- user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, +-- source, name, message, error_id) +-- +-- SELECT 'ERROR' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- source, +-- name, +-- message, +-- error_id +-- FROM errors; +-- +-- +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, label) +-- +-- SELECT 'INPUT' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- label +-- FROM inputs; +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, duration, context, container_type, container_id, container_name, +-- container_src) +-- SELECT 'LONGTASK' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- duration, +-- context, +-- container_type, +-- container_id, +-- container_name, +-- container_src +-- FROM longtasks; +-- +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, url, request_start, response_start, response_end, +-- dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, +-- load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, +-- time_to_interactive) +-- SELECT 'PAGE' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- url, +-- request_start, +-- response_start, +-- response_end, +-- dom_content_loaded_event_start, +-- dom_content_loaded_event_end, +-- load_event_start, +-- load_event_end, +-- first_paint, +-- first_contentful_paint, +-- speed_index, +-- visually_complete, +-- time_to_interactive +-- FROM pages; +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, +-- min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, +-- min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size) +-- SELECT 'PERFORMANCE' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- min_fps, +-- avg_fps, +-- max_fps, +-- min_cpu, +-- avg_cpu, +-- max_cpu, +-- min_total_js_heap_size, +-- avg_total_js_heap_size, +-- max_total_js_heap_size, +-- min_used_js_heap_size, +-- avg_used_js_heap_size, +-- max_used_js_heap_size +-- FROM performance; +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, url, type, duration, header_size, encoded_body_size, +-- decoded_body_size, success, method, status) +-- SELECT 'RESOURCE' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- url, +-- type, +-- duration, +-- header_size, +-- encoded_body_size, +-- decoded_body_size, +-- success, +-- method, +-- status +-- FROM resources; +-- +-- +-- SELECT table, formatReadableSize(size) as size, rows, days, formatReadableSize(avgDaySize) as avgDaySize +-- FROM ( +-- SELECT table, +-- sum(bytes) AS size, +-- sum(rows) AS rows, +-- min(min_date) AS min_date, +-- max(max_date) AS max_date, +-- (max_date - min_date) AS days, +-- size / (max_date - min_date) AS avgDaySize +-- FROM system.parts +-- WHERE active +-- GROUP BY table +-- ORDER BY rows DESC +-- ); +-- +-- SELECT database, +-- table, +-- formatReadableSize(sum(bytes)) as size, +-- min(min_date) as min_date, +-- max(max_date) as max_date +-- FROM system.parts +-- WHERE active +-- GROUP BY database, table; +-- +-- SELECT count(*) +-- FROM single_t.events; +-- -- 449 484 932 +-- -- 449 484 932 +-- +-- SELECT (SELECT count(*) FROM clicks) + (SELECT count(*) FROM inputs) + (SELECT count(*) FROM longtasks) + +-- (SELECT count(*) FROM errors) + (SELECT count(*) FROM pages) + (SELECT count(*) FROM resources) + +-- (SELECT count(*) FROM performance) AS totl; +-- +-- +-- +-- CREATE TABLE IF NOT EXISTS single_t.events3 +-- ( +-- session_id UInt64, +-- project_id UInt32, +-- event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), +-- tracker_version LowCardinality(String), +-- rev_id Nullable(String), +-- user_uuid UUID, +-- user_os LowCardinality(String), +-- user_os_version LowCardinality(Nullable(String)), +-- user_browser LowCardinality(String), +-- user_browser_version LowCardinality(Nullable(String)), +-- user_device Nullable(String), +-- user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), +-- user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), +-- datetime DateTime, +-- label Nullable(String), +-- hesitation_time Nullable(UInt32), +-- name Nullable(String), +-- payload Nullable(String), +-- level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), +-- source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), +-- message Nullable(String), +-- error_id Nullable(String), +-- duration Nullable(UInt16), +-- context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), +-- container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), +-- container_id Nullable(String), +-- container_name Nullable(String), +-- container_src Nullable(String), +-- url Nullable(String), +-- url_host Nullable(String) MATERIALIZED lower(domain(url)), +-- url_path Nullable(String) MATERIALIZED lower(pathFull(url)), +-- request_start Nullable(UInt16), +-- response_start Nullable(UInt16), +-- response_end Nullable(UInt16), +-- dom_content_loaded_event_start Nullable(UInt16), +-- dom_content_loaded_event_end Nullable(UInt16), +-- load_event_start Nullable(UInt16), +-- load_event_end Nullable(UInt16), +-- first_paint Nullable(UInt16), +-- first_contentful_paint Nullable(UInt16), +-- speed_index Nullable(UInt16), +-- visually_complete Nullable(UInt16), +-- time_to_interactive Nullable(UInt16), +-- ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), +-- minus(response_start, request_start), Null), +-- ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), +-- minus(response_end, request_start), Null), +-- response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), +-- minus(response_end, response_start), Null), +-- dom_building_time Nullable(UInt16) MATERIALIZED if( +-- greaterOrEquals(dom_content_loaded_event_start, response_end), +-- minus(dom_content_loaded_event_start, response_end), Null), +-- dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( +-- greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), +-- minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), +-- load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), +-- minus(load_event_end, load_event_start), Null), +-- min_fps Nullable(UInt8), +-- avg_fps Nullable(UInt8), +-- max_fps Nullable(UInt8), +-- min_cpu Nullable(UInt8), +-- avg_cpu Nullable(UInt8), +-- max_cpu Nullable(UInt8), +-- min_total_js_heap_size Nullable(UInt64), +-- avg_total_js_heap_size Nullable(UInt64), +-- max_total_js_heap_size Nullable(UInt64), +-- min_used_js_heap_size Nullable(UInt64), +-- avg_used_js_heap_size Nullable(UInt64), +-- max_used_js_heap_size Nullable(UInt64), +-- type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), +-- header_size Nullable(UInt16), +-- encoded_body_size Nullable(UInt32), +-- decoded_body_size Nullable(UInt32), +-- compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), +-- success Nullable(UInt8), +-- method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), +-- status Nullable(UInt16) +-- ) ENGINE = MergeTree +-- PARTITION BY toDate(datetime) +-- ORDER BY (project_id, datetime,event_type); +-- +-- -- INSERT INTO eng_t.events42(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label, hesitation_time, name, payload, level, source, message, error_id, duration, context, container_type, container_id, container_name, container_src, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, time_to_interactive, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, type, header_size, encoded_body_size, decoded_body_size, success, method, status) +-- -- SELECT session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label, hesitation_time, name, payload, level, source, message, error_id, duration, context, container_type, container_id, container_name, container_src, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, time_to_interactive, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, type, header_size, encoded_body_size, decoded_body_size, success, method, status FROM single_t.events; + +CREATE TABLE IF NOT EXISTS single_t.events3 +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + tracker_version LowCardinality(String), + rev_id Nullable(String), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(BOOLEAN), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16) +) ENGINE = MergeTree + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime, event_type); + +INSERT INTO eng_t.events4(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, hesitation_time) +SELECT session_id, + project_id, + 'CLICK' AS event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + hesitation_time +FROM clicks +WHERE mod(session_id, 2) = 1; + + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, + user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, + source, name, message, error_id) + +SELECT 'ERROR' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + source, + name, + message, + error_id +FROM errors +WHERE mod(session_id, 2) = 0; +TRUNCATE TABLE eng_t.events4; +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label) + +SELECT 'INPUT' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label +FROM inputs +WHERE mod(session_id, 2) = 0; + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, duration, context, container_type, container_id, container_name, + container_src) +SELECT 'LONGTASK' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + context, + container_type, + container_id, + container_name, + container_src +FROM longtasks +WHERE mod(session_id, 2) = 0; + + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, request_start, response_start, response_end, + dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, + load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, + time_to_interactive) +SELECT 'PAGE' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint, + speed_index, + visually_complete, + time_to_interactive +FROM pages +WHERE mod(session_id, 2) = 0; + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, + min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, + min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size) +SELECT 'PERFORMANCE' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size +FROM performance +WHERE mod(session_id, 2) = 0; + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status) +SELECT 'RESOURCE' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + if(status IS NOT NULL, status = 1, null) AS status +FROM resources +WHERE type != 'fetch' + AND mod(session_id, 2) = 0; + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status) +SELECT 'REQUEST' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + if(status IS NOT NULL, status = 1, null) AS status +FROM resources +WHERE type = 'fetch' + AND mod(session_id, 2) = 0; + +CREATE TABLE IF NOT EXISTS eng_t.events4 +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + tracker_version LowCardinality(String), + rev_id Nullable(String), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(BOOLEAN), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16) +) ENGINE = Join(ALL, INNER, session_id); + +TRUNCATE TABLE eng_t.events4; + + +-- merge metadata with events +CREATE DATABASE full_meerge; + +CREATE TABLE IF NOT EXISTS massive.events6 +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + tracker_version LowCardinality(String), + rev_id Nullable(String), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(UInt8), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16), + user_id Nullable(String), + user_anonymous_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String) +) ENGINE = MergeTree + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime); +-- TTL datetime + INTERVAL 1 MONTH; +INSERT INTO massive.events6(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, hesitation_time, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10) +SELECT session_id + 6651141467121565 * 3 AS session_id, + project_id, + 'CLICK' AS event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + hesitation_time, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.clicks + LEFT JOIN default.sessions_metadata USING (session_id); + + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, + datetime, source, name, message, error_id, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) + +SELECT 'ERROR' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + source, + name, + message, + error_id, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.errors + LEFT JOIN default.sessions_metadata USING (session_id); + + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) + +SELECT 'INPUT' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.inputs + LEFT JOIN default.sessions_metadata USING (session_id); + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, duration, context, container_type, container_id, container_name, + container_src, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) +SELECT 'LONGTASK' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + context, + container_type, + container_id, + container_name, + container_src, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.longtasks + LEFT JOIN default.sessions_metadata USING (session_id); + + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, request_start, response_start, response_end, + dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, + load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, + time_to_interactive, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) +SELECT 'PAGE' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint, + speed_index, + visually_complete, + time_to_interactive, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.pages + LEFT JOIN default.sessions_metadata USING (session_id); + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, + min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, + min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) +SELECT 'PERFORMANCE' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.performance + LEFT JOIN default.sessions_metadata USING (session_id); + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) +SELECT 'RESOURCE' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + if(status IS NOT NULL, status = 1, null) AS status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.resources + LEFT JOIN default.sessions_metadata USING (session_id) +WHERE type != 'fetch'; + +INSERT INTO massive2.events7(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) +SELECT 'REQUEST' AS event_type, + session_id + 6651141467121565 * 4 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + if(status IS NOT NULL, status = 1, null) AS status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.resources + LEFT JOIN default.sessions_metadata USING (session_id) +WHERE type = 'fetch' + AND mod(project_id, 2) = 0; + +-- -- TO GENERATE RANDOM USER IDS +-- INSERT INTO sessions_metadata(session_id, user_id, datetime, project_id, user_device_type) +-- SELECT session_id, +-- arrayElement( +-- array('Ze2wc7lvYi', 'NYd7m0Ytg8', 'qgNpvEkXap', 'wvWqM4Ow2G', 'n5Y6DK7ZdP', 'uW4SEYjXxI', 't4EfJiNxk9', +-- 'qWQ8WuIRLS', 'fnRWCwkFyB', '8wf298MFWR', 'G3A3DL0Fdd', 'cQcZHNNiAJ', 'MKcW2adQ38', 'OBzk9EFxVe', +-- '8SBiqoFail', '3Wh9Ur0eOr', 'z6KuuxiPXX', '7j4HaReEsF', 'Ros0kDOVeV', 'PvHi3cBkgV', 'HLjUo6oBlJ', +-- '4Tmi34faA0', 'O9ZATbPjaB', '7ATvuWQCIH', 'kXW4LHnW5X', 'HIHc9TTyTc', 'i5p9jRe7I0', '7dRnUEFoZO', +-- 'u3PDLkI5uG', 'HTYjxmDJCG', '6hKHjcKniO', 'qmPNUWgDIx', 'RfoN9oeYZD', 'HHXpBaYm3k', 'VdpZDfnL9J', +-- 'Qfwa1dPrrF', 'cgdD2GfFVT', 'iRvT6l7qj3', 'QokprB2GMV', 'umqISqbncX', '7bvRdQ4al3', 'VGKZAUIRjy', +-- 'SNTEGLKbCD', 'zfUaVSD8Jn', 'De7zUojKNt', 'lXiotVRkil', 'bQaDX5kESw', 'tngESCaH6I', 'uucUZvTpPd', +-- 'BFJpni8D3I'), mod(session_id, 50)) AS user_id, +-- datetime, +-- project_id, +-- user_device_type +-- FROM sessions +-- WHERE project_id = 2460; + +INSERT INTO massive2.sessions2(session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, + user_browser, user_browser_version, user_device, user_device_type, user_country, + datetime, + duration, events_count, errors_count, utm_source, utm_medium, utm_campaign) +SELECT session_id + 6651141467121565 * 4 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign +FROM default.sessions; + + +CREATE DATABASE massive2; +CREATE TABLE IF NOT EXISTS massive2.events7 +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + tracker_version LowCardinality(String), + rev_id Nullable(String), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(UInt8), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16), + user_id Nullable(String), + user_anonymous_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String) +) ENGINE = MergeTree + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime, event_type); + + + +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 43 +FROM massive.events6 +WHERE event_type = 'REQUEST' + AND mod(project_id, 2) = 0; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 42 +FROM massive.events6 +WHERE event_type = 'REQUEST' + AND mod(project_id, 2) = 1; + +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 41 +FROM massive.events6 +WHERE event_type = 'RESOURCE' + AND mod(project_id, 2) = 0; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 40 +FROM massive.events6 +WHERE event_type = 'RESOURCE' + AND mod(project_id, 2) = 1; + +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, hesitation_time, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + hesitation_time, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'CLICK'; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, + min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, + min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, + metadata_7, metadata_8, metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'PERFORMANCE'; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, request_start, response_start, response_end, + dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, + load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, + time_to_interactive, user_id, user_anonymous_id, metadata_1, metadata_2, metadata_3, + metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, + riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint, + speed_index, + visually_complete, + time_to_interactive, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'PAGE'; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, user_id, user_anonymous_id, metadata_1, metadata_2, + metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, + metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'INPUT'; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, source, name, message, error_id, user_id, user_anonymous_id, + metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, + metadata_8, metadata_9, metadata_10, riteration) +SELECT session_id + 6651141467121565 * 4 AS session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + source, + name, + message, + error_id, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'ERROR'; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, duration, context, container_type, container_id, container_name, + container_src, user_id, user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, + metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, riteration) +SELECT session_id + 6651141467121565 * 4 AS session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + context, + container_type, + container_id, + container_name, + container_src, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'LONGTASK'; + +ALTER TABLE massive2.events7 + ADD COLUMN riteration UInt8 DEFAULT 0; +ALTER TABLE massive2.sessions2 + ADD COLUMN riteration UInt8 DEFAULT 0; + + + +INSERT INTO massive2.sessions2(session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, + user_browser, user_browser_version, user_device, user_device_type, user_country, + datetime, + duration, events_count, errors_count, utm_source, utm_medium, utm_campaign, riteration) +SELECT session_id + 6651141467121565 * 4 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign, + 4 +FROM massive.sessions; + +SELECT COUNT(*) +FROM massive2.events7; + +CREATE DATABASE massive30; +CREATE TABLE IF NOT EXISTS massive30.events30 +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + tracker_version LowCardinality(String), + rev_id Nullable(String), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(UInt8), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16), + user_id Nullable(String), + user_anonymous_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String) +) ENGINE = MergeTree + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, event_type); + +ALTER TABLE massive30.events30 + ADD COLUMN riteration UInt8; + +INSERT INTO massive30.events30(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, hesitation_time, name, payload, level, source, message, + error_id, duration, context, container_type, container_id, container_name, container_src, + url, request_start, response_start, response_end, dom_content_loaded_event_start, + dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, + first_contentful_paint, speed_index, visually_complete, time_to_interactive, min_fps, + avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, + avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, + avg_used_js_heap_size, max_used_js_heap_size, type, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + hesitation_time, + name, + payload, + level, + source, + message, + error_id, + duration, + context, + container_type, + container_id, + container_name, + container_src, + url, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint, + speed_index, + visually_complete, + time_to_interactive, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + type, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 9 AS riteration +FROM massive2.events7 +WHERE mod(project_id, 10) = 9; +-- ORDER BY datetime LIMIT 500000; + +DROP TABLE massive30.events30; + + +DESCRIBE TABLE massive2.events7; + + +-- ----------------------------------------------------- +CREATE DATABASE massive_split; +CREATE TABLE IF NOT EXISTS massive_split.events_s +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(UInt8), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16) +) ENGINE = MergeTree + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, event_type, session_id); + +CREATE TABLE IF NOT EXISTS massive_split.sessions_s +( + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + duration UInt32, + pages_count UInt16, + events_count UInt16, + errors_count UInt16, + utm_source Nullable(String), + utm_medium Nullable(String), + utm_campaign Nullable(String), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 1 MONTH + SETTINGS index_granularity = 512; + + +CREATE TABLE IF NOT EXISTS massive_split.metadata_s +( + session_id UInt64, + project_id UInt32, + datetime DateTime, + user_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime, session_id); + +INSERT INTO massive_split.sessions_s(session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, duration, pages_count, events_count, errors_count, + utm_source, utm_medium, utm_campaign, riteration) +SELECT session_id + 6651141467121565 * 4 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + mod(rand(), 100) AS pages_count, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign, + 4 AS riteration +FROM default.sessions; + +ALTER TABLE massive_split.sessions_s + ADD COLUMN riteration UInt8; + +INSERT INTO massive_split.metadata_s(session_id, project_id, datetime, user_id, metadata_1, metadata_2, metadata_3, + metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, + metadata_10) +SELECT session_id, + project_id, + datetime, + user_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM massive2.events7 AS s +LIMIT 1 BY session_id; + +INSERT INTO massive_split.events_s(session_id, project_id, event_type, datetime, label, hesitation_time, name, payload, + level, source, message, error_id, duration, context, container_type, container_id, + container_name, container_src, url, request_start, response_start, response_end, + dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, + load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, + time_to_interactive, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, + min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, + min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, type, + header_size, encoded_body_size, decoded_body_size, success, method, status) +SELECT session_id, + project_id, + event_type, + datetime, + label, + hesitation_time, + name, + payload, + level, + source, + message, + error_id, + duration, + context, + container_type, + container_id, + container_name, + container_src, + url, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint, + speed_index, + visually_complete, + time_to_interactive, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + type, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status +FROM massive2.events7; + +SELECT COUNT(*) +FROM massive_split.sessions_s; +SELECT COUNT(*) +FROM massive_split.metadata_s; +SELECT COUNT(*) +FROM massive_split.events_s; +SELECT COUNT(*) +FROM massive2.events7; +-- SELECT COUNT(*) FROM massive2.sessions2; + + +CREATE TABLE IF NOT EXISTS massive_split.sessions_meta +( + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + duration UInt32, + pages_count UInt16, + events_count UInt16, + errors_count UInt16, + utm_source Nullable(String), + utm_medium Nullable(String), + utm_campaign Nullable(String), + user_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 1 MONTH + SETTINGS index_granularity = 512; + +INSERT INTO massive_split.sessions_meta(session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, + user_device_type, user_country, datetime, duration, pages_count, events_count, + errors_count, utm_source, utm_medium, utm_campaign, user_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, + metadata_8, metadata_9, metadata_10) +SELECT session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + pages_count, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign, + user_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM massive_split.sessions_s AS s + LEFT JOIN massive_split.metadata_s AS m ON (s.project_id = m.project_id AND s.session_id = m.session_id); diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/queries.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/queries.sql new file mode 100644 index 000000000..556209c79 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/queries.sql @@ -0,0 +1,983 @@ +-- Q1 +SELECT session_id +-- FROM massive2.events7 +-- FROM events_l7d_mv +FROM events_l24h_mv +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + + +-- Q1.1 +SELECT session_id +FROM massive2.events7 +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q1.2 +SELECT session_id +FROM +-- massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +events_l24h_mv + INNER JOIN metadata_l24h_mv USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q1.2.1 +SELECT session_id +FROM +-- massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- events_l7d_mv AS events_s +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) +events_l24h_mv AS events_s + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) +WHERE events_s.project_id = 2460 + AND events_s.datetime >= '2022-04-02 00:00:00' + AND events_s.datetime <= '2022-04-03 00:00:00' +-- AND events_s.datetime <= '2022-04-10 00:00:00' +-- AND events_s.datetime <= '2022-05-02 00:00:00' + AND metadata_s.project_id = 2460 + AND metadata_s.datetime >= '2022-04-02 00:00:00' + AND metadata_s.datetime <= '2022-04-03 00:00:00' +-- AND metadata_s.datetime <= '2022-04-10 00:00:00' +-- AND metadata_s.datetime <= '2022-05-02 00:00:00' + AND metadata_s.user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q1.3 +SELECT session_id +FROM +-- massive_split.events_s +-- events_l7d_mv +events_l24h_mv + INNER JOIN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q1.4 +SELECT session_id +FROM (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta + -- INNER JOIN massive_split.events_s USING (session_id) +-- INNER JOIN events_l7d_mv USING (session_id) + INNER JOIN events_l24h_mv USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q1.5 +SELECT session_id +-- FROM massive_split.events_s +-- FROM events_l7d_mv +FROM events_l24h_mv +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive2.events7 +-- FROM events_l7d_mv + FROM events_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' + -- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.1 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 + FROM massive2.events7 + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.2 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv +-- INNER JOIN metadata_l7d_mv USING (session_id) + FROM events_l24h_mv + INNER JOIN metadata_l24h_mv USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.2.1 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv AS events_s +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) + FROM events_l24h_mv AS events_s + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) + WHERE events_s.project_id = 2460 + AND events_s.datetime >= '2022-04-02 00:00:00' + AND events_s.datetime <= '2022-04-03 00:00:00' +-- AND events_s.datetime <= '2022-04-10 00:00:00' +-- AND events_s.datetime <= '2022-05-02 00:00:00' + AND metadata_s.project_id = 2460 + AND metadata_s.datetime >= '2022-04-02 00:00:00' + AND metadata_s.datetime <= '2022-04-03 00:00:00' +-- AND metadata_s.datetime <= '2022-04-10 00:00:00' +-- AND metadata_s.datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.3 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- FROM events_l7d_mv + FROM events_l24h_mv + INNER JOIN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' + -- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.4 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 + FROM (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta + -- INNER JOIN massive_split.events_s USING (session_id) +-- INNER JOIN events_l7d_mv USING (session_id) + INNER JOIN events_l24h_mv USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' + -- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.5 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- FROM events_l7d_mv + FROM events_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd')) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3 +SELECT session_id +-- FROM massive_split.events_s +-- FROM events_l7d_mv +FROM events_l24h_mv +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.1 +SELECT session_id +FROM massive2.events7 +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.2 +SELECT session_id +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv +-- INNER JOIN metadata_l7d_mv USING (session_id) +FROM events_l24h_mv + INNER JOIN metadata_l24h_mv USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.2.1 +SELECT session_id +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv AS events_s +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) +FROM events_l24h_mv AS events_s + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) +WHERE events_s.project_id = 2460 + AND events_s.datetime >= '2022-04-02 00:00:00' + AND events_s.datetime <= '2022-04-03 00:00:00' +-- AND events_s.datetime <= '2022-04-10 00:00:00' +-- AND events_s.datetime <= '2022-05-02 00:00:00' + AND (events_s.event_type = 'CLICK' OR events_s.event_type = 'REQUEST') + AND metadata_s.project_id = 2460 + AND metadata_s.datetime >= '2022-04-02 00:00:00' + AND metadata_s.datetime <= '2022-04-03 00:00:00' +-- AND metadata_s.datetime <= '2022-04-10 00:00:00' +-- AND metadata_s.datetime <= '2022-05-02 00:00:00' + AND metadata_s.user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.3 +SELECT session_id +-- FROM massive_split.events_s +-- FROM events_l7d_mv +FROM events_l24h_mv + INNER JOIN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.4 +SELECT session_id +FROM (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta + -- INNER JOIN massive_split.events_s USING (session_id) +-- INNER JOIN events_l7d_mv USING (session_id) + INNER JOIN events_l24h_mv USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.5 +SELECT session_id +-- FROM massive_split.events_s +-- FROM events_l7d_mv +FROM events_l24h_mv +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- FROM events_l7d_mv + FROM events_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST')) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.1 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 + FROM massive2.events7 + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.2 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv +-- INNER JOIN metadata_l7d_mv USING (session_id) + FROM events_l24h_mv + INNER JOIN metadata_l24h_mv USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.2.1 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv AS events_s +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) + FROM events_l24h_mv AS events_s + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) + WHERE events_s.project_id = 2460 + AND events_s.datetime >= '2022-04-02 00:00:00' + AND events_s.datetime <= '2022-04-03 00:00:00' +-- AND events_s.datetime <= '2022-04-10 00:00:00' +-- AND events_s.datetime <= '2022-05-02 00:00:00' + AND (events_s.event_type = 'CLICK' OR events_s.event_type = 'REQUEST') + AND metadata_s.user_id = 'uucUZvTpPd' + AND metadata_s.project_id = 2460 + AND metadata_s.datetime >= '2022-04-02 00:00:00' + AND metadata_s.datetime <= '2022-04-03 00:00:00' + -- AND metadata_s.datetime <= '2022-04-10 00:00:00' +-- AND metadata_s.datetime <= '2022-05-02 00:00:00' + ) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.3 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- FROM events_l7d_mv + FROM events_l24h_mv + INNER JOIN (SELECT DISTINCT session_id +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST')) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.4 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 + FROM (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta + -- INNER JOIN massive_split.events_s USING (session_id) +-- INNER JOIN events_l7d_mv USING (session_id) + INNER JOIN events_l24h_mv USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST')) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.5 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 + FROM events_l24h_mv +-- FROM events_l7d_mv +-- FROM massive_split.events_s + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd')) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, session_id + FROM massive2.events7 AS events + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.1 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, session_id + FROM massive2.events7 AS events + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.2 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv AS events +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) + FROM events_l24h_mv AS events + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.3 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv AS events +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) + FROM events_l24h_mv AS events + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.4 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- FROM events_l7d_mv AS events + FROM events_l24h_mv AS events + INNER JOIN (SELECT DISTINCT session_id, + user_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.4-A +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- FROM events_l7d_mv AS events + FROM events_l24h_mv AS events + INNER JOIN (SELECT DISTINCT session_id, + user_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' + -- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) AS meta USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.5 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, session_id + FROM (SELECT DISTINCT session_id, + user_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta + -- INNER JOIN massive_split.events_s AS events USING (session_id) +-- INNER JOIN events_l7d_mv AS events USING (session_id) + INNER JOIN events_l24h_mv AS events USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.6 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- FROM events_l7d_mv AS events + FROM events_l24h_mv AS events + INNER JOIN (SELECT DISTINCT session_id, + user_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.6-A +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- FROM events_l7d_mv AS events + FROM events_l24h_mv AS events + INNER JOIN (SELECT DISTINCT session_id, + user_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' + -- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) AS meta USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QM4: +SELECT timestamp, + groupArray([toString(t.type), toString(t.count)]) AS types +FROM (SELECT toUnixTimestamp(toStartOfInterval(events7.datetime, INTERVAL 37565 second)) * 1000 AS timestamp, + events7.type, + COUNT(events7.session_id) AS count +-- FROM massive_split.events_s AS events7 +-- FROM events_l7d_mv AS events7 + FROM events_l24h_mv AS events7 + WHERE events7.project_id = toUInt32(2460) + AND toStartOfInterval(events7.datetime, INTERVAL 37565 second) >= '2022-04-02 00:00:00' + AND events7.datetime <= '2022-04-03 00:00:00' +-- AND events7.datetime <= '2022-04-10 00:00:00' +-- AND events7.datetime < '2022-05-02 00:00:00' + AND events7.event_type = 'RESOURCE' + GROUP BY timestamp, events7.type + ORDER BY timestamp) AS t +GROUP BY timestamp + SETTINGS + max_threads = 4;