From 3a331d266cd5c902f7e374de3c36f2540a125d0e Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Fri, 21 Feb 2025 12:56:05 +0100 Subject: [PATCH] Dev (#3052) * fix(chalice): support wrong payload for user-journey drill-down * fix(chalice): ignore event's MV for v1.22 refactor(DB): ignore event's MV for v1.22 --- api/schemas/transformers_validators.py | 3 + ee/api/chalicelib/utils/exp_ch_helper.py | 7 +- .../db/init_dbs/clickhouse/1.22.0/1.22.0.sql | 3 + .../clickhouse/create/init_schema.sql | 72 ------------------ .../rollback_dbs/clickhouse/1.22.0/1.22.0.sql | 74 +++++++++++++++++++ 5 files changed, 84 insertions(+), 75 deletions(-) diff --git a/api/schemas/transformers_validators.py b/api/schemas/transformers_validators.py index e50371aa4..106f8747c 100644 --- a/api/schemas/transformers_validators.py +++ b/api/schemas/transformers_validators.py @@ -38,6 +38,9 @@ def force_is_event(events_enum: list[Type[Enum]]): def fn(value: list): if value is not None and isinstance(value, list): for v in value: + if v.get("type") is None: + v["isEvent"] = False + continue r = False for en in events_enum: if en.has_value(v["type"]) or en.has_value(v["type"].lower()): diff --git a/ee/api/chalicelib/utils/exp_ch_helper.py b/ee/api/chalicelib/utils/exp_ch_helper.py index 282351908..1e19bed2c 100644 --- a/ee/api/chalicelib/utils/exp_ch_helper.py +++ b/ee/api/chalicelib/utils/exp_ch_helper.py @@ -13,9 +13,10 @@ if config("EXP_7D_MV", cast=bool, default=True): def get_main_events_table(timestamp=0, platform="web"): if platform == "web": - return "experimental.events_l7d_mv" \ - if config("EXP_7D_MV", cast=bool, default=True) \ - and timestamp and timestamp >= TimeUTC.now(delta_days=-7) else "experimental.events" + return "product_analytics.events" + # return "experimental.events_l7d_mv" \ + # if config("EXP_7D_MV", cast=bool, default=True) \ + # and timestamp and timestamp >= TimeUTC.now(delta_days=-7) else "experimental.events" else: return "experimental.ios_events" diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql b/ee/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql index 67231a79f..27aadcf39 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql @@ -334,3 +334,6 @@ CREATE TABLE IF NOT EXISTS product_analytics.all_properties _timestamp DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(_timestamp) ORDER BY (project_id, property_name, is_event_property); + + +DROP TABLE IF EXISTS experimental.events_l7d_mv; \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index 413595d79..1fd1a79e7 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -191,78 +191,6 @@ CREATE TABLE IF NOT EXISTS experimental.issues ORDER BY (project_id, issue_id, type) TTL _timestamp + INTERVAL 3 MONTH; -CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv - ENGINE = ReplacingMergeTree(_timestamp) - PARTITION BY toYYYYMMDD(datetime) - ORDER BY (project_id, datetime, event_type, session_id, message_id) - TTL datetime + INTERVAL 7 DAY - POPULATE -AS -SELECT session_id, - project_id, - event_type, - datetime, - label, - hesitation_time, - name, - payload, - level, - source, - message, - error_id, - duration, - context, - url, - url_host, - url_path, - url_hostpath, - request_start, - response_start, - response_end, - dom_content_loaded_event_start, - dom_content_loaded_event_end, - load_event_start, - load_event_end, - first_paint, - first_contentful_paint_time, - speed_index, - visually_complete, - time_to_interactive, - ttfb, - ttlb, - response_time, - dom_building_time, - dom_content_loaded_event_time, - load_event_time, - min_fps, - avg_fps, - max_fps, - min_cpu, - avg_cpu, - max_cpu, - min_total_js_heap_size, - avg_total_js_heap_size, - max_total_js_heap_size, - min_used_js_heap_size, - avg_used_js_heap_size, - max_used_js_heap_size, - method, - status, - success, - request_body, - response_body, - issue_type, - issue_id, - error_tags_keys, - error_tags_values, - transfer_size, - selector, - normalized_x, - normalized_y, - message_id, - _timestamp -FROM experimental.events -WHERE datetime >= now() - INTERVAL 7 DAY; CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.sessions_l7d_mv diff --git a/ee/scripts/schema/db/rollback_dbs/clickhouse/1.22.0/1.22.0.sql b/ee/scripts/schema/db/rollback_dbs/clickhouse/1.22.0/1.22.0.sql index c60511f00..5c7f64a56 100644 --- a/ee/scripts/schema/db/rollback_dbs/clickhouse/1.22.0/1.22.0.sql +++ b/ee/scripts/schema/db/rollback_dbs/clickhouse/1.22.0/1.22.0.sql @@ -1 +1,75 @@ CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.21.0-ee'; + + +CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv + ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, event_type, session_id, message_id) + TTL datetime + INTERVAL 7 DAY + POPULATE +AS +SELECT session_id, + project_id, + event_type, + datetime, + label, + hesitation_time, + name, + payload, + level, + source, + message, + error_id, + duration, + context, + url, + url_host, + url_path, + url_hostpath, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint_time, + speed_index, + visually_complete, + time_to_interactive, + ttfb, + ttlb, + response_time, + dom_building_time, + dom_content_loaded_event_time, + load_event_time, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + method, + status, + success, + request_body, + response_body, + issue_type, + issue_id, + error_tags_keys, + error_tags_values, + transfer_size, + selector, + normalized_x, + normalized_y, + message_id, + _timestamp +FROM experimental.events +WHERE datetime >= now() - INTERVAL 7 DAY; \ No newline at end of file