From 053ec57a7980b2f0b3efdb88fd3031147ef42211 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 26 Aug 2022 17:03:26 +0100 Subject: [PATCH] feat(DB): changed CH MVs --- .../clickhouse/1.8.0/exp_3_events_mv.sql | 65 +++++++++++++++++-- .../clickhouse/1.8.0/exp_3_resources_mv.sql | 20 +++++- .../clickhouse/1.8.0/exp_3_sessions_mv.sql | 36 +++++++++- .../clickhouse/create/exp_3_events_mv.sql | 65 +++++++++++++++++-- .../clickhouse/create/exp_3_resources_mv.sql | 20 +++++- .../clickhouse/create/exp_3_sessions_mv.sql | 36 +++++++++- 6 files changed, 228 insertions(+), 14 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_events_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_events_mv.sql index 136aea1c5..9605c0c09 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_events_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_events_mv.sql @@ -1,10 +1,67 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv ENGINE = MergeTree - PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, session_id) - TTL datetime + INTERVAL 7 DAY + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, event_type, session_id) + TTL datetime + INTERVAL 7 DAY POPULATE AS -SELECT * +SELECT session_id, + project_id, + event_type, + datetime, + label, + hesitation_time, + name, + payload, + level, + source, + message, + error_id, + duration, + context, + container_type, + container_id, + container_name, + container_src, + url, + url_host, + url_path, + url_hostpath, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint_time, + speed_index, + visually_complete, + time_to_interactive, + ttfb, + ttlb, + response_time, + dom_building_time, + dom_content_loaded_event_time, + load_event_time, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + method, + status, + success, + request_body, + response_body, + _timestamp FROM experimental.events WHERE datetime >= now() - INTERVAL 7 DAY; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_resources_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_resources_mv.sql index a5a6fb403..839ffa4f7 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_resources_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_resources_mv.sql @@ -1,10 +1,26 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.resources_l7d_mv ENGINE = MergeTree PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, session_id) + ORDER BY (project_id, datetime, type, session_id) TTL datetime + INTERVAL 7 DAY POPULATE AS -SELECT * +SELECT session_id, + project_id, + datetime, + url, + url_host, + url_path, + url_hostpath, + type, + name, + duration, + ttfb, + header_size, + encoded_body_size, + decoded_body_size, + compression_ratio, + success, + _timestamp FROM experimental.resources WHERE datetime >= now() - INTERVAL 7 DAY; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_sessions_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_sessions_mv.sql index eea689374..6d0a2176c 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_sessions_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_3_sessions_mv.sql @@ -6,7 +6,41 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.sessions_l7d_mv SETTINGS index_granularity = 512 POPULATE AS -SELECT * +SELECT session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + pages_count, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign, + user_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + issue_types, + referrer, + base_referrer, + _timestamp FROM experimental.sessions WHERE datetime >= now() - INTERVAL 7 DAY AND isNotNull(duration) diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_events_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_events_mv.sql index 136aea1c5..9605c0c09 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_events_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_events_mv.sql @@ -1,10 +1,67 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv ENGINE = MergeTree - PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, session_id) - TTL datetime + INTERVAL 7 DAY + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, event_type, session_id) + TTL datetime + INTERVAL 7 DAY POPULATE AS -SELECT * +SELECT session_id, + project_id, + event_type, + datetime, + label, + hesitation_time, + name, + payload, + level, + source, + message, + error_id, + duration, + context, + container_type, + container_id, + container_name, + container_src, + url, + url_host, + url_path, + url_hostpath, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint_time, + speed_index, + visually_complete, + time_to_interactive, + ttfb, + ttlb, + response_time, + dom_building_time, + dom_content_loaded_event_time, + load_event_time, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + method, + status, + success, + request_body, + response_body, + _timestamp FROM experimental.events WHERE datetime >= now() - INTERVAL 7 DAY; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_resources_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_resources_mv.sql index a5a6fb403..839ffa4f7 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_resources_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_resources_mv.sql @@ -1,10 +1,26 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.resources_l7d_mv ENGINE = MergeTree PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, session_id) + ORDER BY (project_id, datetime, type, session_id) TTL datetime + INTERVAL 7 DAY POPULATE AS -SELECT * +SELECT session_id, + project_id, + datetime, + url, + url_host, + url_path, + url_hostpath, + type, + name, + duration, + ttfb, + header_size, + encoded_body_size, + decoded_body_size, + compression_ratio, + success, + _timestamp FROM experimental.resources WHERE datetime >= now() - INTERVAL 7 DAY; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_sessions_mv.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_sessions_mv.sql index eea689374..6d0a2176c 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_sessions_mv.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/exp_3_sessions_mv.sql @@ -6,7 +6,41 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.sessions_l7d_mv SETTINGS index_granularity = 512 POPULATE AS -SELECT * +SELECT session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + pages_count, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign, + user_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + issue_types, + referrer, + base_referrer, + _timestamp FROM experimental.sessions WHERE datetime >= now() - INTERVAL 7 DAY AND isNotNull(duration)