From 03c30d0a2831b457519a8398be6ff109545584e2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 2 Sep 2022 09:15:50 +0100 Subject: [PATCH 1/3] feat(DB): removed longtask from the list of supported events --- ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql | 2 +- ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql index 912b1b7e6..1ebfabeb1 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS experimental.events ( session_id UInt64, project_id UInt16, - event_type Enum8('CLICK'=0, 'INPUT'=1, 'LOCATION'=2,'REQUEST'=3,'PERFORMANCE'=4,'LONGTASK'=5,'ERROR'=6,'CUSTOM'=7, 'GRAPHQL'=8, 'STATEACTION'=9), + event_type Enum8('CLICK'=0, 'INPUT'=1, 'LOCATION'=2,'REQUEST'=3,'PERFORMANCE'=4,'ERROR'=5,'CUSTOM'=6, 'GRAPHQL'=7, 'STATEACTION'=8), datetime DateTime, label Nullable(String), hesitation_time Nullable(UInt32), diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql index 912b1b7e6..1ebfabeb1 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS experimental.events ( session_id UInt64, project_id UInt16, - event_type Enum8('CLICK'=0, 'INPUT'=1, 'LOCATION'=2,'REQUEST'=3,'PERFORMANCE'=4,'LONGTASK'=5,'ERROR'=6,'CUSTOM'=7, 'GRAPHQL'=8, 'STATEACTION'=9), + event_type Enum8('CLICK'=0, 'INPUT'=1, 'LOCATION'=2,'REQUEST'=3,'PERFORMANCE'=4,'ERROR'=5,'CUSTOM'=6, 'GRAPHQL'=7, 'STATEACTION'=8), datetime DateTime, label Nullable(String), hesitation_time Nullable(UInt32), From 586716327a27197958930490311ee28407301e87 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 2 Sep 2022 11:09:10 +0100 Subject: [PATCH 2/3] feat(DB): CH added message_id column for events and resources --- .../helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql | 16 ++++++++++------ .../init_dbs/clickhouse/create/init_schema.sql | 16 ++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql index 1ebfabeb1..3fc31a479 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql @@ -78,10 +78,11 @@ CREATE TABLE IF NOT EXISTS experimental.events success Nullable(UInt8), request_body Nullable(String), response_body Nullable(String), + message_id UInt64 DEFAULT 0, _timestamp DateTime DEFAULT now() -) ENGINE = MergeTree +) ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, event_type, session_id) + ORDER BY (project_id, datetime, event_type, session_id, message_id) TTL datetime + INTERVAL 3 MONTH; CREATE TABLE IF NOT EXISTS experimental.resources @@ -104,10 +105,11 @@ CREATE TABLE IF NOT EXISTS experimental.resources decoded_body_size Nullable(UInt32), compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), success Nullable(UInt8) COMMENT 'currently available for type=img only', + message_id UInt64 DEFAULT 0, _timestamp DateTime DEFAULT now() -) ENGINE = MergeTree +) ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, type, session_id) + ORDER BY (project_id, datetime, type, session_id, message_id) TTL datetime + INTERVAL 3 MONTH; CREATE TABLE IF NOT EXISTS experimental.sessions @@ -191,7 +193,7 @@ CREATE TABLE IF NOT EXISTS experimental.user_viewed_errors TTL _timestamp + INTERVAL 3 MONTH; CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv - ENGINE = MergeTree + ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) ORDER BY (project_id, datetime, event_type, session_id) TTL datetime + INTERVAL 7 DAY @@ -254,12 +256,13 @@ SELECT session_id, success, request_body, response_body, + message_id, _timestamp FROM experimental.events WHERE datetime >= now() - INTERVAL 7 DAY; CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.resources_l7d_mv - ENGINE = MergeTree + ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) ORDER BY (project_id, datetime, type, session_id) TTL datetime + INTERVAL 7 DAY @@ -281,6 +284,7 @@ SELECT session_id, decoded_body_size, compression_ratio, success, + message_id, _timestamp FROM experimental.resources WHERE datetime >= now() - INTERVAL 7 DAY; diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql index 1ebfabeb1..3fc31a479 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql @@ -78,10 +78,11 @@ CREATE TABLE IF NOT EXISTS experimental.events success Nullable(UInt8), request_body Nullable(String), response_body Nullable(String), + message_id UInt64 DEFAULT 0, _timestamp DateTime DEFAULT now() -) ENGINE = MergeTree +) ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, event_type, session_id) + ORDER BY (project_id, datetime, event_type, session_id, message_id) TTL datetime + INTERVAL 3 MONTH; CREATE TABLE IF NOT EXISTS experimental.resources @@ -104,10 +105,11 @@ CREATE TABLE IF NOT EXISTS experimental.resources decoded_body_size Nullable(UInt32), compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), success Nullable(UInt8) COMMENT 'currently available for type=img only', + message_id UInt64 DEFAULT 0, _timestamp DateTime DEFAULT now() -) ENGINE = MergeTree +) ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, type, session_id) + ORDER BY (project_id, datetime, type, session_id, message_id) TTL datetime + INTERVAL 3 MONTH; CREATE TABLE IF NOT EXISTS experimental.sessions @@ -191,7 +193,7 @@ CREATE TABLE IF NOT EXISTS experimental.user_viewed_errors TTL _timestamp + INTERVAL 3 MONTH; CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv - ENGINE = MergeTree + ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) ORDER BY (project_id, datetime, event_type, session_id) TTL datetime + INTERVAL 7 DAY @@ -254,12 +256,13 @@ SELECT session_id, success, request_body, response_body, + message_id, _timestamp FROM experimental.events WHERE datetime >= now() - INTERVAL 7 DAY; CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.resources_l7d_mv - ENGINE = MergeTree + ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) ORDER BY (project_id, datetime, type, session_id) TTL datetime + INTERVAL 7 DAY @@ -281,6 +284,7 @@ SELECT session_id, decoded_body_size, compression_ratio, success, + message_id, _timestamp FROM experimental.resources WHERE datetime >= now() - INTERVAL 7 DAY; From 46c46fb71e53306e13c7f352c35d60c4feccb6b0 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 2 Sep 2022 11:19:09 +0100 Subject: [PATCH 3/3] feat(DB): CH added message_id column to MV sort key --- ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql | 4 ++-- ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql index 3fc31a479..01f3a9968 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql @@ -195,7 +195,7 @@ CREATE TABLE IF NOT EXISTS experimental.user_viewed_errors CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, event_type, session_id) + ORDER BY (project_id, datetime, event_type, session_id, message_id) TTL datetime + INTERVAL 7 DAY POPULATE AS @@ -264,7 +264,7 @@ WHERE datetime >= now() - INTERVAL 7 DAY; CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.resources_l7d_mv ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, type, session_id) + ORDER BY (project_id, datetime, type, session_id, message_id) TTL datetime + INTERVAL 7 DAY POPULATE AS diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql index 3fc31a479..01f3a9968 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql @@ -195,7 +195,7 @@ CREATE TABLE IF NOT EXISTS experimental.user_viewed_errors CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, event_type, session_id) + ORDER BY (project_id, datetime, event_type, session_id, message_id) TTL datetime + INTERVAL 7 DAY POPULATE AS @@ -264,7 +264,7 @@ WHERE datetime >= now() - INTERVAL 7 DAY; CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.resources_l7d_mv ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, type, session_id) + ORDER BY (project_id, datetime, type, session_id, message_id) TTL datetime + INTERVAL 7 DAY POPULATE AS