jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(DB): remove TTL for CH tables

Browse source
This commit is contained in:
Taha Yassine Kraiem 2025-05-05 16:48:02 +02:00 • committed by Kraiem Taha Yassine
parent 4b1ca200b4
commit d78b33dcd2
4 changed files with 20 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
ee/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql
View file

@ -151,8 +151,7 @@ CREATE TABLE IF NOT EXISTS product_analytics.events
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
ORDER BY (project_id, "$event_name", created_at, session_id) ORDER BY (project_id, "$event_name", created_at, session_id)
TTL _timestamp + INTERVAL 1 MONTH , TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00';
_deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00';
-- The list of events that should not be ingested, -- The list of events that should not be ingested,
-- according to a specific event_name and optional properties -- according to a specific event_name and optional properties

28
ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql
View file

@ -9,8 +9,7 @@ CREATE TABLE IF NOT EXISTS experimental.autocomplete
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(_timestamp) PARTITION BY toYYYYMM(_timestamp)
ORDER BY (project_id, type, value) ORDER BY (project_id, type, value);
TTL _timestamp + INTERVAL 1 MONTH;
CREATE TABLE IF NOT EXISTS experimental.events CREATE TABLE IF NOT EXISTS experimental.events
( (
@ -87,8 +86,7 @@ CREATE TABLE IF NOT EXISTS experimental.events
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(datetime) PARTITION BY toYYYYMM(datetime)
ORDER BY (project_id, datetime, event_type, session_id, message_id) ORDER BY (project_id, datetime, event_type, session_id, message_id);
TTL datetime + INTERVAL 3 MONTH;
@ -140,7 +138,6 @@ CREATE TABLE IF NOT EXISTS experimental.sessions
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMMDD(datetime) PARTITION BY toYYYYMMDD(datetime)
ORDER BY (project_id, datetime, session_id) ORDER BY (project_id, datetime, session_id)
TTL datetime + INTERVAL 3 MONTH
SETTINGS index_granularity = 512; SETTINGS index_granularity = 512;
CREATE TABLE IF NOT EXISTS experimental.user_favorite_sessions CREATE TABLE IF NOT EXISTS experimental.user_favorite_sessions
@ -152,8 +149,7 @@ CREATE TABLE IF NOT EXISTS experimental.user_favorite_sessions
sign Int8 sign Int8
) ENGINE = CollapsingMergeTree(sign) ) ENGINE = CollapsingMergeTree(sign)
PARTITION BY toYYYYMM(_timestamp) PARTITION BY toYYYYMM(_timestamp)
ORDER BY (project_id, user_id, session_id) ORDER BY (project_id, user_id, session_id);
TTL _timestamp + INTERVAL 3 MONTH;
CREATE TABLE IF NOT EXISTS experimental.user_viewed_sessions CREATE TABLE IF NOT EXISTS experimental.user_viewed_sessions
( (
@ -163,8 +159,7 @@ CREATE TABLE IF NOT EXISTS experimental.user_viewed_sessions
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(_timestamp) PARTITION BY toYYYYMM(_timestamp)
ORDER BY (project_id, user_id, session_id) ORDER BY (project_id, user_id, session_id);
TTL _timestamp + INTERVAL 3 MONTH;
CREATE TABLE IF NOT EXISTS experimental.user_viewed_errors CREATE TABLE IF NOT EXISTS experimental.user_viewed_errors
( (
@ -174,8 +169,7 @@ CREATE TABLE IF NOT EXISTS experimental.user_viewed_errors
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(_timestamp) PARTITION BY toYYYYMM(_timestamp)
ORDER BY (project_id, user_id, error_id) ORDER BY (project_id, user_id, error_id);
TTL _timestamp + INTERVAL 3 MONTH;
CREATE TABLE IF NOT EXISTS experimental.issues CREATE TABLE IF NOT EXISTS experimental.issues
( (
@ -188,8 +182,7 @@ CREATE TABLE IF NOT EXISTS experimental.issues
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(_timestamp) PARTITION BY toYYYYMM(_timestamp)
ORDER BY (project_id, issue_id, type) ORDER BY (project_id, issue_id, type);
TTL _timestamp + INTERVAL 3 MONTH;
@ -292,8 +285,7 @@ CREATE TABLE IF NOT EXISTS experimental.sessions_feature_flags
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(datetime) PARTITION BY toYYYYMM(datetime)
ORDER BY (project_id, datetime, session_id, feature_flag_id, condition_id) ORDER BY (project_id, datetime, session_id, feature_flag_id, condition_id);
TTL datetime + INTERVAL 3 MONTH;
CREATE TABLE IF NOT EXISTS experimental.ios_events CREATE TABLE IF NOT EXISTS experimental.ios_events
( (
@ -329,8 +321,7 @@ CREATE TABLE IF NOT EXISTS experimental.ios_events
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(datetime) PARTITION BY toYYYYMM(datetime)
ORDER BY (project_id, datetime, event_type, session_id, message_id) ORDER BY (project_id, datetime, event_type, session_id, message_id);
TTL datetime + INTERVAL 3 MONTH;
SET allow_experimental_json_type = 1; SET allow_experimental_json_type = 1;
@ -484,8 +475,7 @@ CREATE TABLE IF NOT EXISTS product_analytics.events
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
ORDER BY (project_id, "$event_name", created_at, session_id) ORDER BY (project_id, "$event_name", created_at, session_id)
TTL _timestamp + INTERVAL 1 MONTH , TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00';
_deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00';
-- The list of events that should not be ingested, -- The list of events that should not be ingested,
-- according to a specific event_name and optional properties -- according to a specific event_name and optional properties

16
scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql
View file

@ -9,8 +9,7 @@ CREATE TABLE IF NOT EXISTS experimental.autocomplete
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(_timestamp) PARTITION BY toYYYYMM(_timestamp)
ORDER BY (project_id, type, value) ORDER BY (project_id, type, value);
TTL _timestamp + INTERVAL 1 MONTH;
CREATE TABLE IF NOT EXISTS experimental.events CREATE TABLE IF NOT EXISTS experimental.events
( (
@ -87,8 +86,7 @@ CREATE TABLE IF NOT EXISTS experimental.events
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(datetime) PARTITION BY toYYYYMM(datetime)
ORDER BY (project_id, datetime, event_type, session_id, message_id) ORDER BY (project_id, datetime, event_type, session_id, message_id);
TTL datetime + INTERVAL 1 MONTH;
@ -140,7 +138,6 @@ CREATE TABLE IF NOT EXISTS experimental.sessions
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMMDD(datetime) PARTITION BY toYYYYMMDD(datetime)
ORDER BY (project_id, datetime, session_id) ORDER BY (project_id, datetime, session_id)
TTL datetime + INTERVAL 1 MONTH
SETTINGS index_granularity = 512; SETTINGS index_granularity = 512;
CREATE TABLE IF NOT EXISTS experimental.issues CREATE TABLE IF NOT EXISTS experimental.issues
@ -154,8 +151,7 @@ CREATE TABLE IF NOT EXISTS experimental.issues
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(_timestamp) PARTITION BY toYYYYMM(_timestamp)
ORDER BY (project_id, issue_id, type) ORDER BY (project_id, issue_id, type);
TTL _timestamp + INTERVAL 1 MONTH;
CREATE TABLE IF NOT EXISTS experimental.ios_events CREATE TABLE IF NOT EXISTS experimental.ios_events
( (
@ -191,8 +187,7 @@ CREATE TABLE IF NOT EXISTS experimental.ios_events
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(datetime) PARTITION BY toYYYYMM(datetime)
ORDER BY (project_id, datetime, event_type, session_id, message_id) ORDER BY (project_id, datetime, event_type, session_id, message_id);
TTL datetime + INTERVAL 1 MONTH;
SET allow_experimental_json_type = 1; SET allow_experimental_json_type = 1;
@ -345,8 +340,7 @@ CREATE TABLE IF NOT EXISTS product_analytics.events
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
ORDER BY (project_id, "$event_name", created_at, session_id) ORDER BY (project_id, "$event_name", created_at, session_id)
TTL _timestamp + INTERVAL 1 MONTH , TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00';
_deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00';
-- The list of events that should not be ingested, -- The list of events that should not be ingested,
-- according to a specific event_name and optional properties -- according to a specific event_name and optional properties

16
scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql
View file

@ -9,8 +9,7 @@ CREATE TABLE IF NOT EXISTS experimental.autocomplete
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(_timestamp) PARTITION BY toYYYYMM(_timestamp)
ORDER BY (project_id, type, value) ORDER BY (project_id, type, value);
TTL _timestamp + INTERVAL 1 MONTH;
CREATE TABLE IF NOT EXISTS experimental.events CREATE TABLE IF NOT EXISTS experimental.events
( (
@ -87,8 +86,7 @@ CREATE TABLE IF NOT EXISTS experimental.events
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(datetime) PARTITION BY toYYYYMM(datetime)
ORDER BY (project_id, datetime, event_type, session_id, message_id) ORDER BY (project_id, datetime, event_type, session_id, message_id);
TTL datetime + INTERVAL 1 MONTH;
@ -140,7 +138,6 @@ CREATE TABLE IF NOT EXISTS experimental.sessions
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMMDD(datetime) PARTITION BY toYYYYMMDD(datetime)
ORDER BY (project_id, datetime, session_id) ORDER BY (project_id, datetime, session_id)
TTL datetime + INTERVAL 1 MONTH
SETTINGS index_granularity = 512; SETTINGS index_granularity = 512;
CREATE TABLE IF NOT EXISTS experimental.issues CREATE TABLE IF NOT EXISTS experimental.issues
@ -154,8 +151,7 @@ CREATE TABLE IF NOT EXISTS experimental.issues
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(_timestamp) PARTITION BY toYYYYMM(_timestamp)
ORDER BY (project_id, issue_id, type) ORDER BY (project_id, issue_id, type);
TTL _timestamp + INTERVAL 1 MONTH;
CREATE TABLE IF NOT EXISTS experimental.ios_events CREATE TABLE IF NOT EXISTS experimental.ios_events
( (
@ -191,8 +187,7 @@ CREATE TABLE IF NOT EXISTS experimental.ios_events
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
PARTITION BY toYYYYMM(datetime) PARTITION BY toYYYYMM(datetime)
ORDER BY (project_id, datetime, event_type, session_id, message_id) ORDER BY (project_id, datetime, event_type, session_id, message_id);
TTL datetime + INTERVAL 1 MONTH;
SET allow_experimental_json_type = 1; SET allow_experimental_json_type = 1;
@ -346,8 +341,7 @@ CREATE TABLE IF NOT EXISTS product_analytics.events
_timestamp DateTime DEFAULT now() _timestamp DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(_timestamp) ) ENGINE = ReplacingMergeTree(_timestamp)
ORDER BY (project_id, "$event_name", created_at, session_id) ORDER BY (project_id, "$event_name", created_at, session_id)
TTL _timestamp + INTERVAL 1 MONTH , TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00';
_deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00';
-- The list of events that should not be ingested, -- The list of events that should not be ingested,
-- according to a specific event_name and optional properties -- according to a specific event_name and optional properties