jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(DB): structured JSON attributes

Browse source
This commit is contained in:
Taha Yassine Kraiem 2025-04-28 15:35:13 +01:00 • committed by Kraiem Taha Yassine
parent 9b9248f1f6
commit 9f6dc788c4
3 changed files with 176 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
ee/scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql
View file

@ -1,5 +1,64 @@
CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.23.0-ee';
SET allow_experimental_json_type = 1;
SET enable_json_type = 1;
ALTER TABLE product_analytics.events
MODIFY COLUMN `$properties` JSON(
max_dynamic_paths=0,
label String ,
hesitation_time UInt32 ,
name String ,
payload String ,
level Enum8 ('info'=0, 'error'=1),
source Enum8 ('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9),
message String ,
error_id String ,
duration UInt16,
context Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8),
url_host String ,
url_path String ,
url_hostpath String ,
request_start UInt16 ,
response_start UInt16 ,
response_end UInt16 ,
dom_content_loaded_event_start UInt16 ,
dom_content_loaded_event_end UInt16 ,
load_event_start UInt16 ,
load_event_end UInt16 ,
first_paint UInt16 ,
first_contentful_paint_time UInt16 ,
speed_index UInt16 ,
visually_complete UInt16 ,
time_to_interactive UInt16,
ttfb UInt16,
ttlb UInt16,
response_time UInt16,
dom_building_time UInt16,
dom_content_loaded_event_time UInt16,
load_event_time UInt16,
min_fps UInt8,
avg_fps UInt8,
max_fps UInt8,
min_cpu UInt8,
avg_cpu UInt8,
max_cpu UInt8,
min_total_js_heap_size UInt64,
avg_total_js_heap_size UInt64,
max_total_js_heap_size UInt64,
min_used_js_heap_size UInt64,
avg_used_js_heap_size UInt64,
max_used_js_heap_size UInt64,
method Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8),
status UInt16,
success UInt8,
request_body String,
response_body String,
transfer_size UInt32,
selector String,
normalized_x Float32,
normalized_y Float32,
message_id UInt64
) DEFAULT '{}' COMMENT 'these properties belongs to the auto-captured events';
DROP TABLE IF EXISTS product_analytics.all_events;
CREATE TABLE IF NOT EXISTS product_analytics.all_events

63
scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql
View file

@ -12,6 +12,66 @@ CREATE TABLE IF NOT EXISTS experimental.user_viewed_sessions
ORDER BY (project_id, user_id, session_id)
TTL _timestamp + INTERVAL 3 MONTH;
SET allow_experimental_json_type = 1;
SET enable_json_type = 1;
ALTER TABLE product_analytics.events
MODIFY COLUMN `$properties` JSON(
max_dynamic_paths=0,
label String ,
hesitation_time UInt32 ,
name String ,
payload String ,
level Enum8 ('info'=0, 'error'=1),
source Enum8 ('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9),
message String ,
error_id String ,
duration UInt16,
context Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8),
url_host String ,
url_path String ,
url_hostpath String ,
request_start UInt16 ,
response_start UInt16 ,
response_end UInt16 ,
dom_content_loaded_event_start UInt16 ,
dom_content_loaded_event_end UInt16 ,
load_event_start UInt16 ,
load_event_end UInt16 ,
first_paint UInt16 ,
first_contentful_paint_time UInt16 ,
speed_index UInt16 ,
visually_complete UInt16 ,
time_to_interactive UInt16,
ttfb UInt16,
ttlb UInt16,
response_time UInt16,
dom_building_time UInt16,
dom_content_loaded_event_time UInt16,
load_event_time UInt16,
min_fps UInt8,
avg_fps UInt8,
max_fps UInt8,
min_cpu UInt8,
avg_cpu UInt8,
max_cpu UInt8,
min_total_js_heap_size UInt64,
avg_total_js_heap_size UInt64,
max_total_js_heap_size UInt64,
min_used_js_heap_size UInt64,
avg_used_js_heap_size UInt64,
max_used_js_heap_size UInt64,
method Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8),
status UInt16,
success UInt8,
request_body String,
response_body String,
transfer_size UInt32,
selector String,
normalized_x Float32,
normalized_y Float32,
message_id UInt64
) DEFAULT '{}' COMMENT 'these properties belongs to the auto-captured events';
DROP TABLE IF EXISTS product_analytics.all_events;
CREATE TABLE IF NOT EXISTS product_analytics.all_events
(
@ -156,8 +216,7 @@ CREATE TABLE IF NOT EXISTS product_analytics.property_values_samples
ENGINE = ReplacingMergeTree(_timestamp)
ORDER BY (project_id, property_name, is_event_property);
CREATE MATERIALIZED VIEW IF NOT EXISTS product_analytics.property_values_sampler_mv
REFRESH EVERY 30 HOUR TO product_analytics.property_values_samples AS
CREATE MATERIALIZED VIEW IF NOT EXISTS product_analytics.property_values_sampler_mvREFRESHEVERY30HOURTOproduct_analytics.property_values_samples AS
SELECT project_id,
property_name,
TRUE AS is_event_property,

57
scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql
View file

@ -335,7 +335,62 @@ CREATE TABLE IF NOT EXISTS product_analytics.events
"$source" LowCardinality(String) DEFAULT '' COMMENT 'the name of the integration that sent the event',
"$duration_s" UInt16 DEFAULT 0 COMMENT 'the duration from session-start in seconds',
properties JSON DEFAULT '{}',
"$properties" JSON DEFAULT '{}' COMMENT 'these properties belongs to the auto-captured events',
"$properties" JSON(
max_dynamic_paths=0,
label String ,
hesitation_time UInt32 ,
name String ,
payload String ,
level Enum8 ('info'=0, 'error'=1),
source Enum8 ('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9),
message String ,
error_id String ,
duration UInt16,
context Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8),
url_host String ,
url_path String ,
url_hostpath String ,
request_start UInt16 ,
response_start UInt16 ,
response_end UInt16 ,
dom_content_loaded_event_start UInt16 ,
dom_content_loaded_event_end UInt16 ,
load_event_start UInt16 ,
load_event_end UInt16 ,
first_paint UInt16 ,
first_contentful_paint_time UInt16 ,
speed_index UInt16 ,
visually_complete UInt16 ,
time_to_interactive UInt16,
ttfb UInt16,
ttlb UInt16,
response_time UInt16,
dom_building_time UInt16,
dom_content_loaded_event_time UInt16,
load_event_time UInt16,
min_fps UInt8,
avg_fps UInt8,
max_fps UInt8,
min_cpu UInt8,
avg_cpu UInt8,
max_cpu UInt8,
min_total_js_heap_size UInt64,
avg_total_js_heap_size UInt64,
max_total_js_heap_size UInt64,
min_used_js_heap_size UInt64,
avg_used_js_heap_size UInt64,
max_used_js_heap_size UInt64,
method Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8),
status UInt16,
success UInt8,
request_body String,
response_body String,
transfer_size UInt32,
selector String,
normalized_x Float32,
normalized_y Float32,
message_id UInt64
) DEFAULT '{}' COMMENT 'these properties belongs to the auto-captured events',
description String DEFAULT '',
group_id1 Array(String) DEFAULT [],
group_id2 Array(String) DEFAULT [],