From 2492867e9c3c6064f27e412125f9ab48d46424f3 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 5 Oct 2022 20:05:11 +0200 Subject: [PATCH] feat(chalice): error tags feat(DB): new table for issues feat(DB): new columns for events --- ee/api/chalicelib/core/__init__.py | 4 +++ ee/api/chalicelib/core/errors.py | 8 ++++-- ee/api/env.default | 1 + .../db/init_dbs/clickhouse/1.8.2/1.8.2.sql | 28 +++++++++++++++++++ .../clickhouse/create/init_schema.sql | 24 +++++++++++++++- 5 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.8.2/1.8.2.sql diff --git a/ee/api/chalicelib/core/__init__.py b/ee/api/chalicelib/core/__init__.py index 550327129..41c437c93 100644 --- a/ee/api/chalicelib/core/__init__.py +++ b/ee/api/chalicelib/core/__init__.py @@ -18,7 +18,11 @@ else: if config("EXP_ERRORS_SEARCH", cast=bool, default=False): print(">>> Using experimental error search") + from . import errors as errors_legacy from . import errors_exp as errors + + if config("EXP_ERRORS_GET", cast=bool, default=False): + print(">>> Using experimental error get") else: from . import errors as errors diff --git a/ee/api/chalicelib/core/errors.py b/ee/api/chalicelib/core/errors.py index 1db619b15..a1db0c798 100644 --- a/ee/api/chalicelib/core/errors.py +++ b/ee/api/chalicelib/core/errors.py @@ -128,7 +128,8 @@ def get_details(project_id, error_id, user_id, **data): device_partition, country_partition, chart24, - chart30 + chart30, + COALESCE(tags,'{{}}')::jsonb AS custom_tags FROM (SELECT error_id, name, message, @@ -241,7 +242,10 @@ def get_details(project_id, error_id, user_id, **data): WHERE {" AND ".join(pg_sub_query30)}) AS chart_details ON (TRUE) GROUP BY timestamp - ORDER BY timestamp) AS chart_details) AS chart_details30 ON (TRUE); + ORDER BY timestamp) AS chart_details) AS chart_details30 ON (TRUE) + LEFT JOIN (SELECT jsonb_agg(jsonb_build_object(errors_tags.key, errors_tags.value)) + FROM errors_tags INNER JOIN errors USING(error_id) + WHERE {" AND ".join(pg_basic_query)}) AS raw_tags(tags) ON (TRUE); """ # print("--------------------") diff --git a/ee/api/env.default b/ee/api/env.default index 98c94c9b5..673454853 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -55,6 +55,7 @@ FS_DIR=/mnt/efs EXP_SESSIONS_SEARCH=false EXP_AUTOCOMPLETE=false EXP_ERRORS_SEARCH=false +EXP_ERRORS_GET=false EXP_METRICS=true EXP_7D_MV=false EXP_ALERTS=false diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.2/1.8.2.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.2/1.8.2.sql new file mode 100644 index 000000000..65acd9fb5 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.2/1.8.2.sql @@ -0,0 +1,28 @@ +ALTER TABLE experimental.events + ADD COLUMN IF NOT EXISTS error_tags_keys Array(String); +ALTER TABLE experimental.events + ADD COLUMN IF NOT EXISTS error_tags_values Array(Nullable(String)); + +ALTER TABLE experimental.events + ADD COLUMN IF NOT EXISTS issue_type Nullable(Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19)); +ALTER TABLE experimental.events + ADD COLUMN IF NOT EXISTS issue_id Nullable(String); +ALTER TABLE experimental.events + MODIFY COLUMN event_type Enum8('CLICK'=0, 'INPUT'=1, 'LOCATION'=2,'REQUEST'=3,'PERFORMANCE'=4,'ERROR'=5,'CUSTOM'=6, 'GRAPHQL'=7, 'STATEACTION'=8, 'ISSUE'=9); + + +CREATE TABLE IF NOT EXISTS experimental.issues +( + project_id UInt16, + issue_id String, + type Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19), + context_string text NOT NULL, + context_keys Array(String), + context_values Array(Nullable(String)), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(_timestamp) + ORDER BY (project_id, issue_id, type) + TTL _timestamp + INTERVAL 3 MONTH; + +-- TODO: find a way to update materialized views; or drop and re-create them diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql index 01f3a9968..6ebcfc319 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS experimental.events ( session_id UInt64, project_id UInt16, - event_type Enum8('CLICK'=0, 'INPUT'=1, 'LOCATION'=2,'REQUEST'=3,'PERFORMANCE'=4,'ERROR'=5,'CUSTOM'=6, 'GRAPHQL'=7, 'STATEACTION'=8), + event_type Enum8('CLICK'=0, 'INPUT'=1, 'LOCATION'=2,'REQUEST'=3,'PERFORMANCE'=4,'ERROR'=5,'CUSTOM'=6, 'GRAPHQL'=7, 'STATEACTION'=8, 'ISSUE'=9), datetime DateTime, label Nullable(String), hesitation_time Nullable(UInt32), @@ -78,6 +78,10 @@ CREATE TABLE IF NOT EXISTS experimental.events success Nullable(UInt8), request_body Nullable(String), response_body Nullable(String), + issue_type Nullable(Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19)), + issue_id Nullable(String), + error_tags_keys Array(String), + error_tags_values Array(Nullable(String)), message_id UInt64 DEFAULT 0, _timestamp DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(_timestamp) @@ -192,6 +196,20 @@ CREATE TABLE IF NOT EXISTS experimental.user_viewed_errors ORDER BY (project_id, user_id, error_id) TTL _timestamp + INTERVAL 3 MONTH; +CREATE TABLE IF NOT EXISTS experimental.issues +( + project_id UInt16, + issue_id String, + type Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19), + context_string text NOT NULL, + context_keys Array(String), + context_values Array(Nullable(String)), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(_timestamp) + ORDER BY (project_id, issue_id, type) + TTL _timestamp + INTERVAL 3 MONTH; + CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) @@ -256,6 +274,10 @@ SELECT session_id, success, request_body, response_body, + issue_type, + issue_id, + error_tags_keys, + error_tags_values, message_id, _timestamp FROM experimental.events