diff --git a/ee/api/chalicelib/core/sessions.py b/ee/api/chalicelib/core/sessions.py index f2725188d..0fd171064 100644 --- a/ee/api/chalicelib/core/sessions.py +++ b/ee/api/chalicelib/core/sessions.py @@ -568,14 +568,14 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d def __is_valid_event(is_any: bool, event: schemas._SessionSearchEventSchema): return not (not is_any and len(event.value) == 0 and event.type not in [schemas.EventType.request_details, - schemas.EventType.graphql_details] \ + schemas.EventType.graphql] \ or event.type in [schemas.PerformanceEventType.location_dom_complete, schemas.PerformanceEventType.location_largest_contentful_paint_time, schemas.PerformanceEventType.location_ttfb, schemas.PerformanceEventType.location_avg_cpu_load, schemas.PerformanceEventType.location_avg_memory_usage ] and (event.source is None or len(event.source) == 0) \ - or event.type in [schemas.EventType.request_details, schemas.EventType.graphql_details] and ( + or event.type in [schemas.EventType.request_details, schemas.EventType.graphql] and ( event.filters is None or len(event.filters) == 0)) @@ -1071,7 +1071,7 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr print(f"undefined FETCH filter: {f.type}") if not apply: continue - elif event_type == schemas.EventType.graphql_details: + elif event_type == schemas.EventType.graphql: event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " for j, f in enumerate(event.filters): is_any = _isAny_opreator(f.operator) @@ -1558,30 +1558,40 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, value_key=e_k)) events_conditions[-1]["condition"] = event_where[-1] - elif event_type == events.event_type.GRAPHQL.ui_type: - event_from = event_from % f"final.events AS main" - event_where.append(f"main.event_type='GRAPHQL'") - events_conditions.append({"type": event_where[-1]}) - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, - value_key=e_k)) - events_conditions[-1]["condition"] = event_where[-1] + # elif event_type == events.event_type.GRAPHQL.ui_type: + # event_from = event_from % f"final.events AS main" + # event_where.append(f"main.event_type='GRAPHQL'") + # events_conditions.append({"type": event_where[-1]}) + # if not is_any: + # event_where.append( + # _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, + # value_key=e_k)) + # events_conditions[-1]["condition"] = event_where[-1] elif event_type == events.event_type.STATEACTION.ui_type: - event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='STATEACTION'") + events_conditions.append({"type": event_where[-1]}) if not is_any: event_where.append( _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", event.value, value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] elif event_type == events.event_type.ERROR.ui_type: - event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" + + event_from = event_from % f"final.events AS main INNER JOIN final.errors AS main1 USING(error_id)" + event_where.append(f"main.event_type='ERROR'") + events_conditions.append({"type": event_where[-1]}) event.source = tuple(event.source) + events_conditions[-1]["condition"] = [] if not is_any and event.value not in [None, "*", ""]: event_where.append( _multiple_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", event.value, value_key=e_k)) + events_conditions[-1]["condition"].append(event_where[-1]) if event.source[0] not in [None, "*", ""]: event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.value, value_key=s_k)) + events_conditions[-1]["condition"].append(event_where[-1]) + events_conditions[-1]["condition"] = " AND ".join(events_conditions[-1]["condition"]) elif event_type == schemas.PerformanceEventType.fetch_failed: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " @@ -1723,9 +1733,9 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, else: events_conditions[-1]["condition"] = " AND ".join(events_conditions[-1]["condition"]) - elif event_type == schemas.EventType.graphql_details: + elif event_type == schemas.EventType.graphql: event_from = event_from % f"final.events AS main " - event_where.append(f"main.event_type='REQUEST'") + event_where.append(f"main.event_type='GRAPHQL'") events_conditions.append({"type": event_where[-1]}) events_conditions[-1]["condition"] = [] for j, f in enumerate(event.filters): diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql index f6ba9d751..c4b5aa8f8 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql @@ -4,7 +4,7 @@ ALTER TABLE sessions CREATE TABLE projects_metadata ( - project_id UInt32, + project_id UInt16, metadata_1 Nullable(String), metadata_2 Nullable(String), metadata_3 Nullable(String), @@ -24,8 +24,8 @@ CREATE TABLE projects_metadata CREATE TABLE IF NOT EXISTS events_s ( session_id UInt64, - project_id UInt32, - event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + project_id UInt16, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8, 'GRAPHQL'=9, 'STATEACTION'=10), datetime DateTime, label Nullable(String), hesitation_time Nullable(UInt32), @@ -90,6 +90,8 @@ CREATE TABLE IF NOT EXISTS events_s success Nullable(UInt8), method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), status Nullable(UInt16), + request_body Nullable(String), + response_body Nullable(String), _timestamp DateTime DEFAULT now() ) ENGINE = MergeTree PARTITION BY toYYYYMM(datetime) @@ -99,7 +101,7 @@ CREATE TABLE IF NOT EXISTS events_s CREATE TABLE IF NOT EXISTS sessions ( session_id UInt64, - project_id UInt32, + project_id UInt16, tracker_version LowCardinality(String), rev_id LowCardinality(Nullable(String)), user_uuid UUID, @@ -138,15 +140,31 @@ CREATE TABLE IF NOT EXISTS sessions CREATE TABLE IF NOT EXISTS autocomplete ( - project_id UInt32 NOT NULL, - type LowCardinality(String) NOT NULL, - value String NOT NULL, + project_id UInt16, + type LowCardinality(String), + value String, _timestamp DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(_timestamp) - ORDER BY (project_id, type) + ORDER BY (project_id, type, value) TTL _timestamp + INTERVAL 1 MONTH; +CREATE TABLE IF NOT EXISTS errors +( + error_id String, + project_id UInt16, + source Enum8('js_exception'=1,'bugsnag'=2,'cloudwatch'=3,'datadog'=4,'newrelic'=5,'rollbar'=6,'sentry'=7,'stackdriver'=8,'sumologic'=9, 'elasticsearch'=10), + name Nullable(String), + message String, + payload String, + stacktrace Nullable(String), --to save the stacktrace and not query S3 another time + stacktrace_parsed_at Nullable(DateTime), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(_timestamp) + ORDER BY (project_id, source, error_id) + TTL _timestamp + INTERVAL 1 MONTH + SETTINGS index_granularity = 512; CREATE MATERIALIZED VIEW sessions_l7d_mv ENGINE = ReplacingMergeTree(_timestamp) diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql index e22b73848..d3525e54a 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql @@ -2876,3 +2876,8 @@ SELECT session_id, metadata_10 FROM massive_split.sessions_s AS s LEFT JOIN massive_split.metadata_s AS m ON (s.project_id = m.project_id AND s.session_id = m.session_id); + + +INSERT INTO final.errors(error_id, project_id, source, name, message) +SELECT error_id, project_id, toString(source), name, message +FROM default.errors;