From 664b219581b5045df9c32dea6faac35a1953177f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 10 Jan 2023 10:45:37 +0100 Subject: [PATCH 01/10] feat(chalice): fixed exp_metrics event_type --- api/schemas.py | 8 ++++---- ee/api/chalicelib/core/metrics_exp.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/schemas.py b/api/schemas.py index aeed98880..aef6a9567 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -777,16 +777,16 @@ class FunnelSearchPayloadSchema(FlatSessionsSearchPayloadSchema): class FunnelSchema(BaseModel): name: str = Field(...) filter: FunnelSearchPayloadSchema = Field([]) - is_public: bool = Field(False) + is_public: bool = Field(default=False) class Config: alias_generator = attribute_to_camel_case class UpdateFunnelSchema(FunnelSchema): - name: Optional[str] = Field(None) - filter: Optional[FunnelSearchPayloadSchema] = Field(None) - is_public: Optional[bool] = Field(None) + name: Optional[str] = Field(default=None) + filter: Optional[FunnelSearchPayloadSchema] = Field(default=None) + is_public: Optional[bool] = Field(default=None) class FunnelInsightsPayloadSchema(FlatSessionsSearchPayloadSchema): diff --git a/ee/api/chalicelib/core/metrics_exp.py b/ee/api/chalicelib/core/metrics_exp.py index 268be3d1c..652f708e0 100644 --- a/ee/api/chalicelib/core/metrics_exp.py +++ b/ee/api/chalicelib/core/metrics_exp.py @@ -1351,7 +1351,7 @@ def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1) step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="performance", round_start=True, data=args) - ch_sub_query_chart.append("performance.EventType='PERFORMANCE'") + ch_sub_query_chart.append("event_type='PERFORMANCE'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -1384,7 +1384,7 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="performance", round_start=True, data=args) - ch_sub_query_chart.append("performance.EventType='PERFORMANCE'") + ch_sub_query_chart.append("event_type='PERFORMANCE'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -1417,7 +1417,7 @@ def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="performance", round_start=True, data=args) - ch_sub_query_chart.append("performance.EventType='PERFORMANCE'") + ch_sub_query_chart.append("event_type='PERFORMANCE'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition From a407d872a3669395199035af33d2e218a05d95be Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 10 Jan 2023 15:10:30 +0100 Subject: [PATCH 02/10] feat(DB): changed metrics/cards feat(DB): changed metrics structure feat(DB): changed metrics default values --- .../db/init_dbs/postgresql/1.10.0/1.10.0.sql | 119 +++++++++++++++++ .../db/init_dbs/postgresql/1.9.5/1.9.5.sql | 121 ------------------ .../db/init_dbs/postgresql/init_schema.sql | 22 ++-- .../{1.9.5/1.9.5.sql => 1.10.0/1.10.0.sql} | 28 ++-- .../db/init_dbs/postgresql/init_schema.sql | 7 +- 5 files changed, 147 insertions(+), 150 deletions(-) delete mode 100644 ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql rename scripts/schema/db/init_dbs/postgresql/{1.9.5/1.9.5.sql => 1.10.0/1.10.0.sql} (84%) diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql index a34074856..c80e3dc83 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql @@ -17,4 +17,123 @@ CREATE TABLE IF NOT EXISTS frontend_signals ); CREATE INDEX IF NOT EXISTS frontend_signals_user_id_idx ON frontend_signals (user_id); +CREATE TABLE IF NOT EXISTS assist_records +( + record_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE SET NULL, + created_at bigint NOT NULL DEFAULT (EXTRACT(EPOCH FROM now() at time zone 'utc') * 1000)::bigint, + deleted_at timestamp without time zone NULL DEFAULT NULL, + name text NOT NULL, + file_key text NOT NULL, + duration integer NOT NULL +); + +ALTER TYPE webhook_type ADD VALUE IF NOT EXISTS 'msteams'; + +UPDATE metrics +SET is_public= TRUE; + +ALTER TABLE IF EXISTS metrics + ALTER COLUMN metric_type TYPE text, + ALTER COLUMN metric_type SET DEFAULT 'timeseries', + ALTER COLUMN view_type TYPE text, + ALTER COLUMN view_type SET DEFAULT 'lineChart', + ADD COLUMN IF NOT EXISTS thumbnail text; + +DO +$$ + BEGIN + IF EXISTS(SELECT column_name + FROM information_schema.columns + WHERE table_name = 'metrics' + and column_name = 'is_predefined') THEN + + -- 1. pre transform structure + ALTER TABLE IF EXISTS metrics + ADD COLUMN IF NOT EXISTS o_metric_id INTEGER, + ADD COLUMN IF NOT EXISTS o_widget_id INTEGER; + + -- 2. insert predefined metrics related to dashboards as custom metrics + INSERT INTO metrics(project_id, user_id, name, metric_type, view_type, metric_of, metric_value, + metric_format, default_config, is_public, o_metric_id, o_widget_id) + SELECT dashboards.project_id, + dashboard_widgets.user_id, + metrics.name, + left(category, 1) || right(replace(initcap(category), ' ', ''), -1) AS metric_type, + 'chart' AS view_type, + left(predefined_key, 1) || right(replace(initcap(predefined_key), '_', ''), -1) AS metric_of, + metric_value, + metric_format, + default_config, + TRUE AS is_public, + metrics.metric_id, + dashboard_widgets.widget_id + FROM metrics + INNER JOIN dashboard_widgets USING (metric_id) + INNER JOIN dashboards USING (dashboard_id) + WHERE is_predefined; + + -- 3. update widgets + UPDATE dashboard_widgets + SET metric_id=metrics.metric_id + FROM metrics + WHERE metrics.o_widget_id IS NOT NULL + AND dashboard_widgets.widget_id = metrics.o_widget_id; + + -- 4. delete predefined metrics + DELETE + FROM metrics + WHERE is_predefined; + + ALTER TABLE IF EXISTS metrics + DROP COLUMN IF EXISTS active, + DROP COLUMN IF EXISTS is_predefined, + DROP COLUMN IF EXISTS is_template, + DROP COLUMN IF EXISTS category, + DROP COLUMN IF EXISTS o_metric_id, + DROP COLUMN IF EXISTS o_widget_id, + DROP CONSTRAINT IF EXISTS null_project_id_for_template_only, + DROP CONSTRAINT IF EXISTS metrics_unique_key, + DROP CONSTRAINT IF EXISTS unique_key; + + END IF; + END; +$$ +LANGUAGE plpgsql; + +DROP TYPE IF EXISTS metric_type; +DROP TYPE IF EXISTS metric_view_type; + +ALTER TABLE IF EXISTS events.clicks + ADD COLUMN IF NOT EXISTS path text; + +DROP INDEX IF EXISTS events.clicks_url_gin_idx; +DROP INDEX IF EXISTS events.inputs_label_value_idx; +DROP INDEX IF EXISTS events.inputs_label_idx; +DROP INDEX IF EXISTS events.pages_base_path_idx; +DROP INDEX IF EXISTS events.pages_base_path_idx1; +DROP INDEX IF EXISTS events.pages_base_path_idx2; +DROP INDEX IF EXISTS events.pages_base_referrer_gin_idx1; +DROP INDEX IF EXISTS events.pages_base_referrer_gin_idx2; +DROP INDEX IF EXISTS events.resources_url_gin_idx; +DROP INDEX IF EXISTS events.resources_url_idx; +DROP INDEX IF EXISTS events.resources_url_hostpath_idx; +DROP INDEX IF EXISTS events.resources_session_id_timestamp_idx; +DROP INDEX IF EXISTS events.resources_duration_durationgt0_idx; +DROP INDEX IF EXISTS events.state_actions_name_idx; +DROP INDEX IF EXISTS events_common.requests_query_nn_idx; +DROP INDEX IF EXISTS events_common.requests_host_nn_idx; +DROP INDEX IF EXISTS events_common.issues_context_string_gin_idx; +DROP INDEX IF EXISTS public.sessions_user_country_gin_idx; +DROP INDEX IF EXISTS public.sessions_user_browser_gin_idx; +DROP INDEX IF EXISTS public.sessions_user_os_gin_idx; +DROP INDEX IF EXISTS public.issues_context_string_gin_idx; + COMMIT; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector); +CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_path_idx ON events.clicks (path); +CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_path_gin_idx ON events.clicks USING GIN (path gin_trgm_ops); +CREATE INDEX CONCURRENTLY IF NOT EXISTS issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql deleted file mode 100644 index 62679c9d8..000000000 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql +++ /dev/null @@ -1,121 +0,0 @@ -BEGIN; -CREATE OR REPLACE FUNCTION openreplay_version() - RETURNS text AS -$$ -SELECT 'v1.9.5-ee' -$$ LANGUAGE sql IMMUTABLE; - -CREATE TABLE IF NOT EXISTS assist_records -( - record_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE SET NULL, - created_at bigint NOT NULL DEFAULT (EXTRACT(EPOCH FROM now() at time zone 'utc') * 1000)::bigint, - deleted_at timestamp without time zone NULL DEFAULT NULL, - name text NOT NULL, - file_key text NOT NULL, - duration integer NOT NULL -); - -ALTER TYPE webhook_type ADD VALUE IF NOT EXISTS 'msteams'; - -DO -$$ - BEGIN - IF EXISTS(SELECT column_name - FROM information_schema.columns - WHERE table_name = 'metrics' - and column_name = 'is_predefined') THEN - - -- 1. pre transform structure - ALTER TABLE IF EXISTS metrics - ALTER COLUMN metric_type TYPE text, - ALTER COLUMN metric_type SET DEFAULT 'timeseries', - ALTER COLUMN view_type TYPE text, - ALTER COLUMN view_type SET DEFAULT 'lineChart', - ADD COLUMN IF NOT EXISTS thumbnail text, - ADD COLUMN IF NOT EXISTS o_metric_id INTEGER, - ADD COLUMN IF NOT EXISTS o_widget_id INTEGER; - - -- 2. insert predefined metrics related to dashboards as custom metrics - INSERT INTO metrics(project_id, user_id, name, metric_type, view_type, metric_of, metric_value, - metric_format, - default_config, o_metric_id, o_widget_id) - SELECT dashboards.project_id, - dashboard_widgets.user_id, - metrics.name, - left(category, 1) || right(replace(initcap(category), ' ', ''), -1) AS metric_type, - 'chart' AS view_type, - left(predefined_key, 1) || right(replace(initcap(predefined_key), '_', ''), -1) AS metric_of, - metric_value, - metric_format, - default_config, - metrics.metric_id, - dashboard_widgets.widget_id - FROM metrics - INNER JOIN dashboard_widgets USING (metric_id) - INNER JOIN dashboards USING (dashboard_id) - WHERE is_predefined; - - -- 3. update widgets - UPDATE dashboard_widgets - SET metric_id=metrics.metric_id - FROM metrics - WHERE metrics.o_widget_id IS NOT NULL - AND dashboard_widgets.widget_id = metrics.o_widget_id; - - -- 4. delete predefined metrics - DELETE - FROM metrics - WHERE is_predefined; - - ALTER TABLE IF EXISTS metrics - DROP COLUMN IF EXISTS active, - DROP COLUMN IF EXISTS is_predefined, - DROP COLUMN IF EXISTS is_template, - DROP COLUMN IF EXISTS category, - DROP COLUMN IF EXISTS o_metric_id, - DROP COLUMN IF EXISTS o_widget_id, - DROP CONSTRAINT IF EXISTS null_project_id_for_template_only, - DROP CONSTRAINT IF EXISTS metrics_unique_key; - - END IF; - END; -$$ -LANGUAGE plpgsql; - -DROP TYPE IF EXISTS metric_type; -DROP TYPE IF EXISTS metric_view_type; - -ALTER TABLE IF EXISTS events.clicks - ADD COLUMN IF NOT EXISTS path text; - -DROP INDEX IF EXISTS events.clicks_url_gin_idx; -DROP INDEX IF EXISTS events.inputs_label_value_idx; -DROP INDEX IF EXISTS events.inputs_label_idx; -DROP INDEX IF EXISTS events.pages_base_path_idx; -DROP INDEX IF EXISTS events.pages_base_path_idx1; -DROP INDEX IF EXISTS events.pages_base_path_idx2; -DROP INDEX IF EXISTS events.pages_base_referrer_gin_idx1; -DROP INDEX IF EXISTS events.pages_base_referrer_gin_idx2; -DROP INDEX IF EXISTS events.resources_url_gin_idx; -DROP INDEX IF EXISTS events.resources_url_idx; -DROP INDEX IF EXISTS events.resources_url_hostpath_idx; -DROP INDEX IF EXISTS events.resources_session_id_timestamp_idx; -DROP INDEX IF EXISTS events.resources_duration_durationgt0_idx; -DROP INDEX IF EXISTS events.state_actions_name_idx; -DROP INDEX IF EXISTS events_common.requests_query_nn_idx; -DROP INDEX IF EXISTS events_common.requests_host_nn_idx; -DROP INDEX IF EXISTS events_common.issues_context_string_gin_idx; -DROP INDEX IF EXISTS public.sessions_user_country_gin_idx; -DROP INDEX IF EXISTS public.sessions_user_browser_gin_idx; -DROP INDEX IF EXISTS public.sessions_user_os_gin_idx; -DROP INDEX IF EXISTS public.issues_context_string_gin_idx; - -COMMIT; - -CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector); -CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_path_idx ON events.clicks (path); -CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_path_gin_idx ON events.clicks USING GIN (path gin_trgm_ops); -CREATE INDEX CONCURRENTLY IF NOT EXISTS issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index 9c8a57363..cf60b1939 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -645,13 +645,13 @@ $$ CREATE TABLE IF NOT EXISTS frontend_signals ( - project_id bigint NOT NULL, - user_id text NOT NULL, - timestamp bigint NOT NULL, - action text NOT NULL, - source text NOT NULL, - category text NOT NULL, - data json + project_id bigint NOT NULL, + user_id text NOT NULL, + timestamp bigint NOT NULL, + action text NOT NULL, + source text NOT NULL, + category text NOT NULL, + data json ); CREATE INDEX IF NOT EXISTS frontend_signals_user_id_idx ON frontend_signals (user_id); @@ -751,7 +751,7 @@ $$ project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE, user_id integer REFERENCES users (user_id) ON DELETE SET NULL, name text NOT NULL, - is_public boolean NOT NULL DEFAULT FALSE, + is_public boolean NOT NULL DEFAULT TRUE, created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), deleted_at timestamp, edited_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), @@ -760,16 +760,12 @@ $$ metric_of text NOT NULL DEFAULT 'sessionCount', metric_value text[] NOT NULL DEFAULT '{}'::text[], metric_format text, - is_pinned boolean NOT NULL DEFAULT FALSE, thumbnail text, default_config jsonb NOT NULL DEFAULT '{ "col": 2, "row": 2, "position": 0 - }'::jsonb, - CONSTRAINT null_project_id_for_template_only - CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), - CONSTRAINT unique_key UNIQUE (predefined_key) + }'::jsonb ); CREATE INDEX IF NOT EXISTS metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); CREATE TABLE IF NOT EXISTS metric_series diff --git a/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql b/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql similarity index 84% rename from scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql rename to scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql index ddecd637a..4ba8ca88c 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql @@ -2,11 +2,21 @@ BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.9.5' +SELECT 'v1.10.0' $$ LANGUAGE sql IMMUTABLE; ALTER TYPE webhook_type ADD VALUE IF NOT EXISTS 'msteams'; +UPDATE metrics +SET is_public= TRUE; + +ALTER TABLE IF EXISTS metrics + ALTER COLUMN metric_type TYPE text, + ALTER COLUMN metric_type SET DEFAULT 'timeseries', + ALTER COLUMN view_type TYPE text, + ALTER COLUMN view_type SET DEFAULT 'lineChart', + ADD COLUMN IF NOT EXISTS thumbnail text; + DO $$ BEGIN @@ -17,18 +27,12 @@ $$ -- 1. pre transform structure ALTER TABLE IF EXISTS metrics - ALTER COLUMN metric_type TYPE text, - ALTER COLUMN metric_type SET DEFAULT 'timeseries', - ALTER COLUMN view_type TYPE text, - ALTER COLUMN view_type SET DEFAULT 'lineChart', - ADD COLUMN IF NOT EXISTS thumbnail text, ADD COLUMN IF NOT EXISTS o_metric_id INTEGER, ADD COLUMN IF NOT EXISTS o_widget_id INTEGER; -- 2. insert predefined metrics related to dashboards as custom metrics INSERT INTO metrics(project_id, user_id, name, metric_type, view_type, metric_of, metric_value, - metric_format, - default_config, o_metric_id, o_widget_id) + metric_format, default_config, is_public, o_metric_id, o_widget_id) SELECT dashboards.project_id, dashboard_widgets.user_id, metrics.name, @@ -38,6 +42,7 @@ $$ metric_value, metric_format, default_config, + TRUE AS is_public, metrics.metric_id, dashboard_widgets.widget_id FROM metrics @@ -60,12 +65,14 @@ $$ ALTER TABLE IF EXISTS metrics DROP COLUMN IF EXISTS active, DROP COLUMN IF EXISTS is_predefined, + DROP COLUMN IF EXISTS predefined_key, DROP COLUMN IF EXISTS is_template, DROP COLUMN IF EXISTS category, DROP COLUMN IF EXISTS o_metric_id, DROP COLUMN IF EXISTS o_widget_id, DROP CONSTRAINT IF EXISTS null_project_id_for_template_only, - DROP CONSTRAINT IF EXISTS metrics_unique_key; + DROP CONSTRAINT IF EXISTS metrics_unique_key, + DROP CONSTRAINT IF EXISTS unique_key; END IF; END; @@ -100,10 +107,9 @@ DROP INDEX IF EXISTS public.sessions_user_browser_gin_idx; DROP INDEX IF EXISTS public.sessions_user_os_gin_idx; DROP INDEX IF EXISTS public.issues_context_string_gin_idx; - COMMIT; CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector); CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_path_idx ON events.clicks (path); CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_path_gin_idx ON events.clicks USING GIN (path gin_trgm_ops); -CREATE INDEX CONCURRENTLY IF NOT EXISTS issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); \ No newline at end of file diff --git a/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/scripts/schema/db/init_dbs/postgresql/init_schema.sql index 7414f7ff3..1812591a3 100644 --- a/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -860,7 +860,7 @@ $$ project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE, user_id integer REFERENCES users (user_id) ON DELETE SET NULL, name text NOT NULL, - is_public boolean NOT NULL DEFAULT FALSE, + is_public boolean NOT NULL DEFAULT TRUE, created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), deleted_at timestamp, edited_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), @@ -874,10 +874,7 @@ $$ "col": 2, "row": 2, "position": 0 - }'::jsonb, - CONSTRAINT null_project_id_for_template_only - CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), - CONSTRAINT unique_key UNIQUE (predefined_key) + }'::jsonb ); CREATE INDEX metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); From 9b637133cb7a7b5c33a45d4ba276525b938a2831 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 10 Jan 2023 15:32:03 +0100 Subject: [PATCH 03/10] feat(DB): changed frontend_signals --- .../db/init_dbs/postgresql/1.10.0/1.10.0.sql | 4 ++-- .../schema/db/init_dbs/postgresql/init_schema.sql | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql index c80e3dc83..17a0cf1a8 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql @@ -7,8 +7,8 @@ $$ LANGUAGE sql IMMUTABLE; CREATE TABLE IF NOT EXISTS frontend_signals ( - project_id bigint NOT NULL, - user_id integer NOT NULL references users (user_id) ON DELETE CASCADE, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, timestamp bigint NOT NULL, action text NOT NULL, source text NOT NULL, diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index cf60b1939..0ad352063 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -645,13 +645,13 @@ $$ CREATE TABLE IF NOT EXISTS frontend_signals ( - project_id bigint NOT NULL, - user_id text NOT NULL, - timestamp bigint NOT NULL, - action text NOT NULL, - source text NOT NULL, - category text NOT NULL, - data json + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + action text NOT NULL, + source text NOT NULL, + category text NOT NULL, + data jsonb ); CREATE INDEX IF NOT EXISTS frontend_signals_user_id_idx ON frontend_signals (user_id); From 556cc297bf6c2b50abb931f272d1cdf59027694f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 10 Jan 2023 15:41:36 +0100 Subject: [PATCH 04/10] feat(chalice): fixed exp_metrics event_type --- ee/api/chalicelib/core/metrics_exp.py | 116 +++++++++++++------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/ee/api/chalicelib/core/metrics_exp.py b/ee/api/chalicelib/core/metrics_exp.py index 652f708e0..c41676d4a 100644 --- a/ee/api/chalicelib/core/metrics_exp.py +++ b/ee/api/chalicelib/core/metrics_exp.py @@ -210,10 +210,10 @@ def get_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimesta step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query = __get_basic_constraints(table_name="errors", data=args) - ch_sub_query.append("errors.EventType = 'ERROR'") + ch_sub_query.append("errors.event_type = 'ERROR'") ch_sub_query.append("errors.source = 'js_exception'") ch_sub_query_chart = __get_basic_constraints(table_name="errors", round_start=True, data=args) - ch_sub_query_chart.append("errors.EventType = 'ERROR'") + ch_sub_query_chart.append("errors.event_type = 'ERROR'") ch_sub_query_chart.append("errors.source = 'js_exception'") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition @@ -270,9 +270,9 @@ def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query = __get_basic_constraints(table_name="errors", data=args) - ch_sub_query.append("errors.EventType='ERROR'") + ch_sub_query.append("errors.event_type='ERROR'") ch_sub_query_chart = __get_basic_constraints(table_name="errors", round_start=True, data=args) - ch_sub_query_chart.append("errors.EventType='ERROR'") + ch_sub_query_chart.append("errors.event_type='ERROR'") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition ch_sub_query_chart += meta_condition @@ -290,7 +290,7 @@ def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), toUnixTimestamp(MAX(datetime))*1000 AS lastOccurrenceAt, toUnixTimestamp(MIN(datetime))*1000 AS firstOccurrenceAt FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS errors - WHERE EventType='ERROR' AND project_id=%(project_id)s + WHERE event_type='ERROR' AND project_id=%(project_id)s GROUP BY error_id) AS errors_time USING(error_id) ORDER BY sessions DESC, count DESC LIMIT 10;""" params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, @@ -344,7 +344,7 @@ def get_page_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), def __get_page_metrics(ch, project_id, startTimestamp, endTimestamp, **args): ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition ch_sub_query.append("(pages.dom_content_loaded_event_end>0 OR pages.first_contentful_paint_time>0)") @@ -377,7 +377,7 @@ def get_application_activity(project_id, startTimestamp=TimeUTC.now(delta_days=- def __get_application_activity(ch, project_id, startTimestamp, endTimestamp, **args): result = {} ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition @@ -390,7 +390,7 @@ def __get_application_activity(ch, project_id, startTimestamp, endTimestamp, **a result = {**result, **row} ch_sub_query = __get_basic_constraints(table_name="resources", data=args) - # ch_sub_query.append("events.EventType='RESOURCE'") + # ch_sub_query.append("events.event_type='RESOURCE'") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition ch_sub_query.append("resources.type= %(type)s") @@ -457,10 +457,10 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), density=7, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) ch_sub_query = __get_basic_constraints(table_name="resources", data=args) - # ch_sub_query.append("events.EventType='RESOURCE'") + # ch_sub_query.append("events.event_type='RESOURCE'") ch_sub_query.append("resources.type = 'img'") ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - # ch_sub_query_chart.append("events.EventType='RESOURCE'") + # ch_sub_query_chart.append("events.event_type='RESOURCE'") ch_sub_query_chart.append("resources.type = 'img'") ch_sub_query_chart.append("resources.url_hostpath IN %(url)s") meta_condition = __get_meta_constraint(args) @@ -527,7 +527,7 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi img_constraints = [] request_constraints = [] ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - # ch_sub_query_chart.append("EventType='RESOURCE'") + # ch_sub_query_chart.append("event_type='RESOURCE'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -577,7 +577,7 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi end_time=endTimestamp, density=density, neutral={"avg": 0})] ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") ch_sub_query_chart += meta_condition ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, @@ -835,7 +835,7 @@ def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - # ch_sub_query_chart.append("events.EventType='RESOURCE'") + # ch_sub_query_chart.append("events.event_type='RESOURCE'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -922,7 +922,7 @@ def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=- endTimestamp=TimeUTC.now(), density=19, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") if url is not None: ch_sub_query_chart.append(f"pages.url_path = %(value)s") ch_sub_query_chart.append("isNotNull(pages.dom_building_time)") @@ -1045,7 +1045,7 @@ def get_sessions_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), def get_speed_index_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") ch_sub_query.append("isNotNull(pages.speed_index)") ch_sub_query.append("pages.speed_index>0") meta_condition = __get_meta_constraint(args) @@ -1074,7 +1074,7 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 endTimestamp=TimeUTC.now(), density=7, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") ch_sub_query_chart.append("isNotNull(pages.response_time)") ch_sub_query_chart.append("pages.response_time>0") meta_condition = __get_meta_constraint(args) @@ -1110,7 +1110,7 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 def get_pages_response_time_distribution(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=20, **args): ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") ch_sub_query.append("isNotNull(pages.response_time)") ch_sub_query.append("pages.response_time>0") meta_condition = __get_meta_constraint(args) @@ -1249,7 +1249,7 @@ def get_busiest_time_of_day(project_id, startTimestamp=TimeUTC.now(delta_days=-1 def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), value=None, **args): ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition @@ -1281,7 +1281,7 @@ def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") ch_sub_query_chart.append("isNotNull(pages.visually_complete)") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -1316,7 +1316,7 @@ def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(d endTimestamp=TimeUTC.now(), value=None, density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") ch_sub_query.append("isNotNull(pages.response_time)") ch_sub_query.append("pages.response_time>0") sch_sub_query = ch_sub_query[:] @@ -1351,7 +1351,7 @@ def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1) step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="performance", round_start=True, data=args) - ch_sub_query_chart.append("event_type='PERFORMANCE'") + ch_sub_query_chart.append("performance.event_type='PERFORMANCE'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -1384,7 +1384,7 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="performance", round_start=True, data=args) - ch_sub_query_chart.append("event_type='PERFORMANCE'") + ch_sub_query_chart.append("performance.event_type='PERFORMANCE'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -1417,7 +1417,7 @@ def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="performance", round_start=True, data=args) - ch_sub_query_chart.append("event_type='PERFORMANCE'") + ch_sub_query_chart.append("performance.event_type='PERFORMANCE'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -1532,7 +1532,7 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query = __get_basic_constraints(table_name="requests", round_start=True, data=args) - ch_sub_query.append("requests.EventType='REQUEST'") + ch_sub_query.append("requests.event_type='REQUEST'") ch_sub_query.append("intDiv(requests.status, 100) == %(status_code)s") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition @@ -1577,7 +1577,7 @@ def __get_domains_errors_4xx_and_5xx(status, project_id, startTimestamp=TimeUTC. endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query = __get_basic_constraints(table_name="requests", round_start=True, data=args) - ch_sub_query.append("requests.EventType='REQUEST'") + ch_sub_query.append("requests.event_type='REQUEST'") ch_sub_query.append("intDiv(requests.status, 100) == %(status_code)s") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition @@ -1658,7 +1658,7 @@ def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): ch_sub_query = __get_basic_constraints(table_name="requests", data=args) - ch_sub_query.append("requests.EventType = 'REQUEST'") + ch_sub_query.append("requests.event_type = 'REQUEST'") ch_sub_query.append("requests.success = 0") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition @@ -1731,7 +1731,7 @@ def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=- def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, **args): ch_sub_query = __get_basic_constraints(table_name="requests", data=args) - ch_sub_query.append("requests.EventType = 'REQUEST'") + ch_sub_query.append("requests.event_type = 'REQUEST'") ch_sub_query.append("intDiv(requests.status, 100) != 2") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition @@ -1758,7 +1758,7 @@ def __get_calls_errors_4xx_or_5xx(status, project_id, startTimestamp=TimeUTC.now endTimestamp=TimeUTC.now(), platform=None, **args): ch_sub_query = __get_basic_constraints(table_name="requests", data=args) - ch_sub_query.append("requests.EventType = 'REQUEST'") + ch_sub_query.append("requests.event_type = 'REQUEST'") ch_sub_query.append(f"intDiv(requests.status, 100) == {status}") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition @@ -1799,17 +1799,17 @@ def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), e step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="events", round_start=True, data=args) - ch_sub_query_chart.append("(events.EventType = 'REQUEST' OR events.EventType = 'ERROR')") - ch_sub_query_chart.append("(events.status>200 OR events.EventType = 'ERROR')") + ch_sub_query_chart.append("(events.event_type = 'REQUEST' OR events.event_type = 'ERROR')") + ch_sub_query_chart.append("(events.status>200 OR events.event_type = 'ERROR')") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - SUM(events.EventType = 'REQUEST' AND intDiv(events.status, 100) == 4) AS _4xx, - SUM(events.EventType = 'REQUEST' AND intDiv(events.status, 100) == 5) AS _5xx, - SUM(events.EventType = 'ERROR' AND events.source == 'js_exception') AS js, - SUM(events.EventType = 'ERROR' AND events.source != 'js_exception') AS integrations + SUM(events.event_type = 'REQUEST' AND intDiv(events.status, 100) == 4) AS _4xx, + SUM(events.event_type = 'REQUEST' AND intDiv(events.status, 100) == 5) AS _5xx, + SUM(events.event_type = 'ERROR' AND events.source == 'js_exception') AS js, + SUM(events.event_type = 'ERROR' AND events.source != 'js_exception') AS integrations FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS events WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1834,7 +1834,7 @@ def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_d ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) ch_sub_query_chart_response_end = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart_response_end.append("pages.EventType='LOCATION'") + ch_sub_query_chart_response_end.append("pages.event_type='LOCATION'") ch_sub_query_chart_response_end.append("isNotNull(pages.response_end)") ch_sub_query_chart_response_end.append("pages.response_end>0") meta_condition = __get_meta_constraint(args) @@ -1876,7 +1876,7 @@ def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(de endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="errors", round_start=True, data=args) - ch_sub_query_chart.append("errors.EventType='ERROR'") + ch_sub_query_chart.append("errors.event_type='ERROR'") ch_sub_query_chart.append("errors.source == 'js_exception'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -2012,9 +2012,9 @@ def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1) endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query = __get_basic_constraints(table_name="requests", round_start=True, data=args) - ch_sub_query.append("requests.EventType='REQUEST'") + ch_sub_query.append("requests.event_type='REQUEST'") ch_sub_query.append("requests.success = 0") - sch_sub_query = ["rs.project_id =toUInt16(%(project_id)s)", "rs.EventType='REQUEST'"] + sch_sub_query = ["rs.project_id =toUInt16(%(project_id)s)", "rs.event_type='REQUEST'"] meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition # sch_sub_query += meta_condition @@ -2073,7 +2073,7 @@ def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeU def __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args): ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition ch_sub_query.append("pages.load_event_end>0") @@ -2111,7 +2111,7 @@ def get_performance_avg_page_load_time(ch, project_id, startTimestamp=TimeUTC.no ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") ch_sub_query_chart += meta_condition ch_sub_query_chart.append("pages.load_event_end>0") @@ -2308,7 +2308,7 @@ def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeU def __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args): ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition ch_sub_query.append("pages.dom_content_loaded_event_end>0") @@ -2325,7 +2325,7 @@ def __get_page_metrics_avg_dom_content_load_start_chart(ch, project_id, startTim **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -2371,7 +2371,7 @@ def get_page_metrics_avg_first_contentful_pixel(project_id, startTimestamp=TimeU def __get_page_metrics_avg_first_contentful_pixel(ch, project_id, startTimestamp, endTimestamp, **args): ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition ch_sub_query.append("pages.first_contentful_paint_time>0") @@ -2390,7 +2390,7 @@ def __get_page_metrics_avg_first_contentful_pixel_chart(ch, project_id, startTim **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -2438,7 +2438,7 @@ def get_user_activity_avg_visited_pages(project_id, startTimestamp=TimeUTC.now(d def __get_user_activity_avg_visited_pages(ch, project_id, startTimestamp, endTimestamp, **args): ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition @@ -2459,7 +2459,7 @@ def __get_user_activity_avg_visited_pages(ch, project_id, startTimestamp, endTim def __get_user_activity_avg_visited_pages_chart(ch, project_id, startTimestamp, endTimestamp, density=20, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition @@ -2553,11 +2553,11 @@ def get_top_metrics_avg_response_time(project_id, startTimestamp=TimeUTC.now(del endTimestamp=TimeUTC.now(), value=None, density=20, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") ch_sub_query += meta_condition if value is not None: @@ -2592,11 +2592,11 @@ def get_top_metrics_count_requests(project_id, startTimestamp=TimeUTC.now(delta_ endTimestamp=TimeUTC.now(), value=None, density=20, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") ch_sub_query += meta_condition if value is not None: @@ -2631,12 +2631,12 @@ def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta endTimestamp=TimeUTC.now(), value=None, density=20, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") ch_sub_query += meta_condition if value is not None: @@ -2672,12 +2672,12 @@ def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.no endTimestamp=TimeUTC.now(), value=None, density=19, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") ch_sub_query += meta_condition if value is not None: @@ -2716,12 +2716,12 @@ def get_top_metrics_avg_till_first_bit(project_id, startTimestamp=TimeUTC.now(de endTimestamp=TimeUTC.now(), value=None, density=20, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") ch_sub_query += meta_condition if value is not None: @@ -2760,12 +2760,12 @@ def get_top_metrics_avg_time_to_interactive(project_id, startTimestamp=TimeUTC.n endTimestamp=TimeUTC.now(), value=None, density=20, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.EventType='LOCATION'") + ch_sub_query_chart.append("pages.event_type='LOCATION'") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.EventType='LOCATION'") + ch_sub_query.append("pages.event_type='LOCATION'") ch_sub_query += meta_condition if value is not None: From 9005e8f299f4d7e0b580435d4de41e28879cb0d6 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 10 Jan 2023 16:10:12 +0100 Subject: [PATCH 05/10] feat(chalice): fixed predefined_key change between PY/SQL --- api/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/schemas.py b/api/schemas.py index aef6a9567..7c735cd6c 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -894,8 +894,8 @@ class MetricType(str, Enum): class MetricOfErrors(str, Enum): calls_errors = "callsErrors" # calls_errors - domains_errors_4xx = "domainsErrors4Xx" # domains_errors_4xx - domains_errors_5xx = "domainsErrors5Xx" # domains_errors_5xx + domains_errors_4xx = "domainsErrors4xx" # domains_errors_4xx + domains_errors_5xx = "domainsErrors5xx" # domains_errors_5xx errors_per_domains = "errorsPerDomains" # errors_per_domains errors_per_type = "errorsPerType" # errors_per_type impacted_sessions_by_js_errors = "impactedSessionsByJsErrors" # impacted_sessions_by_js_errors From 4d599096d250ccd52626b04fcf867f984eb55fb5 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 10 Jan 2023 16:41:49 +0100 Subject: [PATCH 06/10] feat(chalice): cards remove useless series feat(chalice): cards handle missing insight-categories --- api/schemas.py | 3 +++ ee/api/schemas_ee.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/api/schemas.py b/api/schemas.py index 7c735cd6c..43dd83ef9 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -1016,6 +1016,9 @@ class CreateCardSchema(CardChartSchema): if values.get("metricType") == MetricType.funnel.value and \ values.get("series") is not None and len(values["series"]) > 1: values["series"] = [values["series"][0]] + elif values.get("metricType") not in (MetricType.table.value, MetricType.timeseries.value) and \ + values.get("series") is not None and len(values["series"]) > 0: + values["series"] = [] return values diff --git a/ee/api/schemas_ee.py b/ee/api/schemas_ee.py index bbcd62c9e..e226c68bb 100644 --- a/ee/api/schemas_ee.py +++ b/ee/api/schemas_ee.py @@ -53,6 +53,16 @@ class GetInsightsSchema(BaseModel): # time_step: int = Field(default=3600) categories: List[InsightCategories] = Field(..., min_items=1) + # This is used to handle wrong values sent by the UI + @root_validator(pre=True) + def transform(cls, values): + if values.get("categories") is None or len(values.get("categories")) == 0: + values["categories"] = [] + for v in InsightCategories: + values["categories"].append(v.value) + + return values + class Config: alias_generator = schemas.attribute_to_camel_case @@ -153,6 +163,17 @@ class CreateCardSchema(schemas.CreateCardSchema): def restrictions(cls, values): return values + # This is used to handle wrong values sent by the UI + @root_validator(pre=True) + def transform_ee(cls, values): + if values.get("metricType") == schemas.MetricType.insights and ( + values.get("metricValue") is None or len(values.get("metricValue")) == 0): + values["metricValue"] = [] + for v in InsightCategories: + values["metricValue"].append(v.value) + + return values + @root_validator def validator(cls, values): values = super().validator(values) From fff0eaf206a4e9c8393eb4f08613e1b0c1937c8a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 10 Jan 2023 16:58:28 +0100 Subject: [PATCH 07/10] feat(chalice): fixed get session changes --- ee/api/chalicelib/core/sessions.py | 4 ++-- ee/api/chalicelib/core/sessions_exp.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ee/api/chalicelib/core/sessions.py b/ee/api/chalicelib/core/sessions.py index 4aec1778d..3502c9110 100644 --- a/ee/api/chalicelib/core/sessions.py +++ b/ee/api/chalicelib/core/sessions.py @@ -87,7 +87,7 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu session_id=session_id) data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) else: - data['events'] = events.get_by_sessionId2_pg(project_id=project_id, session_id=session_id, + data['events'] = events.get_by_session_id(project_id=project_id, session_id=session_id, group_clickrage=True) all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] @@ -95,7 +95,7 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu # limit the number of errors to reduce the response-body size data['errors'] = [errors.format_first_stack_frame(e) for e in all_errors if e['source'] == "js_exception"][:500] - data['userEvents'] = events.get_customs_by_sessionId2_pg(project_id=project_id, + data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, session_id=session_id) data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index 3258b388a..271351fc9 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -105,16 +105,16 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu session_id=session_id) data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) else: - data['events'] = events.get_by_sessionId2_pg(project_id=project_id, session_id=session_id, - group_clickrage=True) + data['events'] = events.get_by_session_id(project_id=project_id, session_id=session_id, + group_clickrage=True) all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] # to keep only the first stack # limit the number of errors to reduce the response-body size data['errors'] = [errors.format_first_stack_frame(e) for e in all_errors if e['source'] == "js_exception"][:500] - data['userEvents'] = events.get_customs_by_sessionId2_pg(project_id=project_id, - session_id=session_id) + data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, + session_id=session_id) data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, From 51249127fd9d699d8d041d95b4deb2e4e7c022d4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 10 Jan 2023 17:10:35 +0100 Subject: [PATCH 08/10] feat(chalice): changed insights payload transform --- api/schemas.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/schemas.py b/api/schemas.py index 43dd83ef9..459f52faa 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -1016,8 +1016,10 @@ class CreateCardSchema(CardChartSchema): if values.get("metricType") == MetricType.funnel.value and \ values.get("series") is not None and len(values["series"]) > 1: values["series"] = [values["series"][0]] - elif values.get("metricType") not in (MetricType.table.value, MetricType.timeseries.value) and \ - values.get("series") is not None and len(values["series"]) > 0: + elif values.get("metricType") not in (MetricType.table.value, + MetricType.timeseries.value, + MetricType.insights.value) \ + and values.get("series") is not None and len(values["series"]) > 0: values["series"] = [] return values From 197c7fc8d8a6194768a162f6285e1fe64a8806ff Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 10 Jan 2023 17:17:31 +0100 Subject: [PATCH 09/10] feat(chalice): changed empty insights categories handling --- ee/api/chalicelib/core/sessions_insights.py | 4 ++++ ee/api/schemas_ee.py | 23 +-------------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/ee/api/chalicelib/core/sessions_insights.py b/ee/api/chalicelib/core/sessions_insights.py index 12ec73215..7d037224c 100644 --- a/ee/api/chalicelib/core/sessions_insights.py +++ b/ee/api/chalicelib/core/sessions_insights.py @@ -308,6 +308,10 @@ def query_click_rage_by_period(project_id, start_time, end_time, conn=None): def fetch_selected(project_id, data: schemas_ee.GetInsightsSchema): output = {} + if data.categories is None or len(data.categories) == 0: + data.categories = [] + for v in schemas_ee.InsightCategories: + data.categories.append(v) with ch_client.ClickHouseClient() as conn: if schemas_ee.InsightCategories.errors in data.categories: output[schemas_ee.InsightCategories.errors] = query_most_errors_by_period(project_id=project_id, diff --git a/ee/api/schemas_ee.py b/ee/api/schemas_ee.py index e226c68bb..ed785dcb2 100644 --- a/ee/api/schemas_ee.py +++ b/ee/api/schemas_ee.py @@ -51,17 +51,7 @@ class GetInsightsSchema(BaseModel): startTimestamp: int = Field(TimeUTC.now(-7)) endTimestamp: int = Field(TimeUTC.now()) # time_step: int = Field(default=3600) - categories: List[InsightCategories] = Field(..., min_items=1) - - # This is used to handle wrong values sent by the UI - @root_validator(pre=True) - def transform(cls, values): - if values.get("categories") is None or len(values.get("categories")) == 0: - values["categories"] = [] - for v in InsightCategories: - values["categories"].append(v.value) - - return values + categories: List[InsightCategories] = Field(...) class Config: alias_generator = schemas.attribute_to_camel_case @@ -163,17 +153,6 @@ class CreateCardSchema(schemas.CreateCardSchema): def restrictions(cls, values): return values - # This is used to handle wrong values sent by the UI - @root_validator(pre=True) - def transform_ee(cls, values): - if values.get("metricType") == schemas.MetricType.insights and ( - values.get("metricValue") is None or len(values.get("metricValue")) == 0): - values["metricValue"] = [] - for v in InsightCategories: - values["metricValue"].append(v.value) - - return values - @root_validator def validator(cls, values): values = super().validator(values) From 9af58ee8a39952825f8e1459da25a76004f157bb Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 10 Jan 2023 17:39:53 +0100 Subject: [PATCH 10/10] feat(chalice): fixed cards api after DB changes --- api/chalicelib/core/custom_metrics.py | 2 +- ee/api/chalicelib/core/custom_metrics.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 3c6e6fd68..b3df735a7 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -379,7 +379,7 @@ def search_all(project_id, user_id, data: schemas.SearchCardsSchema, include_ser query = cur.mogrify( f"""SELECT metric_id, project_id, user_id, name, is_public, created_at, edited_at, metric_type, metric_of, metric_format, metric_value, view_type, is_pinned, - predefined_key, dashboards, owner_email, default_config AS config, thumbnail + dashboards, owner_email, default_config AS config, thumbnail FROM metrics {sub_join} LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards diff --git a/ee/api/chalicelib/core/custom_metrics.py b/ee/api/chalicelib/core/custom_metrics.py index 406bf3c6a..3f9d3aaf7 100644 --- a/ee/api/chalicelib/core/custom_metrics.py +++ b/ee/api/chalicelib/core/custom_metrics.py @@ -409,7 +409,7 @@ def search_all(project_id, user_id, data: schemas.SearchCardsSchema, include_ser query = cur.mogrify( f"""SELECT metric_id, project_id, user_id, name, is_public, created_at, edited_at, metric_type, metric_of, metric_format, metric_value, view_type, is_pinned, - predefined_key, dashboards, owner_email, default_config AS config, thumbnail + dashboards, owner_email, default_config AS config, thumbnail FROM metrics {sub_join} LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards