From 5ef720e1c7b900acb09893ab70a7c59c9cbbc1f4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 4 Mar 2022 17:12:07 +0100 Subject: [PATCH 1/4] feat(DB): DB structure changes --- .../db/init_dbs/postgresql/1.5.99/1.5.99.sql | 90 ++++++++++++++++++- .../db/init_dbs/postgresql/init_schema.sql | 52 +++++++++-- .../db/init_dbs/postgresql/1.5.99/1.5.99.sql | 89 +++++++++++++++++- .../db/init_dbs/postgresql/1.5.X/1.5.X.sql | 78 ---------------- .../db/init_dbs/postgresql/init_schema.sql | 33 +++++-- 5 files changed, 248 insertions(+), 94 deletions(-) delete mode 100644 scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql index 88e1aab74..936e56968 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql @@ -51,4 +51,92 @@ ALTER TABLE metrics ADD COLUMN IF NOT EXISTS metric_format text; -COMMIT; \ No newline at end of file + +DO +$$ + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + INNER JOIN pg_namespace nsp + ON nsp.oid = typ.typnamespace + WHERE typ.typname = 'http_method') THEN + CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); + END IF; + END; +$$ +LANGUAGE plpgsql; + + +ALTER TABLE events.graphql + ADD COLUMN IF NOT EXISTS request_body text NULL, + ADD COLUMN IF NOT EXISTS response_body text NULL, + ADD COLUMN IF NOT EXISTS status_code smallint NULL, + ADD COLUMN IF NOT EXISTS method http_method NULL, + ADD COLUMN IF NOT EXISTS duration integer NULL; + +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS request_body text NULL, + ADD COLUMN IF NOT EXISTS response_body text NULL, + ADD COLUMN IF NOT EXISTS status_code smallint NULL, + ADD COLUMN IF NOT EXISTS method http_method NULL; + +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS schema text NULL, + ADD COLUMN IF NOT EXISTS host text NULL, + ADD COLUMN IF NOT EXISTS base_path text NULL, + ADD COLUMN IF NOT EXISTS query_string text NULL; + +COMMIT; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_duration_nn_gt0_idx ON events.graphql (duration) WHERE duration IS NOT NULL AND duration > 0; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_idx ON events_common.requests (query_string) WHERE query_string IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_gin_idx ON events_common.requests USING GIN (query_string gin_trgm_ops) WHERE query_string IS NOT NULL; + + + +-- Split requests-URL: Takes too long to use +-- UPDATE events_common.requests +-- SET schema=CASE WHEN POSITION('://' IN url) > 0 THEN SUBSTRING(url, 1, POSITION('://' IN url) - 1) END, +-- host=CASE +-- WHEN POSITION('://' IN url) = 0 THEN NULL +-- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( +-- SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), 1, +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) +-- ELSE SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1) END, +-- base_path=CASE +-- WHEN POSITION('://' IN url) = 0 THEN +-- CASE +-- WHEN POSITION('?' IN url) > 0 THEN +-- SUBSTRING(url, 1, POSITION('?' IN url) - 1) +-- ELSE url END +-- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN +-- CASE +-- WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN +-- SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1, +-- POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) +-- ELSE SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + +-- 1) END +-- END, +-- query_string=CASE +-- WHEN POSITION('?' IN url) > 0 THEN SUBSTRING(url, POSITION('?' IN url) + 1) +-- END; + diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 1f0f4d6f1..92e25c9e1 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -994,7 +994,11 @@ $$ CREATE INDEX IF NOT EXISTS errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id); CREATE INDEX IF NOT EXISTS errors_error_id_idx ON events.errors (error_id); - + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'http_method') THEN + CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); + END IF; CREATE TABLE IF NOT EXISTS events.graphql ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, @@ -1006,6 +1010,12 @@ $$ CREATE INDEX IF NOT EXISTS graphql_name_idx ON events.graphql (name); CREATE INDEX IF NOT EXISTS graphql_name_gin_idx ON events.graphql USING GIN (name gin_trgm_ops); CREATE INDEX IF NOT EXISTS graphql_timestamp_idx ON events.graphql (timestamp); + CREATE INDEX IF NOT EXISTS graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; + CREATE INDEX IF NOT EXISTS graphql_duration_nn_gt0_idx ON events.graphql (duration) WHERE duration IS NOT NULL AND duration > 0; CREATE TABLE IF NOT EXISTS events.state_actions ( @@ -1147,15 +1157,27 @@ $$ CREATE INDEX IF NOT EXISTS issues_issue_id_timestamp_idx ON events_common.issues (issue_id, timestamp); CREATE INDEX IF NOT EXISTS issues_timestamp_idx ON events_common.issues (timestamp); - + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'http_method') THEN + CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); + END IF; CREATE TABLE IF NOT EXISTS events_common.requests ( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - timestamp bigint NOT NULL, - seq_index integer NOT NULL, - url text NOT NULL, - duration integer NOT NULL, - success boolean NOT NULL, + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + seq_index integer NOT NULL, + url text NOT NULL, + duration integer NOT NULL, + success boolean NOT NULL, + request_body text NULL, + response_body text NULL, + status_code smallint NULL, + method http_method NULL, + schema text NULL, + host text NULL, + base_path text NULL, + query_string text NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); CREATE INDEX IF NOT EXISTS requests_url_idx ON events_common.requests (url); @@ -1172,6 +1194,20 @@ $$ ELSE 0 END)) gin_trgm_ops); CREATE INDEX IF NOT EXISTS requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; + CREATE INDEX IF NOT EXISTS requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + + CREATE INDEX IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_query_string_nn_idx ON events_common.requests (query_string) WHERE query_string IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_query_string_nn_gin_idx ON events_common.requests USING GIN (query_string gin_trgm_ops) WHERE query_string IS NOT NULL; + + END IF; END; $$ diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql b/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql index b471511f4..e661e526a 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql @@ -50,4 +50,91 @@ ALTER TABLE metrics ADD COLUMN IF NOT EXISTS metric_format text; -COMMIT; \ No newline at end of file +DO +$$ + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + INNER JOIN pg_namespace nsp + ON nsp.oid = typ.typnamespace + WHERE typ.typname = 'http_method') THEN + CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); + END IF; + END; +$$ +LANGUAGE plpgsql; + + +ALTER TABLE events.graphql + ADD COLUMN IF NOT EXISTS request_body text NULL, + ADD COLUMN IF NOT EXISTS response_body text NULL, + ADD COLUMN IF NOT EXISTS status_code smallint NULL, + ADD COLUMN IF NOT EXISTS method http_method NULL, + ADD COLUMN IF NOT EXISTS duration integer NULL; + +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS request_body text NULL, + ADD COLUMN IF NOT EXISTS response_body text NULL, + ADD COLUMN IF NOT EXISTS status_code smallint NULL, + ADD COLUMN IF NOT EXISTS method http_method NULL; + +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS schema text NULL, + ADD COLUMN IF NOT EXISTS host text NULL, + ADD COLUMN IF NOT EXISTS base_path text NULL, + ADD COLUMN IF NOT EXISTS query_string text NULL; + +COMMIT; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_duration_nn_gt0_idx ON events.graphql (duration) WHERE duration IS NOT NULL AND duration > 0; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_idx ON events_common.requests (query_string) WHERE query_string IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_gin_idx ON events_common.requests USING GIN (query_string gin_trgm_ops) WHERE query_string IS NOT NULL; + + + +-- Split requests-URL: Takes too long to use +-- UPDATE events_common.requests +-- SET schema=CASE WHEN POSITION('://' IN url) > 0 THEN SUBSTRING(url, 1, POSITION('://' IN url) - 1) END, +-- host=CASE +-- WHEN POSITION('://' IN url) = 0 THEN NULL +-- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( +-- SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), 1, +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) +-- ELSE SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1) END, +-- base_path=CASE +-- WHEN POSITION('://' IN url) = 0 THEN +-- CASE +-- WHEN POSITION('?' IN url) > 0 THEN +-- SUBSTRING(url, 1, POSITION('?' IN url) - 1) +-- ELSE url END +-- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN +-- CASE +-- WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN +-- SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1, +-- POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) +-- ELSE SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + +-- 1) END +-- END, +-- query_string=CASE +-- WHEN POSITION('?' IN url) > 0 THEN SUBSTRING(url, POSITION('?' IN url) + 1) +-- END; + diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql b/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql deleted file mode 100644 index 2b71f3ac4..000000000 --- a/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql +++ /dev/null @@ -1,78 +0,0 @@ --- !!!!NOT IN CREATE DB YET - -BEGIN; -CREATE OR REPLACE FUNCTION openreplay_version() - RETURNS text AS -$$ -SELECT 'v1.5.X' -$$ LANGUAGE sql IMMUTABLE; - -CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); - -ALTER TABLE events_common.requests - ADD COLUMN IF NOT EXISTS schema text NULL, - ADD COLUMN IF NOT EXISTS host text NULL, - ADD COLUMN IF NOT EXISTS base_path text NULL, - ADD COLUMN IF NOT EXISTS query_string text NULL, - ADD COLUMN IF NOT EXISTS request_body text NULL, - ADD COLUMN IF NOT EXISTS response_body text NULL, - ADD COLUMN IF NOT EXISTS status_code smallint NULL, - ADD COLUMN IF NOT EXISTS method http_method NULL; - --- Split requests-URL: Takes too long to use --- UPDATE events_common.requests --- SET schema=CASE WHEN POSITION('://' IN url) > 0 THEN SUBSTRING(url, 1, POSITION('://' IN url) - 1) END, --- host=CASE --- WHEN POSITION('://' IN url) = 0 THEN NULL --- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( --- SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), 1, --- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) --- ELSE SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1) END, --- base_path=CASE --- WHEN POSITION('://' IN url) = 0 THEN --- CASE --- WHEN POSITION('?' IN url) > 0 THEN --- SUBSTRING(url, 1, POSITION('?' IN url) - 1) --- ELSE url END --- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN --- CASE --- WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN --- SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), --- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1, --- POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - --- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) --- ELSE SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), --- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + --- 1) END --- END, --- query_string=CASE --- WHEN POSITION('?' IN url) > 0 THEN SUBSTRING(url, POSITION('?' IN url) + 1) --- END; - - -ALTER TABLE events.graphql - ADD COLUMN IF NOT EXISTS request_body text NULL, - ADD COLUMN IF NOT EXISTS response_body text NULL, - ADD COLUMN IF NOT EXISTS status_code smallint NULL, - ADD COLUMN IF NOT EXISTS method http_method NULL, - ADD COLUMN IF NOT EXISTS duration integer NULL; - -COMMIT; - -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_idx ON events_common.requests (query_string) WHERE query_string IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_gin_idx ON events_common.requests USING GIN (query_string gin_trgm_ops) WHERE query_string IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; - -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 232596696..57fae7794 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -631,14 +631,23 @@ $$ CREATE INDEX issues_timestamp_idx ON events_common.issues (timestamp); CREATE INDEX issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); + CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); CREATE TABLE events_common.requests ( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - timestamp bigint NOT NULL, - seq_index integer NOT NULL, - url text NOT NULL, - duration integer NOT NULL, - success boolean NOT NULL, + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + seq_index integer NOT NULL, + url text NOT NULL, + duration integer NOT NULL, + success boolean NOT NULL, + request_body text NULL, + response_body text NULL, + status_code smallint NULL, + method http_method NULL, + schema text NULL, + host text NULL, + base_path text NULL, + query_string text NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); CREATE INDEX requests_url_idx ON events_common.requests (url); @@ -653,6 +662,18 @@ $$ ELSE 0 END)) gin_trgm_ops); CREATE INDEX requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; + CREATE INDEX requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; + CREATE INDEX requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; + CREATE INDEX requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; + CREATE INDEX requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; + CREATE INDEX requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + CREATE INDEX requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; + CREATE INDEX requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; + CREATE INDEX requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; + CREATE INDEX requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; + CREATE INDEX requests_query_string_nn_idx ON events_common.requests (query_string) WHERE query_string IS NOT NULL; + CREATE INDEX requests_query_string_nn_gin_idx ON events_common.requests USING GIN (query_string gin_trgm_ops) WHERE query_string IS NOT NULL; + -- --- events.sql --- CREATE SCHEMA IF NOT EXISTS events; From 9028e67dfa430e3b96f18aedc9d9bdb4cd240fc4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 4 Mar 2022 17:20:05 +0100 Subject: [PATCH 2/4] feat(DB): DB changed version number --- .../init_dbs/postgresql/{1.5.99/1.5.99.sql => 1.5.3/1.5.3.sql} | 2 +- ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql | 2 +- .../init_dbs/postgresql/{1.5.99/1.5.99.sql => 1.5.3/1.5.3.sql} | 2 +- scripts/helm/db/init_dbs/postgresql/init_schema.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename ee/scripts/helm/db/init_dbs/postgresql/{1.5.99/1.5.99.sql => 1.5.3/1.5.3.sql} (99%) rename scripts/helm/db/init_dbs/postgresql/{1.5.99/1.5.99.sql => 1.5.3/1.5.3.sql} (99%) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql similarity index 99% rename from ee/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql rename to ee/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql index 936e56968..399a2199f 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql @@ -2,7 +2,7 @@ BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.5.X-ee' +SELECT 'v1.5.3-ee' $$ LANGUAGE sql IMMUTABLE; UPDATE metrics diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 92e25c9e1..6a81a107d 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -7,7 +7,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.5.2-ee' +SELECT 'v1.5.3-ee' $$ LANGUAGE sql IMMUTABLE; diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql b/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql similarity index 99% rename from scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql rename to scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql index e661e526a..0e25a7396 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql @@ -2,7 +2,7 @@ BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.5.X' +SELECT 'v1.5.3' $$ LANGUAGE sql IMMUTABLE; UPDATE metrics diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 57fae7794..fb2e7bb63 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS events; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.5.2' +SELECT 'v1.5.3' $$ LANGUAGE sql IMMUTABLE; -- --- accounts.sql --- From 08d00727e8824495b85a329e5cf3f138693056af Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 4 Mar 2022 19:04:18 +0100 Subject: [PATCH 3/4] feat(api): autocomplete for FETCH_URL feat(api): no alerts for table-custom-metrics --- api/chalicelib/core/alerts_processor.py | 26 ------------------------- api/chalicelib/core/custom_metrics.py | 1 + api/routers/core.py | 5 +++-- 3 files changed, 4 insertions(+), 28 deletions(-) diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index 80973fadd..21249773c 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -119,12 +119,6 @@ def Build(a): q = f"""SELECT coalesce(value,0) AS value, coalesce(value,0) {a["query"]["operator"]} {a["query"]["right"]} AS valid""" - # if len(colDef.group) > 0 { - # subQ = subQ.Column(colDef.group + " AS group_value") - # subQ = subQ.GroupBy(colDef.group) - # q = q.Column("group_value") - # } - if a["detectionMethod"] == schemas.AlertDetectionMethod.threshold: if a["seriesId"] is not None: q += f""" FROM ({subQ}) AS stat""" @@ -134,16 +128,6 @@ def Build(a): params = {**params, **full_args, "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000} else: if a["options"]["change"] == schemas.AlertDetectionChangeType.change: - # if len(colDef.group) > 0: - # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) - # sub2, args2, _ := subQ.Where( - # sq.And{ - # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), - # sq.Expr("timestamp>=$4 ", time.Now().Unix()-2 * a.Options.CurrentPeriod * 60), - # }).ToSql() - # sub1 := sq.Select("group_value", "(stat1.value-stat2.value) AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) - # q = q.FromSelect(sub1, "stat") - # else: if a["seriesId"] is not None: sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") sub1 = f"SELECT (({subQ})-({sub2})) AS value" @@ -163,16 +147,6 @@ def Build(a): q += f" FROM ( {sub1} ) AS stat" else: - # if len(colDef.group) >0 { - # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) - # sub2, args2, _ := subQ.Where( - # sq.And{ - # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), - # sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod * 60-a.Options.CurrentPeriod * 60), - # }).ToSql() - # sub1 := sq.Select("group_value", "(stat1.value/stat2.value-1)*100 AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) - # q = q.FromSelect(sub1, "stat") - # } else { if a["seriesId"] is not None: sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") sub1 = f"SELECT (({subQ})/NULLIF(({sub2}),0)-1)*100 AS value" diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 06a2c328b..6c8e7ddc3 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -288,6 +288,7 @@ def get_series_for_alert(project_id, user_id): INNER JOIN metrics USING (metric_id) WHERE metrics.deleted_at ISNULL AND metrics.project_id = %(project_id)s + AND metrics.metric_type = 'timeseries' AND (user_id = %(user_id)s OR is_public) ORDER BY name;""", {"project_id": project_id, "user_id": user_id} diff --git a/api/routers/core.py b/api/routers/core.py index 8aacdb99b..d523176e1 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -101,7 +101,8 @@ def comment_assignment(projectId: int, sessionId: int, issueId: str, data: schem @app.get('/{projectId}/events/search', tags=["events"]) def events_search(projectId: int, q: str, - type: Union[schemas.FilterType, schemas.EventType, schemas.PerformanceEventType] = None, + type: Union[schemas.FilterType, schemas.EventType, + schemas.PerformanceEventType, schemas.FetchFilterType._url] = None, key: str = None, source: str = None, context: schemas.CurrentContext = Depends(OR_context)): if len(q) == 0: @@ -114,7 +115,7 @@ def events_search(projectId: int, q: str, schemas.PerformanceEventType.location_avg_memory_usage ]: type = schemas.EventType.location - elif type in [schemas.PerformanceEventType.fetch_failed]: + elif type in [schemas.PerformanceEventType.fetch_failed, schemas.FetchFilterType._url]: type = schemas.EventType.request else: return {"data": []} From 1d04f7f372b6e573053da3903051673b6aebcf93 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 4 Mar 2022 19:11:55 +0100 Subject: [PATCH 4/4] feat(api): custom_metrics changed VISITED_URL to LOCATION --- api/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/schemas.py b/api/schemas.py index ac50aad71..062900fc2 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -762,7 +762,7 @@ class TableMetricOfType(str, Enum): user_country = FilterType.user_country.value user_id = FilterType.user_id.value issues = FilterType.issue.value - visited_url = "VISITED_URL" + visited_url = EventType.location.value class TimeseriesMetricOfType(str, Enum):