From 5ef720e1c7b900acb09893ab70a7c59c9cbbc1f4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 4 Mar 2022 17:12:07 +0100 Subject: [PATCH] feat(DB): DB structure changes --- .../db/init_dbs/postgresql/1.5.99/1.5.99.sql | 90 ++++++++++++++++++- .../db/init_dbs/postgresql/init_schema.sql | 52 +++++++++-- .../db/init_dbs/postgresql/1.5.99/1.5.99.sql | 89 +++++++++++++++++- .../db/init_dbs/postgresql/1.5.X/1.5.X.sql | 78 ---------------- .../db/init_dbs/postgresql/init_schema.sql | 33 +++++-- 5 files changed, 248 insertions(+), 94 deletions(-) delete mode 100644 scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql index 88e1aab74..936e56968 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql @@ -51,4 +51,92 @@ ALTER TABLE metrics ADD COLUMN IF NOT EXISTS metric_format text; -COMMIT; \ No newline at end of file + +DO +$$ + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + INNER JOIN pg_namespace nsp + ON nsp.oid = typ.typnamespace + WHERE typ.typname = 'http_method') THEN + CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); + END IF; + END; +$$ +LANGUAGE plpgsql; + + +ALTER TABLE events.graphql + ADD COLUMN IF NOT EXISTS request_body text NULL, + ADD COLUMN IF NOT EXISTS response_body text NULL, + ADD COLUMN IF NOT EXISTS status_code smallint NULL, + ADD COLUMN IF NOT EXISTS method http_method NULL, + ADD COLUMN IF NOT EXISTS duration integer NULL; + +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS request_body text NULL, + ADD COLUMN IF NOT EXISTS response_body text NULL, + ADD COLUMN IF NOT EXISTS status_code smallint NULL, + ADD COLUMN IF NOT EXISTS method http_method NULL; + +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS schema text NULL, + ADD COLUMN IF NOT EXISTS host text NULL, + ADD COLUMN IF NOT EXISTS base_path text NULL, + ADD COLUMN IF NOT EXISTS query_string text NULL; + +COMMIT; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_duration_nn_gt0_idx ON events.graphql (duration) WHERE duration IS NOT NULL AND duration > 0; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_idx ON events_common.requests (query_string) WHERE query_string IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_gin_idx ON events_common.requests USING GIN (query_string gin_trgm_ops) WHERE query_string IS NOT NULL; + + + +-- Split requests-URL: Takes too long to use +-- UPDATE events_common.requests +-- SET schema=CASE WHEN POSITION('://' IN url) > 0 THEN SUBSTRING(url, 1, POSITION('://' IN url) - 1) END, +-- host=CASE +-- WHEN POSITION('://' IN url) = 0 THEN NULL +-- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( +-- SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), 1, +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) +-- ELSE SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1) END, +-- base_path=CASE +-- WHEN POSITION('://' IN url) = 0 THEN +-- CASE +-- WHEN POSITION('?' IN url) > 0 THEN +-- SUBSTRING(url, 1, POSITION('?' IN url) - 1) +-- ELSE url END +-- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN +-- CASE +-- WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN +-- SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1, +-- POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) +-- ELSE SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + +-- 1) END +-- END, +-- query_string=CASE +-- WHEN POSITION('?' IN url) > 0 THEN SUBSTRING(url, POSITION('?' IN url) + 1) +-- END; + diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 1f0f4d6f1..92e25c9e1 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -994,7 +994,11 @@ $$ CREATE INDEX IF NOT EXISTS errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id); CREATE INDEX IF NOT EXISTS errors_error_id_idx ON events.errors (error_id); - + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'http_method') THEN + CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); + END IF; CREATE TABLE IF NOT EXISTS events.graphql ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, @@ -1006,6 +1010,12 @@ $$ CREATE INDEX IF NOT EXISTS graphql_name_idx ON events.graphql (name); CREATE INDEX IF NOT EXISTS graphql_name_gin_idx ON events.graphql USING GIN (name gin_trgm_ops); CREATE INDEX IF NOT EXISTS graphql_timestamp_idx ON events.graphql (timestamp); + CREATE INDEX IF NOT EXISTS graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; + CREATE INDEX IF NOT EXISTS graphql_duration_nn_gt0_idx ON events.graphql (duration) WHERE duration IS NOT NULL AND duration > 0; CREATE TABLE IF NOT EXISTS events.state_actions ( @@ -1147,15 +1157,27 @@ $$ CREATE INDEX IF NOT EXISTS issues_issue_id_timestamp_idx ON events_common.issues (issue_id, timestamp); CREATE INDEX IF NOT EXISTS issues_timestamp_idx ON events_common.issues (timestamp); - + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'http_method') THEN + CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); + END IF; CREATE TABLE IF NOT EXISTS events_common.requests ( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - timestamp bigint NOT NULL, - seq_index integer NOT NULL, - url text NOT NULL, - duration integer NOT NULL, - success boolean NOT NULL, + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + seq_index integer NOT NULL, + url text NOT NULL, + duration integer NOT NULL, + success boolean NOT NULL, + request_body text NULL, + response_body text NULL, + status_code smallint NULL, + method http_method NULL, + schema text NULL, + host text NULL, + base_path text NULL, + query_string text NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); CREATE INDEX IF NOT EXISTS requests_url_idx ON events_common.requests (url); @@ -1172,6 +1194,20 @@ $$ ELSE 0 END)) gin_trgm_ops); CREATE INDEX IF NOT EXISTS requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; + CREATE INDEX IF NOT EXISTS requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + + CREATE INDEX IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_query_string_nn_idx ON events_common.requests (query_string) WHERE query_string IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_query_string_nn_gin_idx ON events_common.requests USING GIN (query_string gin_trgm_ops) WHERE query_string IS NOT NULL; + + END IF; END; $$ diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql b/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql index b471511f4..e661e526a 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.99/1.5.99.sql @@ -50,4 +50,91 @@ ALTER TABLE metrics ADD COLUMN IF NOT EXISTS metric_format text; -COMMIT; \ No newline at end of file +DO +$$ + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + INNER JOIN pg_namespace nsp + ON nsp.oid = typ.typnamespace + WHERE typ.typname = 'http_method') THEN + CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); + END IF; + END; +$$ +LANGUAGE plpgsql; + + +ALTER TABLE events.graphql + ADD COLUMN IF NOT EXISTS request_body text NULL, + ADD COLUMN IF NOT EXISTS response_body text NULL, + ADD COLUMN IF NOT EXISTS status_code smallint NULL, + ADD COLUMN IF NOT EXISTS method http_method NULL, + ADD COLUMN IF NOT EXISTS duration integer NULL; + +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS request_body text NULL, + ADD COLUMN IF NOT EXISTS response_body text NULL, + ADD COLUMN IF NOT EXISTS status_code smallint NULL, + ADD COLUMN IF NOT EXISTS method http_method NULL; + +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS schema text NULL, + ADD COLUMN IF NOT EXISTS host text NULL, + ADD COLUMN IF NOT EXISTS base_path text NULL, + ADD COLUMN IF NOT EXISTS query_string text NULL; + +COMMIT; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_duration_nn_gt0_idx ON events.graphql (duration) WHERE duration IS NOT NULL AND duration > 0; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_idx ON events_common.requests (query_string) WHERE query_string IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_gin_idx ON events_common.requests USING GIN (query_string gin_trgm_ops) WHERE query_string IS NOT NULL; + + + +-- Split requests-URL: Takes too long to use +-- UPDATE events_common.requests +-- SET schema=CASE WHEN POSITION('://' IN url) > 0 THEN SUBSTRING(url, 1, POSITION('://' IN url) - 1) END, +-- host=CASE +-- WHEN POSITION('://' IN url) = 0 THEN NULL +-- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( +-- SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), 1, +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) +-- ELSE SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1) END, +-- base_path=CASE +-- WHEN POSITION('://' IN url) = 0 THEN +-- CASE +-- WHEN POSITION('?' IN url) > 0 THEN +-- SUBSTRING(url, 1, POSITION('?' IN url) - 1) +-- ELSE url END +-- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN +-- CASE +-- WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN +-- SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1, +-- POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) +-- ELSE SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), +-- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + +-- 1) END +-- END, +-- query_string=CASE +-- WHEN POSITION('?' IN url) > 0 THEN SUBSTRING(url, POSITION('?' IN url) + 1) +-- END; + diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql b/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql deleted file mode 100644 index 2b71f3ac4..000000000 --- a/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql +++ /dev/null @@ -1,78 +0,0 @@ --- !!!!NOT IN CREATE DB YET - -BEGIN; -CREATE OR REPLACE FUNCTION openreplay_version() - RETURNS text AS -$$ -SELECT 'v1.5.X' -$$ LANGUAGE sql IMMUTABLE; - -CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); - -ALTER TABLE events_common.requests - ADD COLUMN IF NOT EXISTS schema text NULL, - ADD COLUMN IF NOT EXISTS host text NULL, - ADD COLUMN IF NOT EXISTS base_path text NULL, - ADD COLUMN IF NOT EXISTS query_string text NULL, - ADD COLUMN IF NOT EXISTS request_body text NULL, - ADD COLUMN IF NOT EXISTS response_body text NULL, - ADD COLUMN IF NOT EXISTS status_code smallint NULL, - ADD COLUMN IF NOT EXISTS method http_method NULL; - --- Split requests-URL: Takes too long to use --- UPDATE events_common.requests --- SET schema=CASE WHEN POSITION('://' IN url) > 0 THEN SUBSTRING(url, 1, POSITION('://' IN url) - 1) END, --- host=CASE --- WHEN POSITION('://' IN url) = 0 THEN NULL --- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( --- SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), 1, --- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) --- ELSE SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1) END, --- base_path=CASE --- WHEN POSITION('://' IN url) = 0 THEN --- CASE --- WHEN POSITION('?' IN url) > 0 THEN --- SUBSTRING(url, 1, POSITION('?' IN url) - 1) --- ELSE url END --- WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN --- CASE --- WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN --- SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), --- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1, --- POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - --- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) --- ELSE SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), --- POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + --- 1) END --- END, --- query_string=CASE --- WHEN POSITION('?' IN url) > 0 THEN SUBSTRING(url, POSITION('?' IN url) + 1) --- END; - - -ALTER TABLE events.graphql - ADD COLUMN IF NOT EXISTS request_body text NULL, - ADD COLUMN IF NOT EXISTS response_body text NULL, - ADD COLUMN IF NOT EXISTS status_code smallint NULL, - ADD COLUMN IF NOT EXISTS method http_method NULL, - ADD COLUMN IF NOT EXISTS duration integer NULL; - -COMMIT; - -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_idx ON events_common.requests (query_string) WHERE query_string IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_gin_idx ON events_common.requests USING GIN (query_string gin_trgm_ops) WHERE query_string IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; - -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 232596696..57fae7794 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -631,14 +631,23 @@ $$ CREATE INDEX issues_timestamp_idx ON events_common.issues (timestamp); CREATE INDEX issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); + CREATE TYPE http_method AS ENUM ('GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'); CREATE TABLE events_common.requests ( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - timestamp bigint NOT NULL, - seq_index integer NOT NULL, - url text NOT NULL, - duration integer NOT NULL, - success boolean NOT NULL, + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + seq_index integer NOT NULL, + url text NOT NULL, + duration integer NOT NULL, + success boolean NOT NULL, + request_body text NULL, + response_body text NULL, + status_code smallint NULL, + method http_method NULL, + schema text NULL, + host text NULL, + base_path text NULL, + query_string text NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); CREATE INDEX requests_url_idx ON events_common.requests (url); @@ -653,6 +662,18 @@ $$ ELSE 0 END)) gin_trgm_ops); CREATE INDEX requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; + CREATE INDEX requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; + CREATE INDEX requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; + CREATE INDEX requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; + CREATE INDEX requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; + CREATE INDEX requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + CREATE INDEX requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; + CREATE INDEX requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; + CREATE INDEX requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; + CREATE INDEX requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; + CREATE INDEX requests_query_string_nn_idx ON events_common.requests (query_string) WHERE query_string IS NOT NULL; + CREATE INDEX requests_query_string_nn_gin_idx ON events_common.requests USING GIN (query_string gin_trgm_ops) WHERE query_string IS NOT NULL; + -- --- events.sql --- CREATE SCHEMA IF NOT EXISTS events;