From 87fcccd291a0645704a6dedc37e6a05815bd75e7 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 7 Mar 2022 21:36:32 +0100 Subject: [PATCH] feat(utilities): changed vars feat(DB): changed graphql schema feat(api): removed graphql unused filters --- api/chalicelib/core/sessions.py | 6 -- api/schemas.py | 2 - .../db/init_dbs/postgresql/1.5.3/1.5.3.sql | 12 ++-- .../db/init_dbs/postgresql/init_schema.sql | 56 ++++++++++--------- frontend/app/constants/filterOptions.js | 12 ++-- .../db/init_dbs/postgresql/1.5.3/1.5.3.sql | 11 ++-- .../db/init_dbs/postgresql/init_schema.sql | 54 ++++++++++-------- 7 files changed, 76 insertions(+), 77 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index bb5e9b974..7ce5a888e 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -863,15 +863,9 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr event_where.append( _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value, value_key=e_k_f)) - elif f.type == schemas.GraphqlFilterType._status_code: - event_where.append( - _multiple_conditions(f"main.status_code {op} %({e_k_f})s", f.value, value_key=e_k_f)) elif f.type == schemas.GraphqlFilterType._method: event_where.append( _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) - elif f.type == schemas.GraphqlFilterType._duration: - event_where.append( - _multiple_conditions(f"main.duration {op} %({e_k_f})s", f.value, value_key=e_k_f)) elif f.type == schemas.GraphqlFilterType._request_body: event_where.append( _multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) diff --git a/api/schemas.py b/api/schemas.py index 062900fc2..a3403b15b 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -515,9 +515,7 @@ class FetchFilterType(str, Enum): class GraphqlFilterType(str, Enum): _name = "GRAPHQL_NAME" - _status_code = "GRAPHQL_STATUS_CODE" _method = "GRAPHQL_METHOD" - _duration = "GRAPHQL_DURATION" _request_body = "GRAPHQL_REQUEST_BODY" _response_body = "GRAPHQL_RESPONSE_BODY" diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql index 1e08dfeae..887eb4ce7 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql @@ -70,9 +70,7 @@ LANGUAGE plpgsql; ALTER TABLE events.graphql ADD COLUMN IF NOT EXISTS request_body text NULL, ADD COLUMN IF NOT EXISTS response_body text NULL, - ADD COLUMN IF NOT EXISTS status_code smallint NULL, - ADD COLUMN IF NOT EXISTS method http_method NULL, - ADD COLUMN IF NOT EXISTS duration integer NULL; + ADD COLUMN IF NOT EXISTS method http_method NULL; ALTER TABLE events_common.requests ADD COLUMN IF NOT EXISTS request_body text NULL, @@ -83,6 +81,9 @@ ALTER TABLE events_common.requests UPDATE tenants SET version_number= openreplay_version(); + +ALTER TABLE projects + ADD COLUMN save_request_payloads boolean NOT NULL DEFAULT FALSE; COMMIT; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; @@ -94,7 +95,4 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_status_code_nn_idx ON events_co CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_duration_nn_gt0_idx ON events.graphql (duration) WHERE duration IS NOT NULL AND duration > 0; - +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index ae724f60b..633e64caa 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -290,26 +290,27 @@ $$ CREATE TABLE IF NOT EXISTS projects ( - project_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_key varchar(20) NOT NULL UNIQUE DEFAULT generate_api_key(20), - tenant_id integer NOT NULL REFERENCES tenants (tenant_id) ON DELETE CASCADE, - name text NOT NULL, - active boolean NOT NULL, - sample_rate smallint NOT NULL DEFAULT 100 CHECK (sample_rate >= 0 AND sample_rate <= 100), - created_at timestamp without time zone NOT NULL DEFAULT (now() at time zone 'utc'), - deleted_at timestamp without time zone NULL DEFAULT NULL, - max_session_duration integer NOT NULL DEFAULT 7200000, - metadata_1 text DEFAULT NULL, - metadata_2 text DEFAULT NULL, - metadata_3 text DEFAULT NULL, - metadata_4 text DEFAULT NULL, - metadata_5 text DEFAULT NULL, - metadata_6 text DEFAULT NULL, - metadata_7 text DEFAULT NULL, - metadata_8 text DEFAULT NULL, - metadata_9 text DEFAULT NULL, - metadata_10 text DEFAULT NULL, - gdpr jsonb NOT NULL DEFAULT'{ + project_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_key varchar(20) NOT NULL UNIQUE DEFAULT generate_api_key(20), + tenant_id integer NOT NULL REFERENCES tenants (tenant_id) ON DELETE CASCADE, + name text NOT NULL, + active boolean NOT NULL, + sample_rate smallint NOT NULL DEFAULT 100 CHECK (sample_rate >= 0 AND sample_rate <= 100), + created_at timestamp without time zone NOT NULL DEFAULT (now() at time zone 'utc'), + deleted_at timestamp without time zone NULL DEFAULT NULL, + max_session_duration integer NOT NULL DEFAULT 7200000, + metadata_1 text DEFAULT NULL, + metadata_2 text DEFAULT NULL, + metadata_3 text DEFAULT NULL, + metadata_4 text DEFAULT NULL, + metadata_5 text DEFAULT NULL, + metadata_6 text DEFAULT NULL, + metadata_7 text DEFAULT NULL, + metadata_8 text DEFAULT NULL, + metadata_9 text DEFAULT NULL, + metadata_10 text DEFAULT NULL, + save_request_payloads boolean NOT NULL DEFAULT FALSE, + gdpr jsonb NOT NULL DEFAULT'{ "maskEmails": true, "sampleRate": 33, "maskNumbers": false, @@ -1001,10 +1002,13 @@ $$ END IF; CREATE TABLE IF NOT EXISTS events.graphql ( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - message_id bigint NOT NULL, - timestamp bigint NOT NULL, - name text NOT NULL, + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + message_id bigint NOT NULL, + timestamp bigint NOT NULL, + name text NOT NULL, + request_body text NULL, + response_body text NULL, + method http_method NULL, PRIMARY KEY (session_id, message_id) ); CREATE INDEX IF NOT EXISTS graphql_name_idx ON events.graphql (name); @@ -1014,9 +1018,7 @@ $$ CREATE INDEX IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; CREATE INDEX IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; CREATE INDEX IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; - CREATE INDEX IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; - CREATE INDEX IF NOT EXISTS graphql_duration_nn_gt0_idx ON events.graphql (duration) WHERE duration IS NOT NULL AND duration > 0; - + CREATE TABLE IF NOT EXISTS events.state_actions ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, diff --git a/frontend/app/constants/filterOptions.js b/frontend/app/constants/filterOptions.js index 560ad7912..03df3ab51 100644 --- a/frontend/app/constants/filterOptions.js +++ b/frontend/app/constants/filterOptions.js @@ -62,12 +62,12 @@ export const metricTypes = [ ]; export const tableColumnName = { - [FilterKey.USERID]: 'User', - [FilterKey.ISSUE]: 'Issue', - [FilterKey.USER_BROWSER]: 'Browser', - [FilterKey.USER_DEVICE]: 'Device', - [FilterKey.USER_COUNTRY]: 'Country', - [FilterKey.LOCATION]: 'URL', + [FilterKey.USERID]: 'Users', + [FilterKey.ISSUE]: 'Issues', + [FilterKey.USER_BROWSER]: 'Browsers', + [FilterKey.USER_DEVICE]: 'Devices', + [FilterKey.USER_COUNTRY]: 'Countries', + [FilterKey.LOCATION]: 'URLs', } export const metricOf = [ diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql b/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql index 1030ba55d..08b1630f2 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.3/1.5.3.sql @@ -68,9 +68,7 @@ LANGUAGE plpgsql; ALTER TABLE events.graphql ADD COLUMN IF NOT EXISTS request_body text NULL, ADD COLUMN IF NOT EXISTS response_body text NULL, - ADD COLUMN IF NOT EXISTS status_code smallint NULL, - ADD COLUMN IF NOT EXISTS method http_method NULL, - ADD COLUMN IF NOT EXISTS duration integer NULL; + ADD COLUMN IF NOT EXISTS method http_method NULL; ALTER TABLE events_common.requests ADD COLUMN IF NOT EXISTS request_body text NULL, @@ -81,6 +79,9 @@ ALTER TABLE events_common.requests UPDATE tenants SET version_number= openreplay_version(); +ALTER TABLE projects + ADD COLUMN save_request_payloads boolean NOT NULL DEFAULT FALSE; + COMMIT; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; @@ -92,6 +93,4 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_status_code_nn_idx ON events_co CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_duration_nn_gt0_idx ON events.graphql (duration) WHERE duration IS NOT NULL AND duration > 0; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index c7d1f2110..f1d141861 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -236,25 +236,26 @@ $$ CREATE TABLE projects ( - project_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_key varchar(20) NOT NULL UNIQUE DEFAULT generate_api_key(20), - name text NOT NULL, - active boolean NOT NULL, - sample_rate smallint NOT NULL DEFAULT 100 CHECK (sample_rate >= 0 AND sample_rate <= 100), - created_at timestamp without time zone NOT NULL DEFAULT (now() at time zone 'utc'), - deleted_at timestamp without time zone NULL DEFAULT NULL, - max_session_duration integer NOT NULL DEFAULT 7200000, - metadata_1 text DEFAULT NULL, - metadata_2 text DEFAULT NULL, - metadata_3 text DEFAULT NULL, - metadata_4 text DEFAULT NULL, - metadata_5 text DEFAULT NULL, - metadata_6 text DEFAULT NULL, - metadata_7 text DEFAULT NULL, - metadata_8 text DEFAULT NULL, - metadata_9 text DEFAULT NULL, - metadata_10 text DEFAULT NULL, - gdpr jsonb NOT NULL DEFAULT '{ + project_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_key varchar(20) NOT NULL UNIQUE DEFAULT generate_api_key(20), + name text NOT NULL, + active boolean NOT NULL, + sample_rate smallint NOT NULL DEFAULT 100 CHECK (sample_rate >= 0 AND sample_rate <= 100), + created_at timestamp without time zone NOT NULL DEFAULT (now() at time zone 'utc'), + deleted_at timestamp without time zone NULL DEFAULT NULL, + max_session_duration integer NOT NULL DEFAULT 7200000, + metadata_1 text DEFAULT NULL, + metadata_2 text DEFAULT NULL, + metadata_3 text DEFAULT NULL, + metadata_4 text DEFAULT NULL, + metadata_5 text DEFAULT NULL, + metadata_6 text DEFAULT NULL, + metadata_7 text DEFAULT NULL, + metadata_8 text DEFAULT NULL, + metadata_9 text DEFAULT NULL, + metadata_10 text DEFAULT NULL, + save_request_payloads boolean NOT NULL DEFAULT FALSE, + gdpr jsonb NOT NULL DEFAULT '{ "maskEmails": true, "sampleRate": 33, "maskNumbers": false, @@ -787,15 +788,22 @@ $$ CREATE TABLE events.graphql ( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - message_id bigint NOT NULL, - timestamp bigint NOT NULL, - name text NOT NULL, + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + message_id bigint NOT NULL, + timestamp bigint NOT NULL, + name text NOT NULL, + request_body text NULL, + response_body text NULL, + method http_method NULL, PRIMARY KEY (session_id, message_id) ); CREATE INDEX graphql_name_idx ON events.graphql (name); CREATE INDEX graphql_name_gin_idx ON events.graphql USING GIN (name gin_trgm_ops); CREATE INDEX graphql_timestamp_idx ON events.graphql (timestamp); + CREATE INDEX graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; + CREATE INDEX graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; + CREATE INDEX graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; + CREATE INDEX graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; CREATE TABLE events.state_actions (