jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/api-v1.5.5' into dev

Browse source
This commit is contained in:
Taha Yassine Kraiem 2022-04-22 12:45:53 +02:00
commit 7cb842fc51
7 changed files with 123 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
api/chalicelib/core/events.py
View file

@ -339,9 +339,9 @@ def __generic_autocomplete(event: Event):
class event_type:
CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label")
INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label")
LOCATION = Event(ui_type=schemas.EventType.location, table="events.pages", column="base_path")
LOCATION = Event(ui_type=schemas.EventType.location, table="events.pages", column="path")
CUSTOM = Event(ui_type=schemas.EventType.custom, table="events_common.customs", column="name")
REQUEST = Event(ui_type=schemas.EventType.request, table="events_common.requests", column="base_path")
REQUEST = Event(ui_type=schemas.EventType.request, table="events_common.requests", column="path")
GRAPHQL = Event(ui_type=schemas.EventType.graphql, table="events.graphql", column="name")
STATEACTION = Event(ui_type=schemas.EventType.state_action, table="events.state_actions", column="name")
ERROR = Event(ui_type=schemas.EventType.error, table="events.errors",

2
api/chalicelib/core/insights.py
View file

@ -21,7 +21,7 @@ def __transform_journey(rows):
JOURNEY_DEPTH = 5
JOURNEY_TYPES = {
"PAGES": {"table": "events.pages", "column": "base_path", "table_id": "message_id"},
"PAGES": {"table": "events.pages", "column": "path", "table_id": "message_id"},
"CLICK": {"table": "events.clicks", "column": "label", "table_id": "message_id"},
# "VIEW": {"table": "events_ios.views", "column": "name", "table_id": "seq_index"}, TODO: enable this for SAAS only
"EVENT": {"table": "events_common.customs", "column": "name", "table_id": "seq_index"}

4
api/chalicelib/core/sessions.py
View file

@ -353,8 +353,8 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d
full_args[arg_name] = metric_value[i]
extra_where = f"WHERE ({' OR '.join(extra_where)})"
elif metric_of == schemas.TableMetricOfType.visited_url:
main_col = "base_path"
extra_col = ", base_path"
main_col = "path"
extra_col = ", path"
main_query = cur.mogrify(f"""{pre_query}
SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS values
FROM (SELECT {main_col} AS name,

39
ee/scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql
View file

@ -53,12 +53,36 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets
);
ALTER TABLE events_common.requests
ADD COLUMN IF NOT EXISTS host text NULL,
ADD COLUMN IF NOT EXISTS base_path text NULL,
ADD COLUMN IF NOT EXISTS query text NULL;
ADD COLUMN IF NOT EXISTS host text NULL,
ADD COLUMN IF NOT EXISTS path text NULL,
ADD COLUMN IF NOT EXISTS query text NULL;
ALTER TABLE events.pages
ADD COLUMN IF NOT EXISTS query text NULL;
DO
$$
BEGIN
IF EXISTS(SELECT *
FROM information_schema.columns
WHERE table_schema = 'events'
AND table_name = 'pages'
AND column_name = 'base_path')
THEN
ALTER TABLE events.pages
DROP COLUMN IF EXISTS path;
ALTER TABLE events.pages
RENAME COLUMN base_path TO path;
DROP INDEX IF EXISTS events.pages_base_path_gin_idx2;
DROP INDEX IF EXISTS pages_base_path_idx2;
ALTER INDEX IF EXISTS events.pages_base_path_gin_idx RENAME TO pages_path_gin_idx;
ALTER INDEX IF EXISTS events.pages_base_path_idx RENAME TO pages_path_idx;
ALTER INDEX IF EXISTS events.pages_base_path_session_id_timestamp_idx RENAME TO pages_path_session_id_timestamp_idx;
ALTER INDEX IF EXISTS events.pages_base_path_base_pathLNGT2_idx RENAME TO pages_path_pathLNGT2_idx;
END IF;
END
$$;
COMMIT;
ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart';
@ -312,10 +336,13 @@ ON CONFLICT (predefined_key) DO UPDATE
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_path_nn_idx ON events_common.requests (path) WHERE path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_path_nn_gin_idx ON events_common.requests USING GIN (path gin_trgm_ops) WHERE path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp);
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2;

103
ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql
View file

@ -100,38 +100,36 @@ $$ LANGUAGE plpgsql;
DO
$$
BEGIN
IF (with to_check (name) as (
values ('alerts'),
('announcements'),
('assigned_sessions'),
('autocomplete'),
('basic_authentication'),
('dashboards'),
('dashboard_widgets'),
('errors'),
('funnels'),
('integrations'),
('issues'),
('jira_cloud'),
('jobs'),
('metric_series'),
('metrics'),
('notifications'),
('oauth_authentication'),
('projects'),
('roles'),
('roles_projects'),
('searches'),
('sessions'),
('tenants'),
('traces'),
('user_favorite_errors'),
('user_favorite_sessions'),
('user_viewed_errors'),
('user_viewed_sessions'),
('users'),
('webhooks')
)
IF (with to_check (name) as (values ('alerts'),
('announcements'),
('assigned_sessions'),
('autocomplete'),
('basic_authentication'),
('dashboards'),
('dashboard_widgets'),
('errors'),
('funnels'),
('integrations'),
('issues'),
('jira_cloud'),
('jobs'),
('metric_series'),
('metrics'),
('notifications'),
('oauth_authentication'),
('projects'),
('roles'),
('roles_projects'),
('searches'),
('sessions'),
('tenants'),
('traces'),
('user_favorite_errors'),
('user_favorite_sessions'),
('user_viewed_errors'),
('user_viewed_sessions'),
('users'),
('webhooks'))
select bool_and(exists(select *
from information_schema.tables t
where table_schema = 'public'
@ -914,16 +912,14 @@ LANGUAGE plpgsql;
DO
$$
BEGIN
IF (with to_check (name) as (
values ('clicks'),
('errors'),
('graphql'),
('inputs'),
('pages'),
('performance'),
('resources'),
('state_actions')
)
IF (with to_check (name) as (values ('clicks'),
('errors'),
('graphql'),
('inputs'),
('pages'),
('performance'),
('resources'),
('state_actions'))
select bool_and(exists(select *
from information_schema.tables t
where table_schema = 'events'
@ -938,7 +934,6 @@ $$
timestamp bigint NOT NULL,
host text NOT NULL,
path text NOT NULL,
base_path text NOT NULL,
query text NULL,
referrer text DEFAULT NULL,
base_referrer text DEFAULT NULL,
@ -956,13 +951,9 @@ $$
PRIMARY KEY (session_id, message_id)
);
CREATE INDEX IF NOT EXISTS pages_session_id_idx ON events.pages (session_id);
CREATE INDEX IF NOT EXISTS pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops);
CREATE INDEX IF NOT EXISTS pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops);
CREATE INDEX IF NOT EXISTS pages_timestamp_idx ON events.pages (timestamp);
CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_idx ON events.pages (session_id, timestamp);
CREATE INDEX IF NOT EXISTS pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops);
CREATE INDEX IF NOT EXISTS pages_base_path_idx ON events.pages (base_path);
CREATE INDEX IF NOT EXISTS pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1));
CREATE INDEX IF NOT EXISTS pages_base_referrer_idx ON events.pages (base_referrer);
CREATE INDEX IF NOT EXISTS pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer,
length(base_referrer) -
@ -997,8 +988,8 @@ $$
0;
CREATE INDEX IF NOT EXISTS pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL;
CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL;
CREATE INDEX IF NOT EXISTS pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp);
CREATE INDEX IF NOT EXISTS pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2;
CREATE INDEX IF NOT EXISTS pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp);
CREATE INDEX IF NOT EXISTS pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2;
CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
@ -1176,11 +1167,9 @@ LANGUAGE plpgsql;
DO
$$
BEGIN
IF (with to_check (name) as (
values ('customs'),
('issues'),
('requests')
)
IF (with to_check (name) as (values ('customs'),
('issues'),
('requests'))
select bool_and(exists(select *
from information_schema.tables t
where table_schema = 'events_common'
@ -1238,7 +1227,7 @@ $$
status_code smallint NULL,
method http_method NULL,
host text NULL,
base_path text NULL,
path text NULL,
query text NULL,
PRIMARY KEY (session_id, timestamp, seq_index)
);
@ -1263,8 +1252,8 @@ $$
CREATE INDEX IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_path_nn_idx ON events_common.requests (path) WHERE path IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_path_nn_gin_idx ON events_common.requests USING GIN (path gin_trgm_ops) WHERE path IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;

37
scripts/helm/db/init_dbs/postgresql/1.6.0/1.6.0.sql
View file

@ -53,12 +53,36 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets
);
ALTER TABLE events_common.requests
ADD COLUMN IF NOT EXISTS host text NULL,
ADD COLUMN IF NOT EXISTS base_path text NULL,
ADD COLUMN IF NOT EXISTS query text NULL;
ADD COLUMN IF NOT EXISTS host text NULL,
ADD COLUMN IF NOT EXISTS path text NULL,
ADD COLUMN IF NOT EXISTS query text NULL;
ALTER TABLE events.pages
ADD COLUMN IF NOT EXISTS query text NULL;
DO
$$
BEGIN
IF EXISTS(SELECT *
FROM information_schema.columns
WHERE table_schema = 'events'
AND table_name = 'pages'
AND column_name = 'base_path')
THEN
ALTER TABLE events.pages
DROP COLUMN IF EXISTS path;
ALTER TABLE events.pages
RENAME COLUMN base_path TO path;
DROP INDEX IF EXISTS events.pages_base_path_gin_idx2;
DROP INDEX IF EXISTS pages_base_path_idx2;
ALTER INDEX IF EXISTS events.pages_base_path_gin_idx RENAME TO pages_path_gin_idx;
ALTER INDEX IF EXISTS events.pages_base_path_idx RENAME TO pages_path_idx;
ALTER INDEX IF EXISTS events.pages_base_path_session_id_timestamp_idx RENAME TO pages_path_session_id_timestamp_idx;
ALTER INDEX IF EXISTS events.pages_base_path_base_pathLNGT2_idx RENAME TO pages_path_pathLNGT2_idx;
END IF;
END
$$;
COMMIT;
ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart';
@ -312,10 +336,13 @@ ON CONFLICT (predefined_key) DO UPDATE
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_path_nn_idx ON events_common.requests (path) WHERE path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_path_nn_gin_idx ON events_common.requests USING GIN (path gin_trgm_ops) WHERE path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp);
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2;

21
scripts/helm/db/init_dbs/postgresql/init_schema.sql
View file

@ -600,7 +600,6 @@ $$
-- --- events_common.sql ---
CREATE SCHEMA IF NOT EXISTS events_common;
CREATE TYPE events_common.custom_level AS ENUM ('info','error');
@ -646,7 +645,7 @@ $$
status_code smallint NULL,
method http_method NULL,
host text NULL,
base_path text NULL,
path text NULL,
query text NULL,
PRIMARY KEY (session_id, timestamp, seq_index)
);
@ -669,13 +668,12 @@ $$
CREATE INDEX requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL;
CREATE INDEX requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL;
CREATE INDEX requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL;
CREATE INDEX requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL;
CREATE INDEX requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL;
CREATE INDEX requests_path_nn_idx ON events_common.requests (path) WHERE path IS NOT NULL;
CREATE INDEX requests_path_nn_gin_idx ON events_common.requests USING GIN (path gin_trgm_ops) WHERE path IS NOT NULL;
CREATE INDEX requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL;
CREATE INDEX requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
-- --- events.sql ---
CREATE SCHEMA IF NOT EXISTS events;
CREATE TABLE events.pages
(
@ -684,7 +682,6 @@ $$
timestamp bigint NOT NULL,
host text NOT NULL,
path text NOT NULL,
base_path text NOT NULL,
query text NULL,
referrer text DEFAULT NULL,
base_referrer text DEFAULT NULL,
@ -702,13 +699,9 @@ $$
PRIMARY KEY (session_id, message_id)
);
CREATE INDEX pages_session_id_idx ON events.pages (session_id);
CREATE INDEX pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops);
CREATE INDEX pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops);
CREATE INDEX pages_timestamp_idx ON events.pages (timestamp);
CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp);
CREATE INDEX pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops);
CREATE INDEX pages_base_path_idx ON events.pages (base_path);
CREATE INDEX pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1));
CREATE INDEX pages_base_referrer_idx ON events.pages (base_referrer);
CREATE INDEX pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer,
length(base_referrer) - (CASE
@ -739,10 +732,10 @@ $$
time_to_interactive > 0;
CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL;
CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL;
CREATE INDEX pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp);
CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2;
CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE INDEX pages_path_session_id_timestamp_idx ON events.pages (path, session_id, timestamp);
CREATE INDEX pages_path_pathLNGT2_idx ON events.pages (path) WHERE length(path) > 2;
CREATE INDEX pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE TABLE events.clicks