Dev (#2336)

* refactor(chalice): upgraded dependencies

* refactor(chalice): upgraded dependencies
feat(chalice): support heatmaps

* feat(chalice): support table-of-browsers showing user-count

* feat(chalice): support table-of-devices showing user-count

* feat(chalice): support table-of-URLs showing user-count

* fix(chalice): fixed Math-operators validation
refactor(chalice): search for sessions that have events for heatmaps

* refactor(chalice): search for sessions that have at least 1 location event for heatmaps

* fix(DB): replaced click's x&y with normalized_x&normalized_y
fix(chalice): fixed heatmaps-clicks-path
refactor(DB): DB upgrade & downgrade scripts

api/chalicelib/core/heatmaps.py

@@ -254,7 +254,7 @@ def get_page_events(session_id):
                     message_id,
                     timestamp,
                     host,
-                    path
+                    path,
                     query,
                     path AS value,
                     path AS url,

ee/api/chalicelib/core/heatmaps.py

@@ -399,7 +399,7 @@ else:
                         message_id,
                         timestamp,
                         host,
-                        path
+                        path,
                         query,
                         path AS value,
                         path AS url,

ee/scripts/schema/db/init_dbs/clickhouse/1.19.0/1.19.0.sql

@@ -1,6 +1,84 @@
 CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.19.0-ee';
 
+DROP TABLE IF EXISTS experimental.events_l7d_mv;
 
 ALTER TABLE experimental.events
     ADD COLUMN IF NOT EXISTS normalized_x Nullable(UInt8),
-    ADD COLUMN IF NOT EXISTS normalized_y Nullable(UInt8);
+    ADD COLUMN IF NOT EXISTS normalized_y Nullable(UInt8),
+    DROP COLUMN IF EXISTS coordinate;
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv
+            ENGINE = ReplacingMergeTree(_timestamp)
+                PARTITION BY toYYYYMMDD(datetime)
+                ORDER BY (project_id, datetime, event_type, session_id, message_id)
+                TTL datetime + INTERVAL 7 DAY
+            POPULATE
+AS
+SELECT session_id,
+       project_id,
+       event_type,
+       datetime,
+       label,
+       hesitation_time,
+       name,
+       payload,
+       level,
+       source,
+       message,
+       error_id,
+       duration,
+       context,
+       url,
+       url_host,
+       url_path,
+       url_hostpath,
+       request_start,
+       response_start,
+       response_end,
+       dom_content_loaded_event_start,
+       dom_content_loaded_event_end,
+       load_event_start,
+       load_event_end,
+       first_paint,
+       first_contentful_paint_time,
+       speed_index,
+       visually_complete,
+       time_to_interactive,
+       ttfb,
+       ttlb,
+       response_time,
+       dom_building_time,
+       dom_content_loaded_event_time,
+       load_event_time,
+       min_fps,
+       avg_fps,
+       max_fps,
+       min_cpu,
+       avg_cpu,
+       max_cpu,
+       min_total_js_heap_size,
+       avg_total_js_heap_size,
+       max_total_js_heap_size,
+       min_used_js_heap_size,
+       avg_used_js_heap_size,
+       max_used_js_heap_size,
+       method,
+       status,
+       success,
+       request_body,
+       response_body,
+       issue_type,
+       issue_id,
+       error_tags_keys,
+       error_tags_values,
+       transfer_size,
+       selector,
+       normalized_x,
+       normalized_y,
+       message_id,
+       _timestamp
+FROM experimental.events
+WHERE datetime >= now() - INTERVAL 7 DAY;
+
+ALTER TABLE experimental.ios_events
+    DROP COLUMN IF EXISTS coordinate;

ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql

@@ -81,7 +81,8 @@ CREATE TABLE IF NOT EXISTS experimental.events
     error_tags_values              Array(Nullable(String)),
     transfer_size                  Nullable(UInt32),
     selector                       Nullable(String),
-    coordinate                     Tuple(x Nullable(UInt16), y Nullable(UInt16)),
+    normalized_x                   Nullable(UInt8),
+    normalized_y                   Nullable(UInt8),
     message_id                     UInt64                               DEFAULT 0,
     _timestamp                     DateTime                             DEFAULT now()
 ) ENGINE = ReplacingMergeTree(_timestamp)
@@ -281,7 +282,8 @@ SELECT session_id,
        error_tags_values,
        transfer_size,
        selector,
-       coordinate,
+       normalized_x,
+       normalized_y,
        message_id,
        _timestamp
 FROM experimental.events
@@ -441,7 +443,6 @@ CREATE TABLE IF NOT EXISTS experimental.ios_events
     issue_type     Nullable(Enum8('tap_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19,'mouse_thrashing'=20,'app_crash'=21)),
     issue_id       Nullable(String),
     transfer_size  Nullable(UInt32),
-    coordinate     Tuple(x Nullable(UInt16), y Nullable(UInt16)),
     direction      Nullable(String),
     reason         Nullable(String),
     stacktrace     Nullable(String),

ee/scripts/schema/db/init_dbs/postgresql/1.19.0/1.19.0.sql

@@ -20,7 +20,9 @@ $fn_def$, :'next_version')
 --
 ALTER TABLE IF EXISTS events.clicks
     ADD COLUMN IF NOT EXISTS normalized_x smallint NULL,
-    ADD COLUMN IF NOT EXISTS normalized_y smallint NULL;
+    ADD COLUMN IF NOT EXISTS normalized_y smallint NULL,
+    DROP COLUMN IF EXISTS x,
+    DROP COLUMN IF EXISTS y;
 
 UPDATE public.metrics
 SET default_config=default_config || '{"col":2}'

ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql

@@ -659,16 +659,16 @@ CREATE INDEX pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_op
 
 CREATE TABLE events.clicks
 (
-    session_id bigint             NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE,
+    session_id   bigint              NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE,
-    message_id bigint             NOT NULL,
+    message_id   bigint              NOT NULL,
-    timestamp  bigint             NOT NULL,
+    timestamp    bigint              NOT NULL,
-    label      text    DEFAULT NULL,
+    label        text     DEFAULT NULL,
-    url        text    DEFAULT '' NOT NULL,
+    url          text     DEFAULT '' NOT NULL,
-    path       text,
+    path         text,
-    selector   text    DEFAULT '' NOT NULL,
+    selector     text     DEFAULT '' NOT NULL,
-    hesitation integer DEFAULT NULL,
+    hesitation   integer  DEFAULT NULL,
-    x          integer DEFAULT NULL,
+    normalized_x smallint DEFAULT NULL,
-    y          integer DEFAULT NULL,
+    normalized_y smallint DEFAULT NULL,
     PRIMARY KEY (session_id, message_id)
 );
 CREATE INDEX clicks_session_id_idx ON events.clicks (session_id);

ee/scripts/schema/db/rollback_dbs/clickhouse/1.19.0/1.19.0.sql

@@ -0,0 +1,81 @@
+CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.18.0-ee';
+
+DROP TABLE IF EXISTS experimental.events_l7d_mv;
+
+ALTER TABLE experimental.events
+    ADD COLUMN IF NOT EXISTS coordinate Tuple(x Nullable(UInt16), y Nullable(UInt16));
+
+ALTER TABLE experimental.ios_events
+    ADD COLUMN IF NOT EXISTS coordinate Tuple(x Nullable(UInt16), y Nullable(UInt16));
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv
+            ENGINE = ReplacingMergeTree(_timestamp)
+                PARTITION BY toYYYYMMDD(datetime)
+                ORDER BY (project_id, datetime, event_type, session_id, message_id)
+                TTL datetime + INTERVAL 7 DAY
+            POPULATE
+AS
+SELECT session_id,
+       project_id,
+       event_type,
+       datetime,
+       label,
+       hesitation_time,
+       name,
+       payload,
+       level,
+       source,
+       message,
+       error_id,
+       duration,
+       context,
+       url,
+       url_host,
+       url_path,
+       url_hostpath,
+       request_start,
+       response_start,
+       response_end,
+       dom_content_loaded_event_start,
+       dom_content_loaded_event_end,
+       load_event_start,
+       load_event_end,
+       first_paint,
+       first_contentful_paint_time,
+       speed_index,
+       visually_complete,
+       time_to_interactive,
+       ttfb,
+       ttlb,
+       response_time,
+       dom_building_time,
+       dom_content_loaded_event_time,
+       load_event_time,
+       min_fps,
+       avg_fps,
+       max_fps,
+       min_cpu,
+       avg_cpu,
+       max_cpu,
+       min_total_js_heap_size,
+       avg_total_js_heap_size,
+       max_total_js_heap_size,
+       min_used_js_heap_size,
+       avg_used_js_heap_size,
+       max_used_js_heap_size,
+       method,
+       status,
+       success,
+       request_body,
+       response_body,
+       issue_type,
+       issue_id,
+       error_tags_keys,
+       error_tags_values,
+       transfer_size,
+       selector,
+       coordinate,
+       message_id,
+       _timestamp
+FROM experimental.events
+WHERE datetime >= now() - INTERVAL 7 DAY;

ee/scripts/schema/db/rollback_dbs/postgresql/1.19.0/1.19.0.sql

@@ -0,0 +1,32 @@
+\set previous_version 'v1.19.0-ee'
+\set next_version 'v1.18.0-ee'
+SELECT openreplay_version()                       AS current_version,
+       openreplay_version() = :'previous_version' AS valid_previous,
+       openreplay_version() = :'next_version'     AS is_next
+\gset
+
+\if :valid_previous
+\echo valid previous DB version :'previous_version', starting DB downgrade to :'next_version'
+BEGIN;
+SELECT format($fn_def$
+CREATE OR REPLACE FUNCTION openreplay_version()
+    RETURNS text AS
+$$
+SELECT '%1$s'
+$$ LANGUAGE sql IMMUTABLE;
+$fn_def$, :'next_version')
+\gexec
+
+--
+
+ALTER TABLE IF EXISTS events.clicks
+    ADD COLUMN IF NOT EXISTS x integer DEFAULT NULL,
+    ADD COLUMN IF NOT EXISTS y integer DEFAULT NULL;
+
+COMMIT;
+
+\elif :is_next
+\echo new version detected :'next_version', nothing to do
+\else
+\warn skipping DB downgrade of :'next_version', expected previous version :'previous_version', found :'current_version'
+\endif

scripts/schema/db/init_dbs/postgresql/1.19.0/1.19.0.sql

@@ -20,7 +20,9 @@ $fn_def$, :'next_version')
 --
 ALTER TABLE IF EXISTS events.clicks
     ADD COLUMN IF NOT EXISTS normalized_x smallint NULL,
-    ADD COLUMN IF NOT EXISTS normalized_y smallint NULL;
+    ADD COLUMN IF NOT EXISTS normalized_y smallint NULL,
+    DROP COLUMN IF EXISTS x,
+    DROP COLUMN IF EXISTS y;
 
 UPDATE public.metrics
 SET default_config=default_config || '{"col":2}'

scripts/schema/db/init_dbs/postgresql/init_schema.sql

@@ -620,16 +620,16 @@ CREATE INDEX pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_op
 
 CREATE TABLE events.clicks
 (
-    session_id bigint             NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE,
+    session_id   bigint              NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE,
-    message_id bigint             NOT NULL,
+    message_id   bigint              NOT NULL,
-    timestamp  bigint             NOT NULL,
+    timestamp    bigint              NOT NULL,
-    label      text    DEFAULT NULL,
+    label        text     DEFAULT NULL,
-    url        text    DEFAULT '' NOT NULL,
+    url          text     DEFAULT '' NOT NULL,
-    path       text,
+    path         text,
-    selector   text    DEFAULT '' NOT NULL,
+    selector     text     DEFAULT '' NOT NULL,
-    hesitation integer DEFAULT NULL,
+    hesitation   integer  DEFAULT NULL,
-    x          integer DEFAULT NULL,
+    normalized_x smallint DEFAULT NULL,
-    y          integer DEFAULT NULL,
+    normalized_y smallint DEFAULT NULL,
     PRIMARY KEY (session_id, message_id)
 );
 CREATE INDEX clicks_session_id_idx ON events.clicks (session_id);

scripts/schema/db/rollback_dbs/postgresql/1.19.0/1.19.0.sql

@@ -0,0 +1,32 @@
+\set previous_version 'v1.19.0'
+\set next_version 'v1.18.0'
+SELECT openreplay_version()                       AS current_version,
+       openreplay_version() = :'previous_version' AS valid_previous,
+       openreplay_version() = :'next_version'     AS is_next
+\gset
+
+\if :valid_previous
+\echo valid previous DB version :'previous_version', starting DB downgrade to :'next_version'
+BEGIN;
+SELECT format($fn_def$
+CREATE OR REPLACE FUNCTION openreplay_version()
+    RETURNS text AS
+$$
+SELECT '%1$s'
+$$ LANGUAGE sql IMMUTABLE;
+$fn_def$, :'next_version')
+\gexec
+
+--
+
+ALTER TABLE IF EXISTS events.clicks
+    ADD COLUMN IF NOT EXISTS x integer DEFAULT NULL,
+    ADD COLUMN IF NOT EXISTS y integer DEFAULT NULL;
+
+COMMIT;
+
+\elif :is_next
+\echo new version detected :'next_version', nothing to do
+\else
+\warn skipping DB downgrade of :'next_version', expected previous version :'previous_version', found :'current_version'
+\endif