jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(chalice): search sessions by click-selector

Browse source 
feat(DB): clicks-selector index
feat(DB): metrics changes
feat(DB): support multi-upgrade
This commit is contained in:
Taha Yassine Kraiem 2022-12-15 12:58:18 +01:00
parent 86101c454d
commit 4da33a891e
7 changed files with 186 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
api/chalicelib/core/sessions.py
View file

@ -1,4 +1,4 @@
from typing import List
from typing import List, Union
import schemas
from chalicelib.core import events, metadata, events_ios, \
@ -114,7 +114,7 @@ def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_
return None
def __get_sql_operator(op: schemas.SearchEventOperator):
def __get_sql_operator(op: Union[schemas.SearchEventOperator, schemas.ClickEventExtraOperator]):
return {
schemas.SearchEventOperator._is: "=",
schemas.SearchEventOperator._is_any: "IN",
@ -684,9 +684,13 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
if event_type == events.event_type.CLICK.ui_type:
event_from = event_from % f"{events.event_type.CLICK.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value,
value_key=e_k))
if event.operator == schemas.ClickEventExtraOperator._on_selector:
event_where.append(
_multiple_conditions(f"main.selector = %({e_k})s", event.value, value_key=e_k))
else:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value,
value_key=e_k))
elif event_type == events.event_type.INPUT.ui_type:
event_from = event_from % f"{events.event_type.INPUT.table} AS main "

10
api/schemas.py
View file

@ -446,6 +446,11 @@ class SearchEventOperator(str, Enum):
_ends_with = "endsWith"
class ClickEventExtraOperator(str, Enum):
_on_selector = "onSelector"
_on_text = "onText"
class PlatformType(str, Enum):
mobile = "mobile"
desktop = "desktop"
@ -531,7 +536,7 @@ class _SessionSearchEventRaw(__MixedSearchFilter):
is_event: bool = Field(default=True, const=True)
value: List[str] = Field(...)
type: Union[EventType, PerformanceEventType] = Field(...)
operator: SearchEventOperator = Field(...)
operator: Union[SearchEventOperator, ClickEventExtraOperator] = Field(...)
source: Optional[List[Union[ErrorSource, int, str]]] = Field(None)
sourceOperator: Optional[MathOperator] = Field(None)
filters: Optional[List[RequestGraphqlFilterSchema]] = Field(None)
@ -570,6 +575,9 @@ class _SessionSearchEventRaw(__MixedSearchFilter):
assert isinstance(values.get("filters"), List) and len(values.get("filters", [])) > 0, \
f"filters should be defined for {EventType.graphql.value}"
if isinstance(values.get("operator"), ClickEventExtraOperator):
assert values.get("type") == EventType.click, \
f"operator:{values['operator']} is only available for event-type: {EventType.click}"
return values

14
ee/api/chalicelib/core/sessions.py
View file

@ -1,4 +1,4 @@
from typing import List
from typing import List, Union
import schemas
import schemas_ee
@ -117,7 +117,7 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu
return None
def __get_sql_operator(op: schemas.SearchEventOperator):
def __get_sql_operator(op: Union[schemas.SearchEventOperator, schemas.ClickEventExtraOperator]):
return {
schemas.SearchEventOperator._is: "=",
schemas.SearchEventOperator._is_any: "IN",
@ -687,9 +687,13 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
if event_type == events.event_type.CLICK.ui_type:
event_from = event_from % f"{events.event_type.CLICK.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value,
value_key=e_k))
if event.operator == schemas.ClickEventExtraOperator._on_selector:
event_where.append(
_multiple_conditions(f"main.selector = %({e_k})s", event.value, value_key=e_k))
else:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value,
value_key=e_k))
elif event_type == events.event_type.INPUT.ui_type:
event_from = event_from % f"{events.event_type.INPUT.table} AS main "

119
ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql
View file

@ -20,70 +20,73 @@ CREATE TABLE IF NOT EXISTS assist_records
ALTER TYPE webhook_type ADD VALUE IF NOT EXISTS 'msteams';
UPDATE metrics_clone
SET metric_of=CASE
WHEN metric_of = 'USEROS' THEN 'userOS'
WHEN metric_of = 'USERBROWSER' THEN 'userBrowser'
WHEN metric_of = 'USERDEVICE' THEN 'userDevice'
WHEN metric_of = 'USERCOUNTRY' THEN 'userCountry'
WHEN metric_of = 'USERID' THEN 'userId'
WHEN metric_of = 'ISSUE' THEN 'issue'
WHEN metric_of = 'LOCATION' THEN 'location'
WHEN metric_of = 'SESSIONS' THEN 'sessions'
WHEN metric_of = 'js_exception' THEN 'jsException'
WHEN metric_of = 'sessionCount' THEN 'sessionCount'
END
WHERE NOT is_predefined;
DO
$$
BEGIN
IF EXISTS(SELECT column_name
FROM information_schema.columns
WHERE table_name = 'metrics'
and column_name = 'is_predefined') THEN
-- 1. pre transform structure
ALTER TABLE IF EXISTS metrics_clone
ALTER COLUMN metric_type TYPE text,
ALTER COLUMN view_type TYPE text,
ADD COLUMN IF NOT EXISTS o_metric_id INTEGER,
ADD COLUMN IF NOT EXISTS o_widget_id INTEGER;
-- 1. pre transform structure
ALTER TABLE IF EXISTS metrics
ALTER COLUMN metric_type TYPE text,
ALTER COLUMN metric_type SET DEFAULT 'timeseries',
ALTER COLUMN view_type TYPE text,
ALTER COLUMN view_type SET DEFAULT 'lineChart',
ADD COLUMN IF NOT EXISTS o_metric_id INTEGER,
ADD COLUMN IF NOT EXISTS o_widget_id INTEGER;
-- 2. insert predefined metrics related to dashboards as custom metrics
INSERT INTO metrics_clone(project_id, user_id, name, metric_type, view_type, metric_of, metric_value, metric_format,
default_config, o_metric_id, o_widget_id)
SELECT dashboards.project_id,
dashboard_widgets.user_id,
metrics_clone.name,
left(category, 1) || right(replace(initcap(category), ' ', ''), -1) AS metric_type,
'chart' AS view_type,
left(predefined_key, 1) || right(replace(initcap(predefined_key), '_', ''), -1) AS metric_of,
metric_value,
metric_format,
default_config,
metrics_clone.metric_id,
dashboard_widgets.widget_id
FROM metrics_clone
INNER JOIN dashboard_widgets USING (metric_id)
INNER JOIN dashboards USING (dashboard_id)
WHERE is_predefined;
-- 2. insert predefined metrics related to dashboards as custom metrics
INSERT INTO metrics(project_id, user_id, name, metric_type, view_type, metric_of, metric_value,
metric_format,
default_config, o_metric_id, o_widget_id)
SELECT dashboards.project_id,
dashboard_widgets.user_id,
metrics.name,
left(category, 1) || right(replace(initcap(category), ' ', ''), -1) AS metric_type,
'chart' AS view_type,
left(predefined_key, 1) || right(replace(initcap(predefined_key), '_', ''), -1) AS metric_of,
metric_value,
metric_format,
default_config,
metrics.metric_id,
dashboard_widgets.widget_id
FROM metrics
INNER JOIN dashboard_widgets USING (metric_id)
INNER JOIN dashboards USING (dashboard_id)
WHERE is_predefined;
-- 3. update widgets
UPDATE dashboard_widgets
SET metric_id=metrics_clone.metric_id
FROM metrics_clone
WHERE metrics_clone.o_widget_id IS NOT NULL
AND dashboard_widgets.widget_id = metrics_clone.o_widget_id;
-- 3. update widgets
UPDATE dashboard_widgets
SET metric_id=metrics.metric_id
FROM metrics
WHERE metrics.o_widget_id IS NOT NULL
AND dashboard_widgets.widget_id = metrics.o_widget_id;
-- 4. delete predefined metrics
DELETE
FROM metrics_clone
WHERE is_predefined;
-- 4. delete predefined metrics
DELETE
FROM metrics
WHERE is_predefined;
ALTER TABLE IF EXISTS metrics_clone
DROP COLUMN IF EXISTS active,
DROP COLUMN IF EXISTS is_predefined,
DROP COLUMN IF EXISTS is_template,
DROP COLUMN IF EXISTS category,
DROP COLUMN IF EXISTS o_metric_id,
DROP COLUMN IF EXISTS o_widget_id,
DROP CONSTRAINT IF EXISTS null_project_id_for_template_only,
DROP CONSTRAINT IF EXISTS metrics_clone_unique_key;
ALTER TABLE IF EXISTS metrics
DROP COLUMN IF EXISTS active,
DROP COLUMN IF EXISTS is_predefined,
DROP COLUMN IF EXISTS is_template,
DROP COLUMN IF EXISTS category,
DROP COLUMN IF EXISTS o_metric_id,
DROP COLUMN IF EXISTS o_widget_id,
DROP CONSTRAINT IF EXISTS null_project_id_for_template_only,
DROP CONSTRAINT IF EXISTS metrics_unique_key;
END IF;
END;
$$
LANGUAGE plpgsql;
DROP TYPE IF EXISTS metric_type;
DROP TYPE IF EXISTS metric_view_type;
COMMIT;
COMMIT;
CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector);

37
ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql
View file

@ -736,30 +736,28 @@ $$
CREATE INDEX IF NOT EXISTS traces_created_at_idx ON traces (created_at);
CREATE INDEX IF NOT EXISTS traces_action_idx ON traces (action);
CREATE TYPE metric_type AS ENUM ('timeseries','table', 'predefined','funnel');
CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart','barChart','stackedBarChart','stackedBarLineChart','overview','map');
CREATE TABLE IF NOT EXISTS metrics
(
metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY,
project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE,
user_id integer REFERENCES users (user_id) ON DELETE SET NULL,
name text NOT NULL,
is_public boolean NOT NULL DEFAULT FALSE,
active boolean NOT NULL DEFAULT TRUE,
created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()),
project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE,
user_id integer REFERENCES users (user_id) ON DELETE SET NULL,
name text NOT NULL,
is_public boolean NOT NULL DEFAULT FALSE,
active boolean NOT NULL DEFAULT TRUE,
created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()),
deleted_at timestamp,
edited_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()),
metric_type metric_type NOT NULL DEFAULT 'timeseries',
view_type metric_view_type NOT NULL DEFAULT 'lineChart',
metric_of text NOT NULL DEFAULT 'sessionCount',
metric_value text[] NOT NULL DEFAULT '{}'::text[],
edited_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()),
metric_type text NOT NULL DEFAULT 'timeseries',
view_type text NOT NULL DEFAULT 'lineChart',
metric_of text NOT NULL DEFAULT 'sessionCount',
metric_value text[] NOT NULL DEFAULT '{}'::text[],
metric_format text,
category text NULL DEFAULT 'custom',
is_pinned boolean NOT NULL DEFAULT FALSE,
is_predefined boolean NOT NULL DEFAULT FALSE,
is_template boolean NOT NULL DEFAULT FALSE,
predefined_key text NULL DEFAULT NULL,
default_config jsonb NOT NULL DEFAULT '{
category text NULL DEFAULT 'custom',
is_pinned boolean NOT NULL DEFAULT FALSE,
is_predefined boolean NOT NULL DEFAULT FALSE,
is_template boolean NOT NULL DEFAULT FALSE,
predefined_key text NULL DEFAULT NULL,
default_config jsonb NOT NULL DEFAULT '{
"col": 2,
"row": 2,
"position": 0
@ -986,6 +984,7 @@ $$
CREATE INDEX IF NOT EXISTS clicks_url_gin_idx ON events.clicks USING GIN (url gin_trgm_ops);
CREATE INDEX IF NOT EXISTS clicks_url_session_id_timestamp_selector_idx ON events.clicks (url, session_id, timestamp, selector);
CREATE INDEX IF NOT EXISTS clicks_session_id_timestamp_idx ON events.clicks (session_id, timestamp);
CREATE INDEX IF NOT EXISTS clicks_selector_idx ON events.clicks (selector);
CREATE TABLE IF NOT EXISTS events.inputs

124
scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql
View file

@ -5,75 +5,75 @@ $$
SELECT 'v1.9.5'
$$ LANGUAGE sql IMMUTABLE;
DELETE
FROM metrics
WHERE is_predefined
AND is_template;
ALTER TYPE webhook_type ADD VALUE IF NOT EXISTS 'msteams';
UPDATE metrics_clone
SET metric_of=CASE
WHEN metric_of = 'USEROS' THEN 'userOS'
WHEN metric_of = 'USERBROWSER' THEN 'userBrowser'
WHEN metric_of = 'USERDEVICE' THEN 'userDevice'
WHEN metric_of = 'USERCOUNTRY' THEN 'userCountry'
WHEN metric_of = 'USERID' THEN 'userId'
WHEN metric_of = 'ISSUE' THEN 'issue'
WHEN metric_of = 'LOCATION' THEN 'location'
WHEN metric_of = 'SESSIONS' THEN 'sessions'
WHEN metric_of = 'js_exception' THEN 'jsException'
WHEN metric_of = 'sessionCount' THEN 'sessionCount'
END
WHERE NOT is_predefined;
DO
$$
BEGIN
IF EXISTS(SELECT column_name
FROM information_schema.columns
WHERE table_name = 'metrics'
and column_name = 'is_predefined') THEN
-- 1. pre transform structure
ALTER TABLE IF EXISTS metrics_clone
ALTER COLUMN metric_type TYPE text,
ALTER COLUMN view_type TYPE text,
ADD COLUMN IF NOT EXISTS o_metric_id INTEGER,
ADD COLUMN IF NOT EXISTS o_widget_id INTEGER;
-- 1. pre transform structure
ALTER TABLE IF EXISTS metrics
ALTER COLUMN metric_type TYPE text,
ALTER COLUMN metric_type SET DEFAULT 'timeseries',
ALTER COLUMN view_type TYPE text,
ALTER COLUMN view_type SET DEFAULT 'lineChart',
ADD COLUMN IF NOT EXISTS o_metric_id INTEGER,
ADD COLUMN IF NOT EXISTS o_widget_id INTEGER;
-- 2. insert predefined metrics related to dashboards as custom metrics
INSERT INTO metrics_clone(project_id, user_id, name, metric_type, view_type, metric_of, metric_value, metric_format,
default_config, o_metric_id, o_widget_id)
SELECT dashboards.project_id,
dashboard_widgets.user_id,
metrics_clone.name,
left(category, 1) || right(replace(initcap(category), ' ', ''), -1) AS metric_type,
'chart' AS view_type,
left(predefined_key, 1) || right(replace(initcap(predefined_key), '_', ''), -1) AS metric_of,
metric_value,
metric_format,
default_config,
metrics_clone.metric_id,
dashboard_widgets.widget_id
FROM metrics_clone
INNER JOIN dashboard_widgets USING (metric_id)
INNER JOIN dashboards USING (dashboard_id)
WHERE is_predefined;
-- 2. insert predefined metrics related to dashboards as custom metrics
INSERT INTO metrics(project_id, user_id, name, metric_type, view_type, metric_of, metric_value,
metric_format,
default_config, o_metric_id, o_widget_id)
SELECT dashboards.project_id,
dashboard_widgets.user_id,
metrics.name,
left(category, 1) || right(replace(initcap(category), ' ', ''), -1) AS metric_type,
'chart' AS view_type,
left(predefined_key, 1) || right(replace(initcap(predefined_key), '_', ''), -1) AS metric_of,
metric_value,
metric_format,
default_config,
metrics.metric_id,
dashboard_widgets.widget_id
FROM metrics
INNER JOIN dashboard_widgets USING (metric_id)
INNER JOIN dashboards USING (dashboard_id)
WHERE is_predefined;
-- 3. update widgets
UPDATE dashboard_widgets
SET metric_id=metrics_clone.metric_id
FROM metrics_clone
WHERE metrics_clone.o_widget_id IS NOT NULL
AND dashboard_widgets.widget_id = metrics_clone.o_widget_id;
-- 3. update widgets
UPDATE dashboard_widgets
SET metric_id=metrics.metric_id
FROM metrics
WHERE metrics.o_widget_id IS NOT NULL
AND dashboard_widgets.widget_id = metrics.o_widget_id;
-- 4. delete predefined metrics
DELETE
FROM metrics_clone
WHERE is_predefined;
-- 4. delete predefined metrics
DELETE
FROM metrics
WHERE is_predefined;
ALTER TABLE IF EXISTS metrics_clone
DROP COLUMN IF EXISTS active,
DROP COLUMN IF EXISTS is_predefined,
DROP COLUMN IF EXISTS is_template,
DROP COLUMN IF EXISTS category,
DROP COLUMN IF EXISTS o_metric_id,
DROP COLUMN IF EXISTS o_widget_id,
DROP CONSTRAINT IF EXISTS null_project_id_for_template_only,
DROP CONSTRAINT IF EXISTS metrics_clone_unique_key;
ALTER TABLE IF EXISTS metrics
DROP COLUMN IF EXISTS active,
DROP COLUMN IF EXISTS is_predefined,
DROP COLUMN IF EXISTS is_template,
DROP COLUMN IF EXISTS category,
DROP COLUMN IF EXISTS o_metric_id,
DROP COLUMN IF EXISTS o_widget_id,
DROP CONSTRAINT IF EXISTS null_project_id_for_template_only,
DROP CONSTRAINT IF EXISTS metrics_unique_key;
END IF;
END;
$$
LANGUAGE plpgsql;
DROP TYPE IF EXISTS metric_type;
DROP TYPE IF EXISTS metric_view_type;
COMMIT;
COMMIT;
CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector);

37
scripts/schema/db/init_dbs/postgresql/init_schema.sql
View file

@ -674,6 +674,7 @@ $$
CREATE INDEX clicks_url_gin_idx ON events.clicks USING GIN (url gin_trgm_ops);
CREATE INDEX clicks_url_session_id_timestamp_selector_idx ON events.clicks (url, session_id, timestamp, selector);
CREATE INDEX clicks_session_id_timestamp_idx ON events.clicks (session_id, timestamp);
CREATE INDEX clicks_selector_idx ON events.clicks (selector);
CREATE TABLE events.inputs
@ -873,30 +874,28 @@ $$
CREATE INDEX jobs_start_at_idx ON jobs (start_at);
CREATE INDEX jobs_project_id_idx ON jobs (project_id);
CREATE TYPE metric_type AS ENUM ('timeseries','table', 'predefined', 'funnel');
CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart','barChart','stackedBarChart','stackedBarLineChart','overview','map');
CREATE TABLE metrics
(
metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY,
project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE,
user_id integer REFERENCES users (user_id) ON DELETE SET NULL,
name text NOT NULL,
is_public boolean NOT NULL DEFAULT FALSE,
active boolean NOT NULL DEFAULT TRUE,
created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()),
project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE,
user_id integer REFERENCES users (user_id) ON DELETE SET NULL,
name text NOT NULL,
is_public boolean NOT NULL DEFAULT FALSE,
active boolean NOT NULL DEFAULT TRUE,
created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()),
deleted_at timestamp,
edited_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()),
metric_type metric_type NOT NULL DEFAULT 'timeseries',
view_type metric_view_type NOT NULL DEFAULT 'lineChart',
metric_of text NOT NULL DEFAULT 'sessionCount',
metric_value text[] NOT NULL DEFAULT '{}'::text[],
edited_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()),
metric_type text NOT NULL DEFAULT 'timeseries',
view_type text NOT NULL DEFAULT 'lineChart',
metric_of text NOT NULL DEFAULT 'sessionCount',
metric_value text[] NOT NULL DEFAULT '{}'::text[],
metric_format text,
category text NULL DEFAULT 'custom',
is_pinned boolean NOT NULL DEFAULT FALSE,
is_predefined boolean NOT NULL DEFAULT FALSE,
is_template boolean NOT NULL DEFAULT FALSE,
predefined_key text NULL DEFAULT NULL,
default_config jsonb NOT NULL DEFAULT '{
category text NULL DEFAULT 'custom',
is_pinned boolean NOT NULL DEFAULT FALSE,
is_predefined boolean NOT NULL DEFAULT FALSE,
is_template boolean NOT NULL DEFAULT FALSE,
predefined_key text NULL DEFAULT NULL,
default_config jsonb NOT NULL DEFAULT '{
"col": 2,
"row": 2,
"position": 0