diff --git a/api/chalicelib/core/heatmaps.py b/api/chalicelib/core/heatmaps.py index fea3fb407..3ade5911b 100644 --- a/api/chalicelib/core/heatmaps.py +++ b/api/chalicelib/core/heatmaps.py @@ -1,26 +1,62 @@ +import schemas +from chalicelib.core import sessions from chalicelib.utils import helper, pg_client -from chalicelib.utils.TimeUTC import TimeUTC -def get_by_url(project_id, data): - args = {"startDate": data.get('startDate', TimeUTC.now(delta_days=-30)), - "endDate": data.get('endDate', TimeUTC.now()), - "project_id": project_id, "url": data["url"]} +def get_by_url(project_id, data: schemas.GetHeatmapPayloadSchema): + args = {"startDate": data.startDate, "endDate": data.endDate, + "project_id": project_id, "url": data.url} + constraints = ["sessions.project_id = %(project_id)s", + "(url = %(url)s OR path= %(url)s)", + "clicks.timestamp >= %(startDate)s", + "clicks.timestamp <= %(endDate)s", + "start_ts >= %(startDate)s", + "start_ts <= %(endDate)s", + "duration IS NOT NULL"] + query_from = "events.clicks INNER JOIN sessions USING (session_id)" + q_count = "count(1) AS count" + if len(data.filters) > 0: + for i, f in enumerate(data.filters): + if f.type == schemas.FilterType.issue and len(f.value) > 0: + q_count = "max(real_count) AS count" + query_from += """INNER JOIN events_common.issues USING (timestamp, session_id) + INNER JOIN issues AS mis USING (issue_id) + INNER JOIN LATERAL ( + SELECT COUNT(1) AS real_count + FROM events.clicks AS sc + INNER JOIN sessions as ss USING (session_id) + WHERE ss.project_id = 2 + AND (sc.url = %(url)s OR sc.path = %(url)s) + AND sc.timestamp >= %(startDate)s + AND sc.timestamp <= %(endDate)s + AND ss.start_ts >= %(startDate)s + AND ss.start_ts <= %(endDate)s + AND sc.selector = clicks.selector) AS r_clicks ON (TRUE)""" + constraints += ["mis.project_id = %(project_id)s", + "issues.timestamp >= %(startDate)s", + "issues.timestamp <= %(endDate)s"] + f_k = f"issue_value{i}" + args = {**args, **sessions._multiple_values(f.value, value_key=f_k)} + constraints.append(sessions._multiple_conditions(f"%({f_k})s = ANY (issue_types)", + f.value, value_key=f_k)) + constraints.append(sessions._multiple_conditions(f"mis.type = %({f_k})s", + f.value, value_key=f_k)) + if len(f.filters) > 0: + for j, sf in enumerate(f.filters): + f_k = f"issue_svalue{i}{j}" + args = {**args, **sessions._multiple_values(sf.value, value_key=f_k)} + if sf.type == schemas.IssueFilterType._on_selector and len(sf.value) > 0: + constraints.append(sessions._multiple_conditions(f"clicks.selector = %({f_k})s", + sf.value, value_key=f_k)) with pg_client.PostgresClient() as cur: - query = cur.mogrify("""SELECT selector, count(1) AS count - FROM events.clicks - INNER JOIN sessions USING (session_id) - WHERE project_id = %(project_id)s - AND url = %(url)s - AND timestamp >= %(startDate)s - AND timestamp <= %(endDate)s - AND start_ts >= %(startDate)s - AND start_ts <= %(endDate)s - AND duration IS NOT NULL - GROUP BY selector;""", - args) - + query = cur.mogrify(f"""SELECT selector, {q_count} + FROM {query_from} + WHERE {" AND ".join(constraints)} + GROUP BY selector;""", args) + # print("---------") + # print(query.decode('UTF-8')) + # print("---------") try: cur.execute(query) except Exception as err: diff --git a/api/routers/core_dynamic.py b/api/routers/core_dynamic.py index 46e0118ad..290d98183 100644 --- a/api/routers/core_dynamic.py +++ b/api/routers/core_dynamic.py @@ -296,7 +296,7 @@ def get_live_session_devtools_file(projectId: int, sessionId: Union[int, str], @app.post('/{projectId}/heatmaps/url', tags=["heatmaps"]) def get_heatmaps_by_url(projectId: int, data: schemas.GetHeatmapPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return {"data": heatmaps.get_by_url(project_id=projectId, data=data.dict())} + return {"data": heatmaps.get_by_url(project_id=projectId, data=data)} @app.get('/{projectId}/sessions/{sessionId}/favorite', tags=["sessions"]) diff --git a/api/schemas.py b/api/schemas.py index bdd9b98f2..79be0ea97 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -176,12 +176,6 @@ class WeeklyReportConfigSchema(BaseModel): alias_generator = attribute_to_camel_case -class GetHeatmapPayloadSchema(BaseModel): - startDate: int = Field(TimeUTC.now(delta_days=-30)) - endDate: int = Field(TimeUTC.now()) - url: str = Field(...) - - class DatadogSchema(BaseModel): apiKey: str = Field(...) applicationKey: str = Field(...) @@ -1219,3 +1213,29 @@ class FlatClickMapSessionsSearch(SessionsSearchPayloadSchema): values["events"] = n_events values["filters"] = n_filters return values + + +class IssueFilterType(str, Enum): + _on_selector = ClickEventExtraOperator._on_selector.value + + +class IssueAdvancedFilter(BaseModel): + type: IssueFilterType = Field(default=IssueFilterType._on_selector) + value: List[str] = Field(default=[]) + operator: SearchEventOperator = Field(default=SearchEventOperator._is) + + +class ClickMapFilterSchema(BaseModel): + value: List[Literal[IssueType.click_rage, IssueType.dead_click]] = Field(default=[]) + type: Literal[FilterType.issue] = Field(...) + operator: Literal[SearchEventOperator._is, MathOperator._equal] = Field(...) + # source: Optional[Union[ErrorSource, str]] = Field(default=None) + filters: List[IssueAdvancedFilter] = Field(default=[]) + + +class GetHeatmapPayloadSchema(BaseModel): + startDate: int = Field(TimeUTC.now(delta_days=-30)) + endDate: int = Field(TimeUTC.now()) + url: str = Field(...) + # issues: List[Literal[IssueType.click_rage, IssueType.dead_click]] = Field(default=[]) + filters: List[ClickMapFilterSchema] = Field(default=[]) diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql index b0922cb96..a53ca073d 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql @@ -88,6 +88,11 @@ LANGUAGE plpgsql; DROP TYPE IF EXISTS metric_type; DROP TYPE IF EXISTS metric_view_type; +ALTER TABLE IF EXISTS events.clicks + ADD COLUMN IF NOT EXISTS path text; + COMMIT; -CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector); \ No newline at end of file +CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector); +CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_path_idx ON events.clicks (path); +CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_path_gin_idx ON events.clicks USING GIN (path gin_trgm_ops); \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index 6a917d5c5..c4ac5980f 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -968,6 +968,7 @@ $$ timestamp bigint NOT NULL, label text DEFAULT NULL, url text DEFAULT '' NOT NULL, + path text, selector text DEFAULT '' NOT NULL, PRIMARY KEY (session_id, message_id) ); @@ -981,6 +982,8 @@ $$ CREATE INDEX IF NOT EXISTS clicks_url_session_id_timestamp_selector_idx ON events.clicks (url, session_id, timestamp, selector); CREATE INDEX IF NOT EXISTS clicks_session_id_timestamp_idx ON events.clicks (session_id, timestamp); CREATE INDEX IF NOT EXISTS clicks_selector_idx ON events.clicks (selector); + CREATE INDEX IF NOT EXISTS clicks_path_idx ON events.clicks (path); + CREATE INDEX IF NOT EXISTS clicks_path_gin_idx ON events.clicks USING GIN (path gin_trgm_ops); CREATE TABLE IF NOT EXISTS events.inputs diff --git a/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql b/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql index 7cc285249..b56a8836c 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql @@ -75,6 +75,11 @@ LANGUAGE plpgsql; DROP TYPE IF EXISTS metric_type; DROP TYPE IF EXISTS metric_view_type; +ALTER TABLE IF EXISTS events.clicks + ADD COLUMN IF NOT EXISTS path text; + COMMIT; -CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector); \ No newline at end of file +CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector); +CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_path_idx ON events.clicks (path); +CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_path_gin_idx ON events.clicks USING GIN (path gin_trgm_ops); diff --git a/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/scripts/schema/db/init_dbs/postgresql/init_schema.sql index 5c3454516..9cb00ef05 100644 --- a/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -662,6 +662,7 @@ $$ timestamp bigint NOT NULL, label text DEFAULT NULL, url text DEFAULT '' NOT NULL, + path text, selector text DEFAULT '' NOT NULL, PRIMARY KEY (session_id, message_id) ); @@ -675,7 +676,8 @@ $$ CREATE INDEX clicks_url_session_id_timestamp_selector_idx ON events.clicks (url, session_id, timestamp, selector); CREATE INDEX clicks_session_id_timestamp_idx ON events.clicks (session_id, timestamp); CREATE INDEX clicks_selector_idx ON events.clicks (selector); - + CREATE INDEX clicks_path_idx ON events.clicks (path); + CREATE INDEX clicks_path_gin_idx ON events.clicks USING GIN (path gin_trgm_ops); CREATE TABLE events.inputs (