From b09292a580846cae42f96e02baa128c0b6bf70f2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 3 Mar 2022 15:15:12 +0100 Subject: [PATCH 1/6] feat(api): requests indexes feat(api): graphql indexes --- .../db/init_dbs/postgresql/1.5.X/1.5.X.sql | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql b/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql index ce3eafd2a..2b71f3ac4 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql @@ -16,7 +16,7 @@ ALTER TABLE events_common.requests ADD COLUMN IF NOT EXISTS query_string text NULL, ADD COLUMN IF NOT EXISTS request_body text NULL, ADD COLUMN IF NOT EXISTS response_body text NULL, - ADD COLUMN IF NOT EXISTS status_code text NULL, + ADD COLUMN IF NOT EXISTS status_code smallint NULL, ADD COLUMN IF NOT EXISTS method http_method NULL; -- Split requests-URL: Takes too long to use @@ -53,8 +53,26 @@ ALTER TABLE events_common.requests ALTER TABLE events.graphql ADD COLUMN IF NOT EXISTS request_body text NULL, ADD COLUMN IF NOT EXISTS response_body text NULL, - ADD COLUMN IF NOT EXISTS status_code text NULL, + ADD COLUMN IF NOT EXISTS status_code smallint NULL, ADD COLUMN IF NOT EXISTS method http_method NULL, ADD COLUMN IF NOT EXISTS duration integer NULL; -COMMIT; \ No newline at end of file +COMMIT; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_idx ON events_common.requests (query_string) WHERE query_string IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_string_nn_gin_idx ON events_common.requests USING GIN (query_string gin_trgm_ops) WHERE query_string IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_idx ON events_common.requests (request_body) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_idx ON events.graphql (request_body) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_request_body_nn_gin_idx ON events.graphql USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_idx ON events.graphql (response_body) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_response_body_nn_gin_idx ON events.graphql USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS graphql_status_code_nn_idx ON events.graphql (status_code) WHERE status_code IS NOT NULL; \ No newline at end of file From 61d6e017089e8eb0e9bd398a9d8a011e3865b0b0 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 3 Mar 2022 16:30:00 +0100 Subject: [PATCH 2/6] feat(api): reverted countProgress formula --- api/chalicelib/core/custom_metrics.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index d4e0b6b45..785324591 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -22,9 +22,9 @@ def __try_live(project_id, data: schemas.CreateCustomMetricsSchema): r["previousCount"] = sessions.search2_series(data=s.filter, project_id=project_id, density=data.density, view_type=data.view_type, metric_type=data.metric_type, metric_of=data.metric_of, metric_value=data.metric_value) - # r["countProgress"] = helper.__progress(old_val=r["previousCount"], new_val=r["count"]) - r["countProgress"] = ((r["count"] - r["previousCount"]) / r["previousCount"]) * 100 \ - if r["previousCount"] > 0 else 0 + r["countProgress"] = helper.__progress(old_val=r["previousCount"], new_val=r["count"]) + # r["countProgress"] = ((r["count"] - r["previousCount"]) / r["previousCount"]) * 100 \ + # if r["previousCount"] > 0 else 0 r["seriesName"] = s.name if s.name else i + 1 r["seriesId"] = s.series_id if s.series_id else None results[-1] = r @@ -63,8 +63,7 @@ def get_sessions(project_id, user_id, metric_id, data: schemas.CustomMetricSessi metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) if metric is None: return None - metric: schemas.CreateCustomMetricsSchema = schemas.CreateCustomMetricsSchema.parse_obj( - {**data.dict(), **metric}) + metric: schemas.CreateCustomMetricsSchema = schemas.CreateCustomMetricsSchema.parse_obj({**data.dict(), **metric}) results = [] for s in metric.series: s.filter.startDate = data.startDate From 2fef9044e10176f87d57b2ebaa75ebad06a2b605 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 3 Mar 2022 17:06:47 +0100 Subject: [PATCH 3/6] feat(api): drill down custom_metrics by merged filters --- api/chalicelib/core/custom_metrics.py | 24 +++++++++++++++++++++--- api/schemas.py | 10 +++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 785324591..05d729dbe 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -1,4 +1,5 @@ import json +from typing import Union import schemas from chalicelib.core import sessions @@ -43,11 +44,28 @@ def merged_live(project_id, data: schemas.CreateCustomMetricsSchema): return results -def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema): +def __get_merged_metric(project_id, user_id, metric_id, + data: Union[schemas.CustomMetricChartPayloadSchema, + schemas.CustomMetricSessionsPayloadSchema]) \ + -> Union[schemas.CreateCustomMetricsSchema, None]: metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) if metric is None: return None metric: schemas.CreateCustomMetricsSchema = schemas.CreateCustomMetricsSchema.parse_obj({**data.dict(), **metric}) + if len(data.filters) > 0 or len(data.events) > 0: + for s in metric.series: + if len(data.filters) > 0: + s.filter.filters += data.filters + if len(data.events) > 0: + s.filter.events += data.events + return metric + + +def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema): + metric: schemas.CreateCustomMetricsSchema = __get_merged_metric(project_id=project_id, user_id=user_id, + metric_id=metric_id, data=data) + if metric is None: + return None series_charts = __try_live(project_id=project_id, data=metric) if metric.view_type == schemas.MetricTimeseriesViewType.progress: return series_charts @@ -60,10 +78,10 @@ def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPa def get_sessions(project_id, user_id, metric_id, data: schemas.CustomMetricSessionsPayloadSchema): - metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) + metric: schemas.CreateCustomMetricsSchema = __get_merged_metric(project_id=project_id, user_id=user_id, + metric_id=metric_id, data=data) if metric is None: return None - metric: schemas.CreateCustomMetricsSchema = schemas.CreateCustomMetricsSchema.parse_obj({**data.dict(), **metric}) results = [] for s in metric.series: s.filter.startDate = data.startDate diff --git a/api/schemas.py b/api/schemas.py index 1b0ab9ac5..e4dc7f1e7 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -428,7 +428,7 @@ class FilterType(str, Enum): duration = "DURATION" platform = "PLATFORM" metadata = "METADATA" - issue = "ISSUE" + issue = "ISSUES" events_count = "EVENTS_COUNT" utm_source = "UTM_SOURCE" utm_medium = "UTM_MEDIUM" @@ -619,7 +619,7 @@ class SessionsSearchPayloadSchema(BaseModel): alias_generator = attribute_to_camel_case -class FlatSessionsSearchPayloadSchema(SessionsSearchPayloadSchema): +class FlatSessionsSearch(BaseModel): events: Optional[List[_SessionSearchEventSchema]] = Field([]) filters: List[Union[SessionSearchFilterSchema, _SessionSearchEventSchema]] = Field([]) @@ -645,6 +645,10 @@ class FlatSessionsSearchPayloadSchema(SessionsSearchPayloadSchema): return values +class FlatSessionsSearchPayloadSchema(FlatSessionsSearch, SessionsSearchPayloadSchema): + pass + + class SessionsSearchCountSchema(FlatSessionsSearchPayloadSchema): # class SessionsSearchCountSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(default=None) @@ -765,7 +769,7 @@ class TimeseriesMetricOfType(str, Enum): session_count = "sessionCount" -class CustomMetricSessionsPayloadSchema(BaseModel): +class CustomMetricSessionsPayloadSchema(FlatSessionsSearch): startDate: int = Field(TimeUTC.now(-7)) endDate: int = Field(TimeUTC.now()) From 62af49af315767dcf1b884a11d6a27ea669167f0 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 3 Mar 2022 17:41:29 +0100 Subject: [PATCH 4/6] feat(api): changes --- api/chalicelib/core/sessions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index dc6bf85c7..bb5e9b974 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -225,9 +225,9 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # ORDER BY favorite DESC, issue_score DESC, {sort} {order};""", # full_args) - print("--------------------") - print(main_query) - print("--------------------") + # print("--------------------") + # print(main_query) + # print("--------------------") cur.execute(main_query) if count_only: From bd9403c47a5cbeed899744d6a8b453587d8f07fe Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 3 Mar 2022 19:00:52 +0100 Subject: [PATCH 5/6] feat(api): fixed metricType:table for chart --- api/chalicelib/core/custom_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 05d729dbe..7d46cc316 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -67,7 +67,7 @@ def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPa if metric is None: return None series_charts = __try_live(project_id=project_id, data=metric) - if metric.view_type == schemas.MetricTimeseriesViewType.progress: + if metric.view_type == schemas.MetricTimeseriesViewType.progress or metric.metric_type == schemas.MetricType.table: return series_charts results = [{}] * len(series_charts[0]) for i in range(len(results)): From 57874c06101509c78ded4973f18a9b96f394a89c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 3 Mar 2022 19:01:13 +0100 Subject: [PATCH 6/6] feat(api): format core --- api/routers/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/routers/core.py b/api/routers/core.py index 9cf9dfd0c..076ca81f4 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -1091,8 +1091,7 @@ def change_client_password(data: schemas.EditUserPasswordSchema = Body(...), @app.put('/{projectId}/custom_metrics/try', tags=["customMetrics"]) def try_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.merged_live - (project_id=projectId, data=data)} + return {"data": custom_metrics.merged_live(project_id=projectId, data=data)} @app.post('/{projectId}/custom_metrics', tags=["customMetrics"])