From c93a7756f2e8b25f079f449ae25e9c5c662486b1 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 23 Aug 2022 18:50:00 +0100 Subject: [PATCH] feat(chalice): refactored domains errors widget --- api/chalicelib/core/metrics.py | 50 ++++++--------------------- ee/api/chalicelib/core/metrics.py | 44 ++++++----------------- ee/api/chalicelib/core/metrics_new.py | 45 ++++++------------------ 3 files changed, 33 insertions(+), 106 deletions(-) diff --git a/api/chalicelib/core/metrics.py b/api/chalicelib/core/metrics.py index c13692a2f..837be1f8c 100644 --- a/api/chalicelib/core/metrics.py +++ b/api/chalicelib/core/metrics.py @@ -1625,8 +1625,8 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return result -def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=6, **args): +def __get_domains_errors_4xx_and_5xx(status, project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, @@ -1655,7 +1655,7 @@ def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, "step_size": step_size, - "status_code": 4, **__get_constraint_values(args)} + "status_code": status, **__get_constraint_values(args)} cur.execute(cur.mogrify(pg_query, params)) rows = cur.fetchall() rows = __nested_array_to_dict_array(rows) @@ -1665,44 +1665,16 @@ def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) return rows +def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=6, **args): + return __get_domains_errors_4xx_and_5xx(status=4, project_id=project_id, startTimestamp=startTimestamp, + endTimestamp=endTimestamp, density=density, **args) + + def get_domains_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, - data=args, main_table="resources", time_column="timestamp", project=False, - duration=False) - pg_sub_query_subset.append("resources.status/100 = %(status_code)s") - - with pg_client.PostgresClient() as cur: - pg_query = f"""WITH resources AS (SELECT resources.url_host, timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - ) - SELECT generated_timestamp AS timestamp, - COALESCE(JSONB_AGG(resources) FILTER ( WHERE resources IS NOT NULL ), '[]'::JSONB) AS keys - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.url_host) AS count - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - GROUP BY url_host - ORDER BY count DESC - LIMIT 5 - ) AS resources ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "step_size": step_size, - "status_code": 5, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - rows = __nested_array_to_dict_array(rows) - neutral = __get_neutral(rows) - rows = __merge_rows_with_neutral(rows, neutral) - - return rows + return __get_domains_errors_4xx_and_5xx(status=5, project_id=project_id, startTimestamp=startTimestamp, + endTimestamp=endTimestamp, density=density, **args) def __nested_array_to_dict_array(rows, key="url_host", value="count"): diff --git a/ee/api/chalicelib/core/metrics.py b/ee/api/chalicelib/core/metrics.py index d0a0f6a42..ee8cd34c9 100644 --- a/ee/api/chalicelib/core/metrics.py +++ b/ee/api/chalicelib/core/metrics.py @@ -1586,8 +1586,8 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return result -def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=6, **args): +def __get_domains_errors_4xx_and_5xx(status, project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query = __get_basic_constraints(table_name="resources", round_start=True, data=args) ch_sub_query.append("intDiv(resources.status, 100) == %(status_code)s") @@ -1609,7 +1609,7 @@ def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, "step_size": step_size, - "status_code": 4, **__get_constraint_values(args)} + "status_code": status, **__get_constraint_values(args)} rows = ch.execute(query=ch_query, params=params) rows = __nested_array_to_dict_array(rows) neutral = __get_domains_errors_neutral(rows) @@ -1620,38 +1620,16 @@ def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) density=density, neutral=neutral) +def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=6, **args): + return __get_domains_errors_4xx_and_5xx(status=4, project_id=project_id, startTimestamp=startTimestamp, + endTimestamp=endTimestamp, density=density, **args) + + def get_domains_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query = __get_basic_constraints(table_name="resources", round_start=True, data=args) - ch_sub_query.append("intDiv(resources.status, 100) == %(status_code)s") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT timestamp, - groupArray([domain, toString(count)]) AS keys - FROM (SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - resources.url_host AS domain, COUNT(resources.session_id) AS count - FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} - WHERE {" AND ".join(ch_sub_query)} - GROUP BY timestamp,resources.url_host - ORDER BY timestamp, count DESC - LIMIT 5) AS domain_stats - GROUP BY timestamp;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "step_size": step_size, - "status_code": 5, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - rows = __nested_array_to_dict_array(rows) - neutral = __get_domains_errors_neutral(rows) - rows = __merge_rows_with_neutral(rows, neutral) - - return __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral=neutral) + return __get_domains_errors_4xx_and_5xx(status=5, project_id=project_id, startTimestamp=startTimestamp, + endTimestamp=endTimestamp, density=density, **args) def __nested_array_to_dict_array(rows): diff --git a/ee/api/chalicelib/core/metrics_new.py b/ee/api/chalicelib/core/metrics_new.py index 6482dff39..c139227f8 100644 --- a/ee/api/chalicelib/core/metrics_new.py +++ b/ee/api/chalicelib/core/metrics_new.py @@ -1599,8 +1599,8 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return result -def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=6, **args): +def __get_domains_errors_4xx_and_5xx(status, project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=6, **args): raise Exception("not supported widget") step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query = __get_basic_constraints(table_name="resources", round_start=True, data=args) @@ -1623,7 +1623,7 @@ def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, "step_size": step_size, - "status_code": 4, **__get_constraint_values(args)} + "status_code": status, **__get_constraint_values(args)} rows = ch.execute(query=ch_query, params=params) rows = __nested_array_to_dict_array(rows) neutral = __get_domains_errors_neutral(rows) @@ -1634,39 +1634,16 @@ def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) density=density, neutral=neutral) +def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=6, **args): + return __get_domains_errors_4xx_and_5xx(status=4, project_id=project_id, startTimestamp=startTimestamp, + endTimestamp=endTimestamp, density=density, **args) + + def get_domains_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query = __get_basic_constraints(table_name="resources", round_start=True, data=args) - ch_sub_query.append("intDiv(resources.status, 100) == %(status_code)s") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT timestamp, - groupArray([domain, toString(count)]) AS keys - FROM (SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - resources.url_host AS domain, COUNT(resources.session_id) AS count - FROM o_resources AS resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} - WHERE {" AND ".join(ch_sub_query)} - GROUP BY timestamp,resources.url_host - ORDER BY timestamp, count DESC - LIMIT 5) AS domain_stats - GROUP BY timestamp;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "step_size": step_size, - "status_code": 5, **__get_constraint_values(args)} - print(ch.execute(query=ch_query, params=params)) - rows = ch.execute(query=ch_query, params=params) - rows = __nested_array_to_dict_array(rows) - neutral = __get_domains_errors_neutral(rows) - rows = __merge_rows_with_neutral(rows, neutral) - - return __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral=neutral) + return __get_domains_errors_4xx_and_5xx(status=5, project_id=project_id, startTimestamp=startTimestamp, + endTimestamp=endTimestamp, density=density, **args) def __nested_array_to_dict_array(rows):