diff --git a/api/chalicelib/core/metrics.py b/api/chalicelib/core/metrics.py index 837be1f8c..d5fe5db52 100644 --- a/api/chalicelib/core/metrics.py +++ b/api/chalicelib/core/metrics.py @@ -1580,27 +1580,27 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, - data=args, main_table="resources", time_column="timestamp", project=False, + data=args, main_table="requests", time_column="timestamp", project=False, duration=False) - pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") - pg_sub_query_subset.append("resources.status/100 = %(status_code)s") + pg_sub_query_subset.append("requests.timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("requests.timestamp<%(endTimestamp)s") + pg_sub_query_subset.append("requests.status/100 = %(status_code)s") with pg_client.PostgresClient() as cur: - pg_query = f"""WITH resources AS(SELECT resources.url_host, timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) + pg_query = f"""WITH requests AS(SELECT requests.host, timestamp + FROM events_common.requests INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query_subset)} ) SELECT generated_timestamp AS timestamp, - COALESCE(JSONB_AGG(resources) FILTER ( WHERE resources IS NOT NULL ), '[]'::JSONB) AS keys + COALESCE(JSONB_AGG(requests) FILTER ( WHERE requests IS NOT NULL ), '[]'::JSONB) AS keys FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.*) AS count - FROM resources + LEFT JOIN LATERAL ( SELECT requests.host, COUNT(*) AS count + FROM requests WHERE {" AND ".join(pg_sub_query_chart)} - GROUP BY url_host + GROUP BY host ORDER BY count DESC LIMIT 5 - ) AS resources ON (TRUE) + ) AS requests ON (TRUE) GROUP BY generated_timestamp ORDER BY generated_timestamp;""" params = {"project_id": project_id, @@ -1630,25 +1630,25 @@ def __get_domains_errors_4xx_and_5xx(status, project_id, startTimestamp=TimeUTC. step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, - data=args, main_table="resources", time_column="timestamp", project=False, + data=args, main_table="requests", time_column="timestamp", project=False, duration=False) - pg_sub_query_subset.append("resources.status/100 = %(status_code)s") + pg_sub_query_subset.append("requests.status/100 = %(status_code)s") with pg_client.PostgresClient() as cur: - pg_query = f"""WITH resources AS (SELECT resources.url_host, timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) + pg_query = f"""WITH requests AS (SELECT host, timestamp + FROM events_common.requests INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query_subset)} ) SELECT generated_timestamp AS timestamp, - COALESCE(JSONB_AGG(resources) FILTER ( WHERE resources IS NOT NULL ), '[]'::JSONB) AS keys + COALESCE(JSONB_AGG(requests) FILTER ( WHERE requests IS NOT NULL ), '[]'::JSONB) AS keys FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.url_host) AS count - FROM resources + LEFT JOIN LATERAL ( SELECT requests.host, COUNT(*) AS count + FROM requests WHERE {" AND ".join(pg_sub_query_chart)} - GROUP BY url_host + GROUP BY host ORDER BY count DESC LIMIT 5 - ) AS resources ON (TRUE) + ) AS requests ON (TRUE) GROUP BY generated_timestamp ORDER BY generated_timestamp;""" params = {"project_id": project_id, diff --git a/ee/api/chalicelib/core/metrics.py b/ee/api/chalicelib/core/metrics.py index ee8cd34c9..ac8afdfb7 100644 --- a/ee/api/chalicelib/core/metrics.py +++ b/ee/api/chalicelib/core/metrics.py @@ -1560,7 +1560,7 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), WHERE {" AND ".join(ch_sub_query)} GROUP BY timestamp,resources.url_host ORDER BY timestamp, count DESC - LIMIT 5) AS domain_stats + LIMIT 5 BY timestamp) AS domain_stats GROUP BY timestamp;""" params = {"project_id": project_id, "startTimestamp": startTimestamp, @@ -1603,7 +1603,7 @@ def __get_domains_errors_4xx_and_5xx(status, project_id, startTimestamp=TimeUTC. WHERE {" AND ".join(ch_sub_query)} GROUP BY timestamp,resources.url_host ORDER BY timestamp, count DESC - LIMIT 5) AS domain_stats + LIMIT 5 BY timestamp) AS domain_stats GROUP BY timestamp;""" params = {"project_id": project_id, "startTimestamp": startTimestamp, diff --git a/ee/api/chalicelib/core/metrics_new.py b/ee/api/chalicelib/core/metrics_new.py index c139227f8..4086f99b8 100644 --- a/ee/api/chalicelib/core/metrics_new.py +++ b/ee/api/chalicelib/core/metrics_new.py @@ -1557,29 +1557,30 @@ def __merge_rows_with_neutral(rows, neutral): def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): - raise Exception("not supported widget") step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query = __get_basic_constraints(table_name="resources", round_start=True, data=args) - ch_sub_query.append("intDiv(resources.status, 100) == %(status_code)s") + ch_sub_query = __get_basic_constraints(table_name="requests", round_start=True, data=args) + ch_sub_query.append("requests.event_type='REQUEST'") + ch_sub_query.append("intDiv(requests.status, 100) == %(status_code)s") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT timestamp, groupArray([domain, toString(count)]) AS keys - FROM (SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - resources.url_host AS domain, COUNT(resources.session_id) AS count - FROM {sessions_helper.get_main_resources_table(startTimestamp)} AS resources + FROM (SELECT toUnixTimestamp(toStartOfInterval(requests.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, + requests.url_host AS domain, COUNT(requests.session_id) AS count + FROM {sessions_helper.get_main_events_table(startTimestamp)} AS requests WHERE {" AND ".join(ch_sub_query)} - GROUP BY timestamp,resources.url_host + GROUP BY timestamp,requests.url_host ORDER BY timestamp, count DESC - LIMIT 5) AS domain_stats + LIMIT 5 BY timestamp) AS domain_stats GROUP BY timestamp;""" params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, "step_size": step_size, "status_code": 4, **__get_constraint_values(args)} + # print(ch.format(query=ch_query, params=params)) rows = ch.execute(query=ch_query, params=params) rows = __nested_array_to_dict_array(rows) neutral = __get_domains_errors_neutral(rows) @@ -1601,23 +1602,23 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), def __get_domains_errors_4xx_and_5xx(status, project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): - raise Exception("not supported widget") step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query = __get_basic_constraints(table_name="resources", round_start=True, data=args) - ch_sub_query.append("intDiv(resources.status, 100) == %(status_code)s") + ch_sub_query = __get_basic_constraints(table_name="requests", round_start=True, data=args) + ch_sub_query.append("requests.event_type='REQUEST'") + ch_sub_query.append("intDiv(requests.status, 100) == %(status_code)s") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT timestamp, groupArray([domain, toString(count)]) AS keys - FROM (SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - resources.url_host AS domain, COUNT(resources.session_id) AS count - FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + FROM (SELECT toUnixTimestamp(toStartOfInterval(requests.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, + requests.url_host AS domain, COUNT(requests.session_id) AS count + FROM {sessions_helper.get_main_events_table(startTimestamp)} AS requests WHERE {" AND ".join(ch_sub_query)} - GROUP BY timestamp,resources.url_host + GROUP BY timestamp,requests.url_host ORDER BY timestamp, count DESC - LIMIT 5) AS domain_stats + LIMIT 5 BY timestamp) AS domain_stats GROUP BY timestamp;""" params = {"project_id": project_id, "startTimestamp": startTimestamp,