feat(chalice): changed domains errors to use requests only instead of all resources
This commit is contained in:
Taha Yassine Kraiem
parent
c93a7756f2
commit
2393af2290
3 changed files with 39 additions and 38 deletions
|
|
@ -1580,27 +1580,27 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
|||
step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1)
|
||||
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args)
|
||||
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True,
|
||||
data=args, main_table="resources", time_column="timestamp", project=False,
|
||||
data=args, main_table="requests", time_column="timestamp", project=False,
|
||||
duration=False)
|
||||
pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s")
|
||||
pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s")
|
||||
pg_sub_query_subset.append("resources.status/100 = %(status_code)s")
|
||||
pg_sub_query_subset.append("requests.timestamp>=%(startTimestamp)s")
|
||||
pg_sub_query_subset.append("requests.timestamp<%(endTimestamp)s")
|
||||
pg_sub_query_subset.append("requests.status/100 = %(status_code)s")
|
||||
|
||||
with pg_client.PostgresClient() as cur:
|
||||
pg_query = f"""WITH resources AS(SELECT resources.url_host, timestamp
|
||||
FROM events.resources INNER JOIN public.sessions USING (session_id)
|
||||
pg_query = f"""WITH requests AS(SELECT requests.host, timestamp
|
||||
FROM events_common.requests INNER JOIN public.sessions USING (session_id)
|
||||
WHERE {" AND ".join(pg_sub_query_subset)}
|
||||
)
|
||||
SELECT generated_timestamp AS timestamp,
|
||||
COALESCE(JSONB_AGG(resources) FILTER ( WHERE resources IS NOT NULL ), '[]'::JSONB) AS keys
|
||||
COALESCE(JSONB_AGG(requests) FILTER ( WHERE requests IS NOT NULL ), '[]'::JSONB) AS keys
|
||||
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
|
||||
LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.*) AS count
|
||||
FROM resources
|
||||
LEFT JOIN LATERAL ( SELECT requests.host, COUNT(*) AS count
|
||||
FROM requests
|
||||
WHERE {" AND ".join(pg_sub_query_chart)}
|
||||
GROUP BY url_host
|
||||
GROUP BY host
|
||||
ORDER BY count DESC
|
||||
LIMIT 5
|
||||
) AS resources ON (TRUE)
|
||||
) AS requests ON (TRUE)
|
||||
GROUP BY generated_timestamp
|
||||
ORDER BY generated_timestamp;"""
|
||||
params = {"project_id": project_id,
|
||||
|
|
@ -1630,25 +1630,25 @@ def __get_domains_errors_4xx_and_5xx(status, project_id, startTimestamp=TimeUTC.
|
|||
step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1)
|
||||
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args)
|
||||
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True,
|
||||
data=args, main_table="resources", time_column="timestamp", project=False,
|
||||
data=args, main_table="requests", time_column="timestamp", project=False,
|
||||
duration=False)
|
||||
pg_sub_query_subset.append("resources.status/100 = %(status_code)s")
|
||||
pg_sub_query_subset.append("requests.status/100 = %(status_code)s")
|
||||
|
||||
with pg_client.PostgresClient() as cur:
|
||||
pg_query = f"""WITH resources AS (SELECT resources.url_host, timestamp
|
||||
FROM events.resources INNER JOIN public.sessions USING (session_id)
|
||||
pg_query = f"""WITH requests AS (SELECT host, timestamp
|
||||
FROM events_common.requests INNER JOIN public.sessions USING (session_id)
|
||||
WHERE {" AND ".join(pg_sub_query_subset)}
|
||||
)
|
||||
SELECT generated_timestamp AS timestamp,
|
||||
COALESCE(JSONB_AGG(resources) FILTER ( WHERE resources IS NOT NULL ), '[]'::JSONB) AS keys
|
||||
COALESCE(JSONB_AGG(requests) FILTER ( WHERE requests IS NOT NULL ), '[]'::JSONB) AS keys
|
||||
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
|
||||
LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.url_host) AS count
|
||||
FROM resources
|
||||
LEFT JOIN LATERAL ( SELECT requests.host, COUNT(*) AS count
|
||||
FROM requests
|
||||
WHERE {" AND ".join(pg_sub_query_chart)}
|
||||
GROUP BY url_host
|
||||
GROUP BY host
|
||||
ORDER BY count DESC
|
||||
LIMIT 5
|
||||
) AS resources ON (TRUE)
|
||||
) AS requests ON (TRUE)
|
||||
GROUP BY generated_timestamp
|
||||
ORDER BY generated_timestamp;"""
|
||||
params = {"project_id": project_id,
|
||||
|
|
|
|||
|
|
@ -1560,7 +1560,7 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
|||
WHERE {" AND ".join(ch_sub_query)}
|
||||
GROUP BY timestamp,resources.url_host
|
||||
ORDER BY timestamp, count DESC
|
||||
LIMIT 5) AS domain_stats
|
||||
LIMIT 5 BY timestamp) AS domain_stats
|
||||
GROUP BY timestamp;"""
|
||||
params = {"project_id": project_id,
|
||||
"startTimestamp": startTimestamp,
|
||||
|
|
@ -1603,7 +1603,7 @@ def __get_domains_errors_4xx_and_5xx(status, project_id, startTimestamp=TimeUTC.
|
|||
WHERE {" AND ".join(ch_sub_query)}
|
||||
GROUP BY timestamp,resources.url_host
|
||||
ORDER BY timestamp, count DESC
|
||||
LIMIT 5) AS domain_stats
|
||||
LIMIT 5 BY timestamp) AS domain_stats
|
||||
GROUP BY timestamp;"""
|
||||
params = {"project_id": project_id,
|
||||
"startTimestamp": startTimestamp,
|
||||
|
|
|
|||
|
|
@ -1557,29 +1557,30 @@ def __merge_rows_with_neutral(rows, neutral):
|
|||
|
||||
def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
||||
endTimestamp=TimeUTC.now(), density=6, **args):
|
||||
raise Exception("not supported widget")
|
||||
step_size = __get_step_size(startTimestamp, endTimestamp, density)
|
||||
ch_sub_query = __get_basic_constraints(table_name="resources", round_start=True, data=args)
|
||||
ch_sub_query.append("intDiv(resources.status, 100) == %(status_code)s")
|
||||
ch_sub_query = __get_basic_constraints(table_name="requests", round_start=True, data=args)
|
||||
ch_sub_query.append("requests.event_type='REQUEST'")
|
||||
ch_sub_query.append("intDiv(requests.status, 100) == %(status_code)s")
|
||||
meta_condition = __get_meta_constraint(args)
|
||||
ch_sub_query += meta_condition
|
||||
|
||||
with ch_client.ClickHouseClient() as ch:
|
||||
ch_query = f"""SELECT timestamp,
|
||||
groupArray([domain, toString(count)]) AS keys
|
||||
FROM (SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp,
|
||||
resources.url_host AS domain, COUNT(resources.session_id) AS count
|
||||
FROM {sessions_helper.get_main_resources_table(startTimestamp)} AS resources
|
||||
FROM (SELECT toUnixTimestamp(toStartOfInterval(requests.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp,
|
||||
requests.url_host AS domain, COUNT(requests.session_id) AS count
|
||||
FROM {sessions_helper.get_main_events_table(startTimestamp)} AS requests
|
||||
WHERE {" AND ".join(ch_sub_query)}
|
||||
GROUP BY timestamp,resources.url_host
|
||||
GROUP BY timestamp,requests.url_host
|
||||
ORDER BY timestamp, count DESC
|
||||
LIMIT 5) AS domain_stats
|
||||
LIMIT 5 BY timestamp) AS domain_stats
|
||||
GROUP BY timestamp;"""
|
||||
params = {"project_id": project_id,
|
||||
"startTimestamp": startTimestamp,
|
||||
"endTimestamp": endTimestamp,
|
||||
"step_size": step_size,
|
||||
"status_code": 4, **__get_constraint_values(args)}
|
||||
# print(ch.format(query=ch_query, params=params))
|
||||
rows = ch.execute(query=ch_query, params=params)
|
||||
rows = __nested_array_to_dict_array(rows)
|
||||
neutral = __get_domains_errors_neutral(rows)
|
||||
|
|
@ -1601,23 +1602,23 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
|||
|
||||
def __get_domains_errors_4xx_and_5xx(status, project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
||||
endTimestamp=TimeUTC.now(), density=6, **args):
|
||||
raise Exception("not supported widget")
|
||||
step_size = __get_step_size(startTimestamp, endTimestamp, density)
|
||||
ch_sub_query = __get_basic_constraints(table_name="resources", round_start=True, data=args)
|
||||
ch_sub_query.append("intDiv(resources.status, 100) == %(status_code)s")
|
||||
ch_sub_query = __get_basic_constraints(table_name="requests", round_start=True, data=args)
|
||||
ch_sub_query.append("requests.event_type='REQUEST'")
|
||||
ch_sub_query.append("intDiv(requests.status, 100) == %(status_code)s")
|
||||
meta_condition = __get_meta_constraint(args)
|
||||
ch_sub_query += meta_condition
|
||||
|
||||
with ch_client.ClickHouseClient() as ch:
|
||||
ch_query = f"""SELECT timestamp,
|
||||
groupArray([domain, toString(count)]) AS keys
|
||||
FROM (SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp,
|
||||
resources.url_host AS domain, COUNT(resources.session_id) AS count
|
||||
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
|
||||
FROM (SELECT toUnixTimestamp(toStartOfInterval(requests.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp,
|
||||
requests.url_host AS domain, COUNT(requests.session_id) AS count
|
||||
FROM {sessions_helper.get_main_events_table(startTimestamp)} AS requests
|
||||
WHERE {" AND ".join(ch_sub_query)}
|
||||
GROUP BY timestamp,resources.url_host
|
||||
GROUP BY timestamp,requests.url_host
|
||||
ORDER BY timestamp, count DESC
|
||||
LIMIT 5) AS domain_stats
|
||||
LIMIT 5 BY timestamp) AS domain_stats
|
||||
GROUP BY timestamp;"""
|
||||
params = {"project_id": project_id,
|
||||
"startTimestamp": startTimestamp,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue