jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(chalice): metrics calls with 4xx&5xx errors on requests instead of resources-fetch

Browse source 
feat(DB): structure optimization for calls with 4xx&5xx
This commit is contained in:
Taha Yassine Kraiem 2022-08-23 17:17:32 +01:00
parent b4e1d1c40a
commit 27ee944af8
4 changed files with 105 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
api/chalicelib/core/metrics.py
View file

@ -1823,7 +1823,7 @@ def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endT
FROM events.resources INNER JOIN sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query)}
GROUP BY resources.method, resources.url_hostpath
ORDER BY (4 + 5), 3 DESC
ORDER BY (4 + 5) DESC, 3 DESC
LIMIT 50;"""
cur.execute(cur.mogrify(pg_query, {"project_id": project_id,
"startTimestamp": startTimestamp,
@ -1832,50 +1832,45 @@ def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endT
return helper.list_to_camel_case(rows)
def get_calls_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),
platform=None, **args):
def __get_calls_errors_4xx_or_5xx(status, project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(),
platform=None, **args):
pg_sub_query = __get_constraints(project_id=project_id, data=args)
pg_sub_query.append("resources.type = 'fetch'")
pg_sub_query.append("resources.method IS NOT NULL")
pg_sub_query.append("resources.status/100 = 4")
pg_sub_query.append("requests.type = 'fetch'")
pg_sub_query.append("requests.method IS NOT NULL")
pg_sub_query.append(f"requests.status/100 = {status}")
with pg_client.PostgresClient() as cur:
pg_query = f"""SELECT resources.method,
resources.url_hostpath,
COUNT(resources.session_id) AS all_requests
FROM events.resources INNER JOIN sessions USING (session_id)
pg_query = f"""SELECT requests.method,
requests.host,
requests.path,
COUNT(requests.session_id) AS all_requests
FROM events_common.requests INNER JOIN sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query)}
GROUP BY resources.method, resources.url_hostpath
GROUP BY requests.method, requests.host, requests.path
ORDER BY all_requests DESC
LIMIT 10;"""
cur.execute(cur.mogrify(pg_query, {"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)}))
rows = cur.fetchall()
for r in rows:
r["url_hostpath"] = r.pop("host") + r.pop("path")
return helper.list_to_camel_case(rows)
def get_calls_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),
platform=None, **args):
return __get_calls_errors_4xx_or_5xx(status=4, project_id=project_id, startTimestamp=startTimestamp,
endTimestamp=endTimestamp,
platform=platform, **args)
def get_calls_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),
platform=None, **args):
pg_sub_query = __get_constraints(project_id=project_id, data=args)
pg_sub_query.append("resources.type = 'fetch'")
pg_sub_query.append("resources.method IS NOT NULL")
pg_sub_query.append("resources.status/100 = 5")
with pg_client.PostgresClient() as cur:
pg_query = f"""SELECT resources.method,
resources.url_hostpath,
COUNT(resources.session_id) AS all_requests
FROM events.resources INNER JOIN sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query)}
GROUP BY resources.method, resources.url_hostpath
ORDER BY all_requests DESC
LIMIT 10;"""
cur.execute(cur.mogrify(pg_query, {"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)}))
rows = cur.fetchall()
return helper.list_to_camel_case(rows)
return __get_calls_errors_4xx_or_5xx(status=5, project_id=project_id, startTimestamp=startTimestamp,
endTimestamp=endTimestamp,
platform=platform, **args)
def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),

73
ee/api/chalicelib/core/metrics.py
View file

@ -1778,7 +1778,7 @@ def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endT
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query)}
GROUP BY resources.method, resources.url_hostpath
ORDER BY (_4xx + _5xx), all_requests DESC
ORDER BY (_4xx + _5xx) DESC, all_requests DESC
LIMIT 50;"""
rows = ch.execute(query=ch_query,
params={"project_id": project_id,
@ -1787,52 +1787,43 @@ def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endT
return helper.list_to_camel_case(rows)
def __get_calls_errors_4xx_or_5xx(status, project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(),
platform=None, **args):
ch_sub_query = __get_basic_constraints(table_name="resources", data=args)
ch_sub_query.append("resources.type = 'fetch'")
ch_sub_query.append(f"intDiv(resources.status, 100) == {status}")
meta_condition = __get_meta_constraint(args)
ch_sub_query += meta_condition
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT resources.method,
resources.url_hostpath,
COUNT(resources.session_id) AS all_requests
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query)}
GROUP BY resources.method, resources.url_hostpath
ORDER BY all_requests DESC
LIMIT 10;"""
rows = ch.execute(query=ch_query,
params={"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)})
return helper.list_to_camel_case(rows)
def get_calls_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),
platform=None, **args):
ch_sub_query = __get_basic_constraints(table_name="resources", data=args)
ch_sub_query.append("resources.type = 'fetch'")
ch_sub_query.append("intDiv(resources.status, 100) == 4")
meta_condition = __get_meta_constraint(args)
ch_sub_query += meta_condition
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT resources.method,
resources.url_hostpath,
COUNT(resources.session_id) AS all_requests
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query)}
GROUP BY resources.method, resources.url_hostpath
ORDER BY all_requests DESC
LIMIT 10;"""
rows = ch.execute(query=ch_query,
params={"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)})
return helper.list_to_camel_case(rows)
return __get_calls_errors_4xx_or_5xx(status=4, project_id=project_id, startTimestamp=startTimestamp,
endTimestamp=endTimestamp,
platform=platform, **args)
def get_calls_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),
platform=None, **args):
ch_sub_query = __get_basic_constraints(table_name="resources", data=args)
ch_sub_query.append("resources.type = 'fetch'")
ch_sub_query.append("intDiv(resources.status, 100) == 5")
meta_condition = __get_meta_constraint(args)
ch_sub_query += meta_condition
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT resources.method,
resources.url_hostpath,
COUNT(resources.session_id) AS all_requests
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query)}
GROUP BY resources.method, resources.url_hostpath
ORDER BY all_requests DESC
LIMIT 10;"""
rows = ch.execute(query=ch_query,
params={"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)})
return helper.list_to_camel_case(rows)
return __get_calls_errors_4xx_or_5xx(status=5, project_id=project_id, startTimestamp=startTimestamp,
endTimestamp=endTimestamp,
platform=platform, **args)
def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),

103
ee/api/chalicelib/core/metrics_new.py
View file

@ -802,11 +802,11 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day
ch_sub_query += meta_condition
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT events.path AS key,
ch_query = f"""SELECT events.url_path AS key,
COUNT(events.session_id) AS doc_count
FROM {sessions_helper.get_main_events_table(startTimestamp)} AS events
WHERE {" AND ".join(ch_sub_query)}
GROUP BY path
GROUP BY url_path
ORDER BY doc_count DESC
LIMIT 10;"""
print(ch.format(query=ch_query, params={"project_id": project_id, "startTimestamp": startTimestamp,
@ -817,7 +817,7 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day
rows = [{"url": i["key"], "sessions": i["doc_count"]} for i in rows]
if len(rows) == 0:
return []
ch_sub_query.append("events.path = %(value)s")
ch_sub_query.append("events.url_path = %(value)s")
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(events.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
COUNT(events.session_id) AS doc_count,
toUnixTimestamp(MAX(events.datetime))*1000 AS max_datatime
@ -1779,23 +1779,22 @@ def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=-
def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),
platform=None, **args):
raise Exception("not supported widget")
ch_sub_query = __get_basic_constraints(table_name="resources", data=args)
ch_sub_query.append("resources.type = 'fetch'")
ch_sub_query.append("intDiv(resources.status, 100) != 2")
ch_sub_query = __get_basic_constraints(table_name="requests", data=args)
ch_sub_query.append("requests.event_type = 'REQUEST'")
ch_sub_query.append("intDiv(requests.status, 100) != 2")
meta_condition = __get_meta_constraint(args)
ch_sub_query += meta_condition
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT resources.method,
resources.path,
COUNT(resources.session_id) AS all_requests,
SUM(if(intDiv(resources.status, 100) == 4, 1, 0)) AS _4xx,
SUM(if(intDiv(resources.status, 100) == 5, 1, 0)) AS _5xx
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
ch_query = f"""SELECT requests.method,
requests.url_hostpath,
COUNT(requests.session_id) AS all_requests,
SUM(if(intDiv(requests.status, 100) == 4, 1, 0)) AS _4xx,
SUM(if(intDiv(requests.status, 100) == 5, 1, 0)) AS _5xx
FROM {sessions_helper.get_main_events_table(startTimestamp)} AS requests
WHERE {" AND ".join(ch_sub_query)}
GROUP BY resources.method, resources.path
ORDER BY (_4xx + _5xx), all_requests DESC
GROUP BY requests.method, requests.url_hostpath
ORDER BY (_4xx + _5xx) DESC, all_requests DESC
LIMIT 50;"""
rows = ch.execute(query=ch_query,
params={"project_id": project_id,
@ -1804,54 +1803,44 @@ def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endT
return helper.list_to_camel_case(rows)
def __get_calls_errors_4xx_or_5xx(status, project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(),
platform=None, **args):
ch_sub_query = __get_basic_constraints(table_name="requests", data=args)
ch_sub_query.append("requests.event_type = 'REQUEST'")
ch_sub_query.append(f"intDiv(requests.status, 100) == {status}")
meta_condition = __get_meta_constraint(args)
ch_sub_query += meta_condition
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT requests.method,
requests.url_hostpath,
COUNT(requests.session_id) AS all_requests
FROM {sessions_helper.get_main_events_table(startTimestamp)} AS requests
WHERE {" AND ".join(ch_sub_query)}
GROUP BY requests.method, requests.url_hostpath
ORDER BY all_requests DESC
LIMIT 10;"""
rows = ch.execute(query=ch_query,
params={"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)})
return helper.list_to_camel_case(rows)
def get_calls_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),
platform=None, **args):
raise Exception("not supported widget")
ch_sub_query = __get_basic_constraints(table_name="resources", data=args)
ch_sub_query.append("resources.type = 'fetch'")
ch_sub_query.append("intDiv(resources.status, 100) == 4")
meta_condition = __get_meta_constraint(args)
ch_sub_query += meta_condition
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT resources.method,
resources.path,
COUNT(resources.session_id) AS all_requests
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query)}
GROUP BY resources.method, resources.path
ORDER BY all_requests DESC
LIMIT 10;"""
rows = ch.execute(query=ch_query,
params={"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)})
return helper.list_to_camel_case(rows)
return __get_calls_errors_4xx_or_5xx(status=4, project_id=project_id, startTimestamp=startTimestamp,
endTimestamp=endTimestamp,
platform=platform, **args)
def get_calls_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),
platform=None, **args):
raise Exception("not supported widget")
ch_sub_query = __get_basic_constraints(table_name="resources", data=args)
ch_sub_query.append("resources.type = 'fetch'")
ch_sub_query.append("intDiv(resources.status, 100) == 5")
meta_condition = __get_meta_constraint(args)
ch_sub_query += meta_condition
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT resources.method,
resources.url_hostpath,
COUNT(resources.session_id) AS all_requests
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query)}
GROUP BY resources.method, resources.url_hostpath
ORDER BY all_requests DESC
LIMIT 10;"""
rows = ch.execute(query=ch_query,
params={"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)})
return helper.list_to_camel_case(rows)
return __get_calls_errors_4xx_or_5xx(status=5, project_id=project_id, startTimestamp=startTimestamp,
endTimestamp=endTimestamp,
platform=platform, **args)
def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),

2
ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql
View file

@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS resources
url String,
url_host String MATERIALIZED lower(domain(url)),
url_path String MATERIALIZED lower(path(url)),
url_hostpath String MATERIALIZED concat(url_host, url_path),
type Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4),
name Nullable(String) MATERIALIZED if(type = 'fetch', null,
coalesce(nullIf(splitByChar('/', url_path)[-1], ''),
@ -70,6 +71,7 @@ CREATE TABLE IF NOT EXISTS events
url Nullable(String),
url_host Nullable(String) MATERIALIZED lower(domain(url)),
url_path Nullable(String) MATERIALIZED lower(pathFull(url)),
url_hostpath Nullable(String) MATERIALIZED concat(url_host, url_path),
request_start Nullable(UInt16),
response_start Nullable(UInt16),
response_end Nullable(UInt16),