Dev (#2714)
* fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * feat(chalice): autocomplete return top 10 with stats * fix(chalice): fixed autocomplete top 10 meta-filters * refactor(chalice): cleaned metrics
This commit is contained in:
Kraiem Taha Yassine
GitHub
parent
73a95503c0
commit
bffcec11fd
2 changed files with 0 additions and 518 deletions
|
|
@ -331,57 +331,6 @@ def __get_page_metrics(cur, project_id, startTimestamp, endTimestamp, **args):
|
|||
return rows
|
||||
|
||||
|
||||
def get_application_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
||||
endTimestamp=TimeUTC.now(), **args):
|
||||
with pg_client.PostgresClient() as cur:
|
||||
row = __get_application_activity(cur, project_id, startTimestamp, endTimestamp, **args)
|
||||
results = helper.dict_to_camel_case(row)
|
||||
diff = endTimestamp - startTimestamp
|
||||
endTimestamp = startTimestamp
|
||||
startTimestamp = endTimestamp - diff
|
||||
row = __get_application_activity(cur, project_id, startTimestamp, endTimestamp, **args)
|
||||
previous = helper.dict_to_camel_case(row)
|
||||
for key in previous.keys():
|
||||
results[key + "Progress"] = helper.__progress(old_val=previous[key], new_val=results[key])
|
||||
return results
|
||||
|
||||
|
||||
def __get_application_activity(cur, project_id, startTimestamp, endTimestamp, **args):
|
||||
result = {}
|
||||
pg_sub_query = __get_constraints(project_id=project_id, data=args)
|
||||
pg_sub_query.append("pages.timestamp >= %(startTimestamp)s")
|
||||
pg_sub_query.append("pages.timestamp > %(endTimestamp)s")
|
||||
pg_sub_query.append("pages.load_time > 0")
|
||||
pg_sub_query.append("pages.load_time IS NOT NULL")
|
||||
pg_query = f"""SELECT COALESCE(AVG(pages.load_time) ,0) AS avg_page_load_time
|
||||
FROM events.pages INNER JOIN public.sessions USING (session_id)
|
||||
WHERE {" AND ".join(pg_sub_query)};"""
|
||||
params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp,
|
||||
**__get_constraint_values(args)}
|
||||
|
||||
cur.execute(cur.mogrify(pg_query, params))
|
||||
row = cur.fetchone()
|
||||
result = {**result, **row}
|
||||
pg_sub_query = __get_constraints(project_id=project_id, data=args)
|
||||
pg_sub_query.append("resources.duration > 0")
|
||||
pg_sub_query.append("resources.type= %(type)s")
|
||||
pg_query = f"""SELECT COALESCE(AVG(resources.duration),0) AS avg
|
||||
FROM events.resources INNER JOIN public.sessions USING (session_id)
|
||||
WHERE {" AND ".join(pg_sub_query)};"""
|
||||
|
||||
cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp,
|
||||
"endTimestamp": endTimestamp, **__get_constraint_values(args)}))
|
||||
row = cur.fetchone()
|
||||
result = {**result, "avg_image_load_time": row["avg"]}
|
||||
cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp,
|
||||
"endTimestamp": endTimestamp, **__get_constraint_values(args)}))
|
||||
|
||||
row = cur.fetchone()
|
||||
result = {**result, "avg_request_load_time": row["avg"]}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_user_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
||||
endTimestamp=TimeUTC.now(), **args):
|
||||
with pg_client.PostgresClient() as cur:
|
||||
|
|
@ -413,181 +362,6 @@ def __get_user_activity(cur, project_id, startTimestamp, endTimestamp, **args):
|
|||
return row
|
||||
|
||||
|
||||
def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
||||
endTimestamp=TimeUTC.now(),
|
||||
density=7, **args):
|
||||
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1)
|
||||
pg_sub_query = __get_constraints(project_id=project_id, data=args)
|
||||
pg_sub_query.append("resources.type = 'img'")
|
||||
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True,
|
||||
chart=True, data=args)
|
||||
pg_sub_query_chart.append("resources.type = 'img'")
|
||||
pg_sub_query_chart.append("resources.url_hostpath = top_img.url_hostpath")
|
||||
|
||||
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
|
||||
chart=False, data=args)
|
||||
pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s")
|
||||
pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s")
|
||||
pg_sub_query_subset.append("resources.duration >0")
|
||||
pg_sub_query_subset.append("resources.duration IS NOT NULL")
|
||||
pg_sub_query_subset.append("resources.type='img'")
|
||||
|
||||
with pg_client.PostgresClient() as cur:
|
||||
pg_query = f"""SELECT *
|
||||
FROM (SELECT resources.url_hostpath,
|
||||
COALESCE(AVG(resources.duration), 0) AS avg_duration,
|
||||
COUNT(resources.session_id) AS sessions_count
|
||||
FROM events.resources
|
||||
INNER JOIN sessions USING (session_id)
|
||||
WHERE {" AND ".join(pg_sub_query_subset)}
|
||||
GROUP BY resources.url_hostpath
|
||||
ORDER BY avg_duration DESC
|
||||
LIMIT 10) AS top_img
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT jsonb_agg(chart) AS chart
|
||||
FROM (SELECT generated_timestamp AS timestamp,
|
||||
COALESCE(AVG(duration), 0) AS avg_duration
|
||||
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
|
||||
LEFT JOIN LATERAL ( SELECT resources.duration
|
||||
FROM events.resources INNER JOIN public.sessions USING (session_id)
|
||||
WHERE {" AND ".join(pg_sub_query_chart)}
|
||||
) AS sessions ON (TRUE)
|
||||
GROUP BY generated_timestamp
|
||||
ORDER BY generated_timestamp) AS chart
|
||||
) AS chart ON (TRUE);"""
|
||||
|
||||
cur.execute(
|
||||
cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
|
||||
"endTimestamp": endTimestamp, **__get_constraint_values(args)}))
|
||||
rows = cur.fetchall()
|
||||
for i in range(len(rows)):
|
||||
rows[i]["sessions"] = rows[i].pop("sessions_count")
|
||||
rows[i] = helper.dict_to_camel_case(rows[i])
|
||||
|
||||
return sorted(rows, key=lambda k: k["sessions"], reverse=True)
|
||||
|
||||
|
||||
def __get_performance_constraint(l):
|
||||
if len(l) == 0:
|
||||
return ""
|
||||
l = [s.decode('UTF-8').replace("%", "%%") for s in l]
|
||||
return f"AND ({' OR '.join(l)})"
|
||||
|
||||
|
||||
def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),
|
||||
density=19, resources=None, **args):
|
||||
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1)
|
||||
location_constraints = []
|
||||
img_constraints = []
|
||||
request_constraints = []
|
||||
|
||||
img_constraints_vals = {}
|
||||
location_constraints_vals = {}
|
||||
request_constraints_vals = {}
|
||||
|
||||
if resources and len(resources) > 0:
|
||||
for r in resources:
|
||||
if r["type"] == "IMG":
|
||||
img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s")
|
||||
img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value']
|
||||
elif r["type"] == "LOCATION":
|
||||
location_constraints.append(f"pages.path = %(val_{len(location_constraints)})s")
|
||||
location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value']
|
||||
else:
|
||||
request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s")
|
||||
request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value']
|
||||
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
|
||||
"endTimestamp": endTimestamp}
|
||||
with pg_client.PostgresClient() as cur:
|
||||
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
|
||||
chart=False, data=args)
|
||||
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False,
|
||||
chart=True, data=args, main_table="resources", time_column="timestamp",
|
||||
duration=False)
|
||||
pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s")
|
||||
pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s")
|
||||
|
||||
pg_query = f"""WITH resources AS (SELECT resources.duration, resources.timestamp
|
||||
FROM events.resources INNER JOIN public.sessions USING (session_id)
|
||||
WHERE {" AND ".join(pg_sub_query_subset)}
|
||||
AND resources.type = 'img' AND resources.duration>0
|
||||
{(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""}
|
||||
)
|
||||
SELECT
|
||||
generated_timestamp AS timestamp,
|
||||
COALESCE(AVG(resources.duration),0) AS avg_image_load_time
|
||||
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT resources.duration
|
||||
FROM resources
|
||||
WHERE {" AND ".join(pg_sub_query_chart)}
|
||||
) AS resources ON (TRUE)
|
||||
GROUP BY timestamp
|
||||
ORDER BY timestamp;"""
|
||||
cur.execute(cur.mogrify(pg_query, {**params, **img_constraints_vals, **__get_constraint_values(args)}))
|
||||
rows = cur.fetchall()
|
||||
images = helper.list_to_camel_case(rows)
|
||||
|
||||
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
|
||||
chart=False, data=args)
|
||||
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False,
|
||||
chart=True, data=args, main_table="resources", time_column="timestamp",
|
||||
duration=False)
|
||||
pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s")
|
||||
pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s")
|
||||
|
||||
pg_query = f"""WITH resources AS(SELECT resources.duration, resources.timestamp
|
||||
FROM events.resources INNER JOIN public.sessions USING (session_id)
|
||||
WHERE {" AND ".join(pg_sub_query_subset)}
|
||||
AND resources.type = 'fetch' AND resources.duration>0
|
||||
{(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""}
|
||||
)
|
||||
SELECT
|
||||
generated_timestamp AS timestamp,
|
||||
COALESCE(AVG(resources.duration),0) AS avg_request_load_time
|
||||
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT resources.duration
|
||||
FROM resources
|
||||
WHERE {" AND ".join(pg_sub_query_chart)}
|
||||
) AS resources ON (TRUE)
|
||||
GROUP BY generated_timestamp
|
||||
ORDER BY generated_timestamp;"""
|
||||
cur.execute(cur.mogrify(pg_query, {**params, **request_constraints_vals, **__get_constraint_values(args)}))
|
||||
rows = cur.fetchall()
|
||||
requests = helper.list_to_camel_case(rows)
|
||||
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
|
||||
chart=False, data=args)
|
||||
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False,
|
||||
chart=True, data=args, main_table="pages", time_column="timestamp",
|
||||
duration=False)
|
||||
pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s")
|
||||
pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s")
|
||||
pg_query = f"""WITH pages AS(SELECT pages.load_time, timestamp
|
||||
FROM events.pages INNER JOIN public.sessions USING (session_id)
|
||||
WHERE {" AND ".join(pg_sub_query_subset)} AND pages.load_time>0 AND pages.load_time IS NOT NULL
|
||||
{(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""}
|
||||
)
|
||||
SELECT
|
||||
generated_timestamp AS timestamp,
|
||||
COALESCE(AVG(pages.load_time),0) AS avg_page_load_time
|
||||
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
|
||||
LEFT JOIN LATERAL ( SELECT pages.load_time
|
||||
FROM pages
|
||||
WHERE {" AND ".join(pg_sub_query_chart)}
|
||||
{(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""}
|
||||
) AS pages ON (TRUE)
|
||||
GROUP BY generated_timestamp
|
||||
ORDER BY generated_timestamp;"""
|
||||
cur.execute(cur.mogrify(pg_query, {**params, **location_constraints_vals, **__get_constraint_values(args)}))
|
||||
rows = cur.fetchall()
|
||||
pages = helper.list_to_camel_case(rows)
|
||||
|
||||
rows = helper.merge_lists_by_key(helper.merge_lists_by_key(pages, requests, "timestamp"), images, "timestamp")
|
||||
|
||||
return {"chart": rows}
|
||||
|
||||
|
||||
RESOURCS_TYPE_TO_DB_TYPE = {
|
||||
"img": "IMG",
|
||||
"fetch": "REQUEST",
|
||||
|
|
@ -815,53 +589,6 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day
|
|||
return rows
|
||||
|
||||
|
||||
def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
||||
endTimestamp=TimeUTC.now(),
|
||||
density=7, **args):
|
||||
step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1)
|
||||
pg_sub_query_subset = __get_constraints(project_id=project_id, data=args)
|
||||
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False,
|
||||
chart=True, data=args, main_table="resources", time_column="timestamp",
|
||||
project=False, duration=False)
|
||||
pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s")
|
||||
pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s")
|
||||
|
||||
with pg_client.PostgresClient() as cur:
|
||||
pg_query = f"""WITH resources AS (SELECT resources.session_id,
|
||||
resources.url_hostpath,
|
||||
timestamp
|
||||
FROM events.resources
|
||||
INNER JOIN public.sessions USING (session_id)
|
||||
WHERE {" AND ".join(pg_sub_query_subset)}
|
||||
)
|
||||
SELECT generated_timestamp AS timestamp,
|
||||
resources.url_hostpath,
|
||||
COUNT(resources.session_id) AS doc_count
|
||||
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
|
||||
LEFT JOIN LATERAL ( SELECT resources.session_id,
|
||||
resources.url_hostpath
|
||||
FROM resources
|
||||
WHERE {" AND ".join(pg_sub_query_chart)}
|
||||
) AS resources ON (TRUE)
|
||||
GROUP BY generated_timestamp, resources.url_hostpath
|
||||
ORDER BY generated_timestamp;"""
|
||||
cur.execute(cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id,
|
||||
"startTimestamp": startTimestamp,
|
||||
"endTimestamp": endTimestamp, **__get_constraint_values(args)}))
|
||||
r = cur.fetchall()
|
||||
results = []
|
||||
|
||||
i = 0
|
||||
while i < len(r):
|
||||
results.append({"timestamp": r[i]["timestamp"], "domains": []})
|
||||
i += 1
|
||||
while i < len(r) and r[i]["timestamp"] == results[-1]["timestamp"]:
|
||||
results[-1]["domains"].append({r[i]["url_hostpath"]: r[i]["doc_count"]})
|
||||
i += 1
|
||||
|
||||
return {"startTimestamp": startTimestamp, "endTimestamp": endTimestamp, "chart": results}
|
||||
|
||||
|
||||
KEYS = {
|
||||
'startTimestamp': args_transformer.int_arg,
|
||||
'endTimestamp': args_transformer.int_arg,
|
||||
|
|
|
|||
|
|
@ -362,59 +362,6 @@ def __get_page_metrics(ch, project_id, startTimestamp, endTimestamp, **args):
|
|||
return rows
|
||||
|
||||
|
||||
def get_application_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
||||
endTimestamp=TimeUTC.now(), **args):
|
||||
with ch_client.ClickHouseClient() as ch:
|
||||
row = __get_application_activity(ch, project_id, startTimestamp, endTimestamp, **args)
|
||||
results = helper.dict_to_camel_case(row)
|
||||
diff = endTimestamp - startTimestamp
|
||||
endTimestamp = startTimestamp
|
||||
startTimestamp = endTimestamp - diff
|
||||
row = __get_application_activity(ch, project_id, startTimestamp, endTimestamp, **args)
|
||||
previous = helper.dict_to_camel_case(row)
|
||||
for key in previous.keys():
|
||||
results[key + "Progress"] = helper.__progress(old_val=previous[key], new_val=results[key])
|
||||
return results
|
||||
|
||||
|
||||
def __get_application_activity(ch, project_id, startTimestamp, endTimestamp, **args):
|
||||
result = {}
|
||||
ch_sub_query = __get_basic_constraints(table_name="pages", data=args)
|
||||
ch_sub_query.append("pages.event_type='LOCATION'")
|
||||
meta_condition = __get_meta_constraint(args)
|
||||
ch_sub_query += meta_condition
|
||||
|
||||
ch_query = f"""SELECT COALESCE(avgOrNull(pages.load_event_end),0) AS avg_page_load_time
|
||||
FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages
|
||||
WHERE {" AND ".join(ch_sub_query)} AND pages.load_event_end>0;"""
|
||||
params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp,
|
||||
**__get_constraint_values(args)}
|
||||
row = ch.execute(query=ch_query, params=params)[0]
|
||||
result = {**result, **row}
|
||||
|
||||
ch_sub_query = __get_basic_constraints(table_name="resources", data=args)
|
||||
# ch_sub_query.append("events.event_type='RESOURCE'")
|
||||
meta_condition = __get_meta_constraint(args)
|
||||
ch_sub_query += meta_condition
|
||||
ch_sub_query.append("resources.type= %(type)s")
|
||||
ch_query = f"""SELECT COALESCE(avgOrNull(resources.duration),0) AS avg
|
||||
FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources
|
||||
WHERE {" AND ".join(ch_sub_query)} AND resources.duration>0;"""
|
||||
row = ch.execute(query=ch_query,
|
||||
params={"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp,
|
||||
"endTimestamp": endTimestamp, **__get_constraint_values(args)})[0]
|
||||
result = {**result, "avg_image_load_time": row["avg"]}
|
||||
row = ch.execute(query=ch_query,
|
||||
params={"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp,
|
||||
"endTimestamp": endTimestamp, **__get_constraint_values(args)})[0]
|
||||
result = {**result, "avg_request_load_time": row["avg"]}
|
||||
|
||||
for k in result:
|
||||
if result[k] is None:
|
||||
result[k] = 0
|
||||
return result
|
||||
|
||||
|
||||
def get_user_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
||||
endTimestamp=TimeUTC.now(), **args):
|
||||
results = {}
|
||||
|
|
@ -455,163 +402,6 @@ def __get_user_activity(ch, project_id, startTimestamp, endTimestamp, **args):
|
|||
return rows
|
||||
|
||||
|
||||
def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
||||
endTimestamp=TimeUTC.now(),
|
||||
density=7, **args):
|
||||
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density)
|
||||
ch_sub_query = __get_basic_constraints(table_name="resources", data=args)
|
||||
# ch_sub_query.append("events.event_type='RESOURCE'")
|
||||
ch_sub_query.append("resources.type = 'img'")
|
||||
ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args)
|
||||
# ch_sub_query_chart.append("events.event_type='RESOURCE'")
|
||||
ch_sub_query_chart.append("resources.type = 'img'")
|
||||
ch_sub_query_chart.append("resources.url_hostpath IN %(url)s")
|
||||
meta_condition = __get_meta_constraint(args)
|
||||
ch_sub_query += meta_condition
|
||||
ch_sub_query_chart += meta_condition
|
||||
|
||||
with ch_client.ClickHouseClient() as ch:
|
||||
ch_query = f"""SELECT resources.url_hostpath AS url,
|
||||
COALESCE(avgOrNull(resources.duration),0) AS avg,
|
||||
COUNT(1) AS count
|
||||
FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources
|
||||
WHERE {" AND ".join(ch_sub_query)} AND resources.duration>0
|
||||
GROUP BY resources.url_hostpath ORDER BY avg DESC LIMIT 10;"""
|
||||
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
|
||||
"endTimestamp": endTimestamp, **__get_constraint_values(args)}
|
||||
rows = ch.execute(query=ch_query, params=params)
|
||||
|
||||
rows = [{"url": i["url"], "avgDuration": i["avg"], "sessions": i["count"]} for i in rows]
|
||||
if len(rows) == 0:
|
||||
return []
|
||||
urls = [row["url"] for row in rows]
|
||||
|
||||
charts = {}
|
||||
ch_query = f"""SELECT url_hostpath AS url,
|
||||
toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
|
||||
COALESCE(avgOrNull(resources.duration),0) AS avg
|
||||
FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources
|
||||
WHERE {" AND ".join(ch_sub_query_chart)} AND resources.duration>0
|
||||
GROUP BY url_hostpath, timestamp
|
||||
ORDER BY url_hostpath, timestamp;"""
|
||||
params["url"] = urls
|
||||
# print(ch.format(query=ch_query, params=params))
|
||||
u_rows = ch.execute(query=ch_query, params=params)
|
||||
for url in urls:
|
||||
sub_rows = []
|
||||
for r in u_rows:
|
||||
if r["url"] == url:
|
||||
sub_rows.append(r)
|
||||
elif len(sub_rows) > 0:
|
||||
break
|
||||
charts[url] = [{"timestamp": int(i["timestamp"]),
|
||||
"avgDuration": i["avg"]}
|
||||
for i in __complete_missing_steps(rows=sub_rows, start_time=startTimestamp,
|
||||
end_time=endTimestamp,
|
||||
density=density, neutral={"avg": 0})]
|
||||
for i in range(len(rows)):
|
||||
rows[i] = helper.dict_to_camel_case(rows[i])
|
||||
rows[i]["chart"] = helper.list_to_camel_case(charts[rows[i]["url"]])
|
||||
|
||||
return sorted(rows, key=lambda k: k["sessions"], reverse=True)
|
||||
|
||||
|
||||
def __get_performance_constraint(l):
|
||||
if len(l) == 0:
|
||||
return ""
|
||||
l = [s.decode('UTF-8').replace("%", "%%") for s in l]
|
||||
return f"AND ({' OR '.join(l)})"
|
||||
|
||||
|
||||
def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),
|
||||
density=19, resources=None, **args):
|
||||
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density)
|
||||
location_constraints = []
|
||||
img_constraints = []
|
||||
request_constraints = []
|
||||
ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args)
|
||||
# ch_sub_query_chart.append("event_type='RESOURCE'")
|
||||
meta_condition = __get_meta_constraint(args)
|
||||
ch_sub_query_chart += meta_condition
|
||||
|
||||
img_constraints_vals = {}
|
||||
location_constraints_vals = {}
|
||||
request_constraints_vals = {}
|
||||
|
||||
if resources and len(resources) > 0:
|
||||
for r in resources:
|
||||
if r["type"] == "IMG":
|
||||
img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s")
|
||||
img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value']
|
||||
elif r["type"] == "LOCATION":
|
||||
location_constraints.append(f"pages.url_path = %(val_{len(location_constraints)})s")
|
||||
location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value']
|
||||
else:
|
||||
request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s")
|
||||
request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value']
|
||||
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
|
||||
"endTimestamp": endTimestamp}
|
||||
with ch_client.ClickHouseClient() as ch:
|
||||
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
|
||||
COALESCE(avgOrNull(resources.duration),0) AS avg
|
||||
FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources
|
||||
WHERE {" AND ".join(ch_sub_query_chart)}
|
||||
AND resources.type = 'img' AND resources.duration>0
|
||||
{(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""}
|
||||
GROUP BY timestamp
|
||||
ORDER BY timestamp;"""
|
||||
rows = ch.execute(query=ch_query, params={**params, **img_constraints_vals, **__get_constraint_values(args)})
|
||||
images = [{"timestamp": i["timestamp"], "avgImageLoadTime": i["avg"]} for i in
|
||||
__complete_missing_steps(rows=rows, start_time=startTimestamp,
|
||||
end_time=endTimestamp,
|
||||
density=density, neutral={"avg": 0})]
|
||||
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
|
||||
COALESCE(avgOrNull(resources.duration),0) AS avg
|
||||
FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources
|
||||
WHERE {" AND ".join(ch_sub_query_chart)}
|
||||
AND resources.type = 'fetch' AND resources.duration>0
|
||||
{(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""}
|
||||
GROUP BY timestamp
|
||||
ORDER BY timestamp;"""
|
||||
rows = ch.execute(query=ch_query,
|
||||
params={**params, **request_constraints_vals, **__get_constraint_values(args)})
|
||||
requests = [{"timestamp": i["timestamp"], "avgRequestLoadTime": i["avg"]} for i in
|
||||
__complete_missing_steps(rows=rows, start_time=startTimestamp,
|
||||
end_time=endTimestamp, density=density,
|
||||
neutral={"avg": 0})]
|
||||
ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args)
|
||||
ch_sub_query_chart.append("pages.event_type='LOCATION'")
|
||||
ch_sub_query_chart += meta_condition
|
||||
|
||||
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
|
||||
COALESCE(avgOrNull(pages.load_event_end),0) AS avg
|
||||
FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages
|
||||
WHERE {" AND ".join(ch_sub_query_chart)} AND pages.load_event_end>0
|
||||
{(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""}
|
||||
GROUP BY timestamp
|
||||
ORDER BY timestamp;"""
|
||||
|
||||
rows = ch.execute(query=ch_query,
|
||||
params={**params, **location_constraints_vals, **__get_constraint_values(args)})
|
||||
pages = [{"timestamp": i["timestamp"], "avgPageLoadTime": i["avg"]} for i in
|
||||
__complete_missing_steps(rows=rows, start_time=startTimestamp,
|
||||
end_time=endTimestamp,
|
||||
density=density, neutral={"avg": 0})]
|
||||
|
||||
rows = helper.merge_lists_by_key(helper.merge_lists_by_key(pages, requests, "timestamp"), images, "timestamp")
|
||||
|
||||
for s in rows:
|
||||
for k in s:
|
||||
if s[k] is None:
|
||||
s[k] = 0
|
||||
return {"chart": helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp,
|
||||
end_time=endTimestamp,
|
||||
density=density,
|
||||
neutral={"avgImageLoadTime": 0,
|
||||
"avgRequestLoadTime": 0,
|
||||
"avgPageLoadTime": 0}))}
|
||||
|
||||
|
||||
RESOURCS_TYPE_TO_DB_TYPE = {
|
||||
"img": "IMG",
|
||||
"fetch": "REQUEST",
|
||||
|
|
@ -833,41 +623,6 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day
|
|||
return rows
|
||||
|
||||
|
||||
def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
|
||||
endTimestamp=TimeUTC.now(),
|
||||
density=7, **args):
|
||||
step_size = __get_step_size(startTimestamp, endTimestamp, density)
|
||||
ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args)
|
||||
# ch_sub_query_chart.append("events.event_type='RESOURCE'")
|
||||
meta_condition = __get_meta_constraint(args)
|
||||
ch_sub_query_chart += meta_condition
|
||||
|
||||
with ch_client.ClickHouseClient() as ch:
|
||||
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
|
||||
resources.url_path, COUNT(1) AS doc_count
|
||||
FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources
|
||||
WHERE {" AND ".join(ch_sub_query_chart)}
|
||||
GROUP BY timestamp, resources.url_path
|
||||
ORDER BY timestamp, doc_count DESC
|
||||
LIMIT 10 BY timestamp;"""
|
||||
params = {"step_size": step_size, "project_id": project_id,
|
||||
"startTimestamp": startTimestamp,
|
||||
"endTimestamp": endTimestamp, **__get_constraint_values(args)}
|
||||
r = ch.execute(query=ch_query, params=params)
|
||||
|
||||
results = []
|
||||
|
||||
i = 0
|
||||
while i < len(r):
|
||||
results.append({"timestamp": r[i]["timestamp"], "domains": []})
|
||||
i += 1
|
||||
while i < len(r) and r[i]["timestamp"] == results[-1]["timestamp"]:
|
||||
results[-1]["domains"].append({r[i]["url_path"]: r[i]["doc_count"]})
|
||||
i += 1
|
||||
|
||||
return {"startTimestamp": startTimestamp, "endTimestamp": endTimestamp, "chart": results}
|
||||
|
||||
|
||||
KEYS = {
|
||||
'startTimestamp': args_transformer.int_arg,
|
||||
'endTimestamp': args_transformer.int_arg,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue