From bffcec11fda303e48e5e48018ea281b09d8d94cc Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Wed, 30 Oct 2024 16:30:33 +0100 Subject: [PATCH] Dev (#2714) * fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * feat(chalice): autocomplete return top 10 with stats * fix(chalice): fixed autocomplete top 10 meta-filters * refactor(chalice): cleaned metrics --- api/chalicelib/core/metrics.py | 273 ------------------------------ ee/api/chalicelib/core/metrics.py | 245 --------------------------- 2 files changed, 518 deletions(-) diff --git a/api/chalicelib/core/metrics.py b/api/chalicelib/core/metrics.py index 573594a8d..c421fff7e 100644 --- a/api/chalicelib/core/metrics.py +++ b/api/chalicelib/core/metrics.py @@ -331,57 +331,6 @@ def __get_page_metrics(cur, project_id, startTimestamp, endTimestamp, **args): return rows -def get_application_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with pg_client.PostgresClient() as cur: - row = __get_application_activity(cur, project_id, startTimestamp, endTimestamp, **args) - results = helper.dict_to_camel_case(row) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - row = __get_application_activity(cur, project_id, startTimestamp, endTimestamp, **args) - previous = helper.dict_to_camel_case(row) - for key in previous.keys(): - results[key + "Progress"] = helper.__progress(old_val=previous[key], new_val=results[key]) - return results - - -def __get_application_activity(cur, project_id, startTimestamp, endTimestamp, **args): - result = {} - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("pages.timestamp >= %(startTimestamp)s") - pg_sub_query.append("pages.timestamp > %(endTimestamp)s") - pg_sub_query.append("pages.load_time > 0") - pg_sub_query.append("pages.load_time IS NOT NULL") - pg_query = f"""SELECT COALESCE(AVG(pages.load_time) ,0) AS avg_page_load_time - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, - **__get_constraint_values(args)} - - cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - result = {**result, **row} - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("resources.duration > 0") - pg_sub_query.append("resources.type= %(type)s") - pg_query = f"""SELECT COALESCE(AVG(resources.duration),0) AS avg - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - row = cur.fetchone() - result = {**result, "avg_image_load_time": row["avg"]} - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - - row = cur.fetchone() - result = {**result, "avg_request_load_time": row["avg"]} - - return result - - def get_user_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: @@ -413,181 +362,6 @@ def __get_user_activity(cur, project_id, startTimestamp, endTimestamp, **args): return row -def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=7, **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("resources.type = 'img'") - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) - pg_sub_query_chart.append("resources.type = 'img'") - pg_sub_query_chart.append("resources.url_hostpath = top_img.url_hostpath") - - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, - chart=False, data=args) - pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") - pg_sub_query_subset.append("resources.duration >0") - pg_sub_query_subset.append("resources.duration IS NOT NULL") - pg_sub_query_subset.append("resources.type='img'") - - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT * - FROM (SELECT resources.url_hostpath, - COALESCE(AVG(resources.duration), 0) AS avg_duration, - COUNT(resources.session_id) AS sessions_count - FROM events.resources - INNER JOIN sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - GROUP BY resources.url_hostpath - ORDER BY avg_duration DESC - LIMIT 10) AS top_img - LEFT JOIN LATERAL ( - SELECT jsonb_agg(chart) AS chart - FROM (SELECT generated_timestamp AS timestamp, - COALESCE(AVG(duration), 0) AS avg_duration - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.duration - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS sessions ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp) AS chart - ) AS chart ON (TRUE);""" - - cur.execute( - cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - rows = cur.fetchall() - for i in range(len(rows)): - rows[i]["sessions"] = rows[i].pop("sessions_count") - rows[i] = helper.dict_to_camel_case(rows[i]) - - return sorted(rows, key=lambda k: k["sessions"], reverse=True) - - -def __get_performance_constraint(l): - if len(l) == 0: - return "" - l = [s.decode('UTF-8').replace("%", "%%") for s in l] - return f"AND ({' OR '.join(l)})" - - -def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), - density=19, resources=None, **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) - location_constraints = [] - img_constraints = [] - request_constraints = [] - - img_constraints_vals = {} - location_constraints_vals = {} - request_constraints_vals = {} - - if resources and len(resources) > 0: - for r in resources: - if r["type"] == "IMG": - img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s") - img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value'] - elif r["type"] == "LOCATION": - location_constraints.append(f"pages.path = %(val_{len(location_constraints)})s") - location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value'] - else: - request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s") - request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - with pg_client.PostgresClient() as cur: - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, - chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, - chart=True, data=args, main_table="resources", time_column="timestamp", - duration=False) - pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") - - pg_query = f"""WITH resources AS (SELECT resources.duration, resources.timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - AND resources.type = 'img' AND resources.duration>0 - {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} - ) - SELECT - generated_timestamp AS timestamp, - COALESCE(AVG(resources.duration),0) AS avg_image_load_time - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT resources.duration - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) - GROUP BY timestamp - ORDER BY timestamp;""" - cur.execute(cur.mogrify(pg_query, {**params, **img_constraints_vals, **__get_constraint_values(args)})) - rows = cur.fetchall() - images = helper.list_to_camel_case(rows) - - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, - chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, - chart=True, data=args, main_table="resources", time_column="timestamp", - duration=False) - pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") - - pg_query = f"""WITH resources AS(SELECT resources.duration, resources.timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - AND resources.type = 'fetch' AND resources.duration>0 - {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} - ) - SELECT - generated_timestamp AS timestamp, - COALESCE(AVG(resources.duration),0) AS avg_request_load_time - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT resources.duration - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {**params, **request_constraints_vals, **__get_constraint_values(args)})) - rows = cur.fetchall() - requests = helper.list_to_camel_case(rows) - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, - chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, - chart=True, data=args, main_table="pages", time_column="timestamp", - duration=False) - pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s") - pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s") - pg_query = f"""WITH pages AS(SELECT pages.load_time, timestamp - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} AND pages.load_time>0 AND pages.load_time IS NOT NULL - {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} - ) - SELECT - generated_timestamp AS timestamp, - COALESCE(AVG(pages.load_time),0) AS avg_page_load_time - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT pages.load_time - FROM pages - WHERE {" AND ".join(pg_sub_query_chart)} - {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {**params, **location_constraints_vals, **__get_constraint_values(args)})) - rows = cur.fetchall() - pages = helper.list_to_camel_case(rows) - - rows = helper.merge_lists_by_key(helper.merge_lists_by_key(pages, requests, "timestamp"), images, "timestamp") - - return {"chart": rows} - - RESOURCS_TYPE_TO_DB_TYPE = { "img": "IMG", "fetch": "REQUEST", @@ -815,53 +589,6 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day return rows -def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, - chart=True, data=args, main_table="resources", time_column="timestamp", - project=False, duration=False) - pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") - - with pg_client.PostgresClient() as cur: - pg_query = f"""WITH resources AS (SELECT resources.session_id, - resources.url_hostpath, - timestamp - FROM events.resources - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - ) - SELECT generated_timestamp AS timestamp, - resources.url_hostpath, - COUNT(resources.session_id) AS doc_count - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.session_id, - resources.url_hostpath - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) - GROUP BY generated_timestamp, resources.url_hostpath - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - r = cur.fetchall() - results = [] - - i = 0 - while i < len(r): - results.append({"timestamp": r[i]["timestamp"], "domains": []}) - i += 1 - while i < len(r) and r[i]["timestamp"] == results[-1]["timestamp"]: - results[-1]["domains"].append({r[i]["url_hostpath"]: r[i]["doc_count"]}) - i += 1 - - return {"startTimestamp": startTimestamp, "endTimestamp": endTimestamp, "chart": results} - - KEYS = { 'startTimestamp': args_transformer.int_arg, 'endTimestamp': args_transformer.int_arg, diff --git a/ee/api/chalicelib/core/metrics.py b/ee/api/chalicelib/core/metrics.py index 2ec25e435..4bb5fe984 100644 --- a/ee/api/chalicelib/core/metrics.py +++ b/ee/api/chalicelib/core/metrics.py @@ -362,59 +362,6 @@ def __get_page_metrics(ch, project_id, startTimestamp, endTimestamp, **args): return rows -def get_application_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with ch_client.ClickHouseClient() as ch: - row = __get_application_activity(ch, project_id, startTimestamp, endTimestamp, **args) - results = helper.dict_to_camel_case(row) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - row = __get_application_activity(ch, project_id, startTimestamp, endTimestamp, **args) - previous = helper.dict_to_camel_case(row) - for key in previous.keys(): - results[key + "Progress"] = helper.__progress(old_val=previous[key], new_val=results[key]) - return results - - -def __get_application_activity(ch, project_id, startTimestamp, endTimestamp, **args): - result = {} - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - ch_query = f"""SELECT COALESCE(avgOrNull(pages.load_event_end),0) AS avg_page_load_time - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)} AND pages.load_event_end>0;""" - params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, - **__get_constraint_values(args)} - row = ch.execute(query=ch_query, params=params)[0] - result = {**result, **row} - - ch_sub_query = __get_basic_constraints(table_name="resources", data=args) - # ch_sub_query.append("events.event_type='RESOURCE'") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - ch_sub_query.append("resources.type= %(type)s") - ch_query = f"""SELECT COALESCE(avgOrNull(resources.duration),0) AS avg - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query)} AND resources.duration>0;""" - row = ch.execute(query=ch_query, - params={"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})[0] - result = {**result, "avg_image_load_time": row["avg"]} - row = ch.execute(query=ch_query, - params={"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})[0] - result = {**result, "avg_request_load_time": row["avg"]} - - for k in result: - if result[k] is None: - result[k] = 0 - return result - - def get_user_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): results = {} @@ -455,163 +402,6 @@ def __get_user_activity(ch, project_id, startTimestamp, endTimestamp, **args): return rows -def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=7, **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) - ch_sub_query = __get_basic_constraints(table_name="resources", data=args) - # ch_sub_query.append("events.event_type='RESOURCE'") - ch_sub_query.append("resources.type = 'img'") - ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - # ch_sub_query_chart.append("events.event_type='RESOURCE'") - ch_sub_query_chart.append("resources.type = 'img'") - ch_sub_query_chart.append("resources.url_hostpath IN %(url)s") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - ch_sub_query_chart += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT resources.url_hostpath AS url, - COALESCE(avgOrNull(resources.duration),0) AS avg, - COUNT(1) AS count - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query)} AND resources.duration>0 - GROUP BY resources.url_hostpath ORDER BY avg DESC LIMIT 10;""" - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - - rows = [{"url": i["url"], "avgDuration": i["avg"], "sessions": i["count"]} for i in rows] - if len(rows) == 0: - return [] - urls = [row["url"] for row in rows] - - charts = {} - ch_query = f"""SELECT url_hostpath AS url, - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(resources.duration),0) AS avg - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query_chart)} AND resources.duration>0 - GROUP BY url_hostpath, timestamp - ORDER BY url_hostpath, timestamp;""" - params["url"] = urls - # print(ch.format(query=ch_query, params=params)) - u_rows = ch.execute(query=ch_query, params=params) - for url in urls: - sub_rows = [] - for r in u_rows: - if r["url"] == url: - sub_rows.append(r) - elif len(sub_rows) > 0: - break - charts[url] = [{"timestamp": int(i["timestamp"]), - "avgDuration": i["avg"]} - for i in __complete_missing_steps(rows=sub_rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"avg": 0})] - for i in range(len(rows)): - rows[i] = helper.dict_to_camel_case(rows[i]) - rows[i]["chart"] = helper.list_to_camel_case(charts[rows[i]["url"]]) - - return sorted(rows, key=lambda k: k["sessions"], reverse=True) - - -def __get_performance_constraint(l): - if len(l) == 0: - return "" - l = [s.decode('UTF-8').replace("%", "%%") for s in l] - return f"AND ({' OR '.join(l)})" - - -def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), - density=19, resources=None, **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) - location_constraints = [] - img_constraints = [] - request_constraints = [] - ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - # ch_sub_query_chart.append("event_type='RESOURCE'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - img_constraints_vals = {} - location_constraints_vals = {} - request_constraints_vals = {} - - if resources and len(resources) > 0: - for r in resources: - if r["type"] == "IMG": - img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s") - img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value'] - elif r["type"] == "LOCATION": - location_constraints.append(f"pages.url_path = %(val_{len(location_constraints)})s") - location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value'] - else: - request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s") - request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(resources.duration),0) AS avg - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query_chart)} - AND resources.type = 'img' AND resources.duration>0 - {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, params={**params, **img_constraints_vals, **__get_constraint_values(args)}) - images = [{"timestamp": i["timestamp"], "avgImageLoadTime": i["avg"]} for i in - __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"avg": 0})] - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(resources.duration),0) AS avg - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query_chart)} - AND resources.type = 'fetch' AND resources.duration>0 - {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, - params={**params, **request_constraints_vals, **__get_constraint_values(args)}) - requests = [{"timestamp": i["timestamp"], "avgRequestLoadTime": i["avg"]} for i in - __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, density=density, - neutral={"avg": 0})] - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.event_type='LOCATION'") - ch_sub_query_chart += meta_condition - - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(pages.load_event_end),0) AS avg - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)} AND pages.load_event_end>0 - {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} - GROUP BY timestamp - ORDER BY timestamp;""" - - rows = ch.execute(query=ch_query, - params={**params, **location_constraints_vals, **__get_constraint_values(args)}) - pages = [{"timestamp": i["timestamp"], "avgPageLoadTime": i["avg"]} for i in - __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"avg": 0})] - - rows = helper.merge_lists_by_key(helper.merge_lists_by_key(pages, requests, "timestamp"), images, "timestamp") - - for s in rows: - for k in s: - if s[k] is None: - s[k] = 0 - return {"chart": helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"avgImageLoadTime": 0, - "avgRequestLoadTime": 0, - "avgPageLoadTime": 0}))} - - RESOURCS_TYPE_TO_DB_TYPE = { "img": "IMG", "fetch": "REQUEST", @@ -833,41 +623,6 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day return rows -def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - # ch_sub_query_chart.append("events.event_type='RESOURCE'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - resources.url_path, COUNT(1) AS doc_count - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp, resources.url_path - ORDER BY timestamp, doc_count DESC - LIMIT 10 BY timestamp;""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - r = ch.execute(query=ch_query, params=params) - - results = [] - - i = 0 - while i < len(r): - results.append({"timestamp": r[i]["timestamp"], "domains": []}) - i += 1 - while i < len(r) and r[i]["timestamp"] == results[-1]["timestamp"]: - results[-1]["domains"].append({r[i]["url_path"]: r[i]["doc_count"]}) - i += 1 - - return {"startTimestamp": startTimestamp, "endTimestamp": endTimestamp, "chart": results} - - KEYS = { 'startTimestamp': args_transformer.int_arg, 'endTimestamp': args_transformer.int_arg,