diff --git a/api/chalicelib/blueprints/subs/bp_dashboard.py b/api/chalicelib/blueprints/subs/bp_dashboard.py index 51bdce092..00b3c0ed4 100644 --- a/api/chalicelib/blueprints/subs/bp_dashboard.py +++ b/api/chalicelib/blueprints/subs/bp_dashboard.py @@ -130,8 +130,9 @@ def get_network_widget(projectId, context): @app.route('/{projectId}/dashboard/{widget}/search', methods=['GET']) def get_dashboard_autocomplete(projectId, widget, context): params = app.current_request.query_params - if params is None: + if params is None or params.get('q') is None or len(params.get('q')) == 0: return {"data": []} + params['q'] = '^' + params['q'] if widget in ['performance']: data = dashboard.search(params.get('q', ''), params.get('type', ''), project_id=projectId, @@ -547,59 +548,3 @@ def get_dashboard_group(projectId, context): *helper.explode_widget(dashboard.get_avg_cpu(project_id=projectId, **{**data, **args})), *helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **{**data, **args})), ]} - - -@app.route('/{projectId}/dashboard/errors_crashes', methods=['GET', 'POST']) -def get_dashboard_group(projectId, context): - data = app.current_request.json_body - if data is None: - data = {} - params = app.current_request.query_params - args = dashboard.dashboard_args(params) - - return {"data": [ - {"key": "errors", - "data": dashboard.get_errors(project_id=projectId, **{**data, **args})}, - {"key": "errors_trend", - "data": dashboard.get_errors_trend(project_id=projectId, **{**data, **args})}, - {"key": "crashes", - "data": dashboard.get_crashes(project_id=projectId, **{**data, **args})}, - {"key": "domains_errors", - "data": dashboard.get_domains_errors(project_id=projectId, **{**data, **args})}, - {"key": "errors_per_domains", - "data": dashboard.get_errors_per_domains(project_id=projectId, **{**data, **args})}, - {"key": "calls_errors", - "data": dashboard.get_calls_errors(project_id=projectId, **{**data, **args})}, - {"key": "errors_per_type", - "data": dashboard.get_errors_per_type(project_id=projectId, **{**data, **args})}, - {"key": "impacted_sessions_by_js_errors", - "data": dashboard.get_impacted_sessions_by_js_errors(project_id=projectId, **{**data, **args})} - ]} - - -@app.route('/{projectId}/dashboard/resources', methods=['GET', 'POST']) -def get_dashboard_group(projectId, context): - data = app.current_request.json_body - if data is None: - data = {} - params = app.current_request.query_params - args = dashboard.dashboard_args(params) - - return {"data": [ - {"key": "slowest_images", - "data": dashboard.get_slowest_images(project_id=projectId, **{**data, **args})}, - {"key": "missing_resources", - "data": dashboard.get_missing_resources_trend(project_id=projectId, **{**data, **args})}, - {"key": "slowest_resources", - "data": dashboard.get_slowest_resources(project_id=projectId, type='all', **{**data, **args})}, - {"key": "resources_loading_time", - "data": dashboard.get_resources_loading_time(project_id=projectId, **{**data, **args})}, - {"key": "resources_by_party", - "data": dashboard.get_resources_by_party(project_id=projectId, **{**data, **args})}, - {"key": "resource_type_vs_response_end", - "data": dashboard.resource_type_vs_response_end(project_id=projectId, **{**data, **args})}, - {"key": "resources_vs_visually_complete", - "data": dashboard.get_resources_vs_visually_complete(project_id=projectId, **{**data, **args})}, - {"key": "resources_count_by_type", - "data": dashboard.get_resources_count_by_type(project_id=projectId, **{**data, **args})} - ]} diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index deef86e95..919d6aa5a 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -30,16 +30,17 @@ def __get_constraints(project_id, time_constraint=True, chart=False, duration=Tr project_identifier="project_id", main_table="sessions", time_column="start_ts", data={}): pg_sub_query = [] + main_table = main_table + "." if main_table is not None and len(main_table) > 0 else "" if project: - pg_sub_query.append(f"{main_table}.{project_identifier} =%({project_identifier})s") + pg_sub_query.append(f"{main_table}{project_identifier} =%({project_identifier})s") if duration: - pg_sub_query.append(f"{main_table}.duration>0") + pg_sub_query.append(f"{main_table}duration>0") if time_constraint: - pg_sub_query.append(f"{main_table}.{time_column} >= %(startTimestamp)s") - pg_sub_query.append(f"{main_table}.{time_column} < %(endTimestamp)s") + pg_sub_query.append(f"{main_table}{time_column} >= %(startTimestamp)s") + pg_sub_query.append(f"{main_table}{time_column} < %(endTimestamp)s") if chart: - pg_sub_query.append(f"{main_table}.{time_column} >= generated_timestamp") - pg_sub_query.append(f"{main_table}.{time_column} < generated_timestamp + %(step_size)s") + pg_sub_query.append(f"{main_table}{time_column} >= generated_timestamp") + pg_sub_query.append(f"{main_table}{time_column} < generated_timestamp + %(step_size)s") return pg_sub_query + __get_meta_constraint(project_id=project_id, data=data) @@ -178,32 +179,36 @@ def get_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimesta density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("m_errors.source = 'js_exception'") - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) - pg_sub_query_chart.append("m_errors.source = 'js_exception'") - + pg_sub_query_subset = __get_constraints(project_id=project_id, data=args, duration=False, main_table="m_errors", + time_constraint=False) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, + chart=True, data=args, main_table="errors", time_column="timestamp", + project=False, duration=False) + pg_sub_query_subset.append("m_errors.source = 'js_exception'") + pg_sub_query_subset.append("errors.timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("errors.timestamp<%(endTimestamp)s") with pg_client.PostgresClient() as cur: - pg_query = f"""\ - SELECT generated_timestamp AS timestamp, - COALESCE(COUNT(sessions), 0) AS count - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT DISTINCT session_id - FROM events.errors - INNER JOIN public.errors AS m_errors USING(error_id) - INNER JOIN public.sessions USING(session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS sessions ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" + pg_query = f"""WITH errors AS (SELECT DISTINCT session_id, timestamp + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT generated_timestamp AS timestamp, + COALESCE(COUNT(sessions), 0) AS count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT session_id + FROM errors + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} cur.execute(cur.mogrify(pg_query, params)) rows = cur.fetchall() results = { "count": 0 if len(rows) == 0 else __count_distinct_errors(cur, project_id, startTimestamp, endTimestamp, - pg_sub_query), + pg_sub_query_subset), "impactedSessions": sum([r["count"] for r in rows]), "chart": rows } @@ -211,18 +216,18 @@ def get_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimesta diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff - count = __count_distinct_errors(cur, project_id, startTimestamp, endTimestamp, pg_sub_query, **args) + count = __count_distinct_errors(cur, project_id, startTimestamp, endTimestamp, pg_sub_query_subset, **args) results["progress"] = helper.__progress(old_val=count, new_val=results["count"]) return results def __count_distinct_errors(cur, project_id, startTimestamp, endTimestamp, pg_sub_query, **args): - pg_query = f"""\ - SELECT COALESCE(COUNT(DISTINCT errors.error_id),0) AS count - FROM events.errors - INNER JOIN public.errors AS m_errors USING(error_id) - INNER JOIN public.sessions USING(session_id) - WHERE {" AND ".join(pg_sub_query)};""" + pg_query = f"""WITH errors AS (SELECT DISTINCT error_id + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + WHERE {" AND ".join(pg_sub_query)}) + SELECT COALESCE(COUNT(*), 0) AS count + FROM errors;""" cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)})) return cur.fetchone()["count"] @@ -233,40 +238,50 @@ def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("m_errors.project_id = %(project_id)s") - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, duration=False, - project=False, main_table="errors", time_column="timestamp", data=args) - pg_sub_query_chart.append("error_id = errors_details.error_id") + + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=False, + chart=False, data=args, main_table="m_errors", duration=False) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="errors_subsest", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("errors.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("errors.timestamp < %(endTimestamp)s") + + pg_sub_query_chart.append("errors_subsest.error_id = top_errors.error_id") with pg_client.PostgresClient() as cur: - pg_query = f""" - SELECT * - FROM (SELECT errors.error_id AS error_id, - m_errors.message AS error, - COUNT(errors.session_id) AS count, - COUNT(DISTINCT errors.session_id) AS sessions_count - FROM events.errors - INNER JOIN public.errors AS m_errors USING (error_id) - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - GROUP BY errors.error_id, m_errors.message - ORDER BY sessions_count DESC, count DESC - LIMIT 10) AS errors_details - INNER JOIN LATERAL (SELECT MAX(timestamp) AS last_occurrence_at, - MIN(timestamp) AS first_occurrence_at - FROM events.errors - WHERE error_id = errors_details.error_id - GROUP BY error_id) AS errors_time ON (TRUE) - INNER JOIN LATERAL (SELECT jsonb_agg(chart) AS chart - FROM (SELECT generated_timestamp AS timestamp, COALESCE(COUNT(sessions), 0) AS count - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT DISTINCT session_id - FROM events.errors - WHERE {" AND ".join(pg_sub_query_chart)}) AS sessions - ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp) AS chart) AS chart ON (TRUE);""" + pg_query = f"""WITH errors_subsest AS (SELECT session_id, error_id, timestamp + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT * + FROM (SELECT error_id, COUNT(sub_errors) AS count, count(DISTINCT session_id) AS sessions_count + FROM (SELECT error_id, session_id + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + WHERE {" AND ".join(pg_sub_query_subset)}) AS sub_errors + GROUP BY error_id + ORDER BY sessions_count DESC, count DESC + LIMIT 10) AS top_errors + INNER JOIN LATERAL (SELECT message AS error + FROM public.errors + WHERE project_id = %(project_id)s + AND errors.error_id = top_errors.error_id) AS errors_details ON(TRUE) + INNER JOIN LATERAL (SELECT MAX(timestamp) AS last_occurrence_at, + MIN(timestamp) AS first_occurrence_at + FROM events.errors + WHERE error_id = top_errors.error_id + GROUP BY error_id) AS errors_time ON (TRUE) + INNER JOIN LATERAL (SELECT jsonb_agg(chart) AS chart + FROM (SELECT generated_timestamp AS timestamp, COALESCE(COUNT(sessions), 0) AS count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT DISTINCT session_id + FROM errors_subsest + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp) AS chart) AS chart ON (TRUE);""" params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} cur.execute(cur.mogrify(pg_query, params)) @@ -299,22 +314,20 @@ def get_page_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), results[key + "Progress"] = helper.__progress(old_val=previous[key], new_val=results[key]) return results + @dev.timed def __get_page_metrics(cur, project_id, startTimestamp, endTimestamp, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) - - pg_query = f"""\ - SELECT COALESCE((SELECT AVG(pages.dom_content_loaded_time) - FROM events.pages - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - AND pages.dom_content_loaded_time > 0 - ), 0) AS avg_dom_content_load_start, - COALESCE((SELECT AVG(pages.first_contentful_paint_time) - FROM events.pages - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - AND pages.first_contentful_paint_time > 0), 0) AS avg_first_contentful_pixel;""" + pg_sub_query.append("pages.timestamp>=%(startTimestamp)s") + pg_sub_query.append("pages.timestamp<%(endTimestamp)s") + pg_sub_query.append("(pages.dom_content_loaded_time > 0 OR pages.first_contentful_paint_time > 0)") + pg_query = f"""SELECT COALESCE(AVG(NULLIF(pages.dom_content_loaded_time, 0)), 0) AS avg_dom_content_load_start, + COALESCE(AVG(NULLIF(pages.first_contentful_paint_time, 0)), 0) AS avg_first_contentful_pixel + FROM (SELECT pages.dom_content_loaded_time, pages.first_contentful_paint_time + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + ) AS pages;""" params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} cur.execute(cur.mogrify(pg_query, params)) @@ -341,7 +354,10 @@ def get_application_activity(project_id, startTimestamp=TimeUTC.now(delta_days=- def __get_application_activity(cur, project_id, startTimestamp, endTimestamp, **args): result = {} pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("pages.timestamp >= %(startTimestamp)s") + pg_sub_query.append("pages.timestamp > %(endTimestamp)s") pg_sub_query.append("pages.load_time > 0") + pg_sub_query.append("pages.load_time IS NOT NULL") pg_query = f"""\ SELECT COALESCE(AVG(pages.load_time) ,0) AS avg_page_load_time FROM events.pages INNER JOIN public.sessions USING (session_id) @@ -352,8 +368,8 @@ def __get_application_activity(cur, project_id, startTimestamp, endTimestamp, ** cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() result = {**result, **row} - - pg_sub_query[-1] = "resources.duration > 0" + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("resources.duration > 0") pg_sub_query.append("resources.type= %(type)s") pg_query = f"""\ SELECT COALESCE(AVG(resources.duration),0) AS avg @@ -416,42 +432,47 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, data=args) pg_sub_query_chart.append("resources.type = 'img'") - pg_sub_query_chart.append("resources.url = %(url)s") + pg_sub_query_chart.append("resources.url = top_img.url") + + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") + pg_sub_query_subset.append("resources.duration >0") + pg_sub_query_subset.append("resources.duration IS NOT NULL") + pg_sub_query_subset.append("resources.type='img'") with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT resources.url, - COALESCE(AVG(NULLIF(resources.duration,0)),0) AS avg_duration, - COUNT(resources.session_id) AS sessions_count - FROM events.resources INNER JOIN sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - GROUP BY resources.url - ORDER BY avg_duration DESC LIMIT 10;""" + pg_query = f"""SELECT * + FROM (SELECT resources.url, + COALESCE(AVG(resources.duration), 0) AS avg_duration, + COUNT(resources.session_id) AS sessions_count + FROM events.resources + INNER JOIN sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + GROUP BY resources.url + ORDER BY avg_duration DESC + LIMIT 10) AS top_img + LEFT JOIN LATERAL ( + SELECT jsonb_agg(chart) AS chart + FROM (SELECT generated_timestamp AS timestamp, + COALESCE(AVG(duration), 0) AS avg_duration + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT resources.duration + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp) AS chart + ) AS chart ON (TRUE);""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "startTimestamp": startTimestamp, + cur.execute(cur.mogrify(pg_query, {"step_size": step_size,"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)})) rows = cur.fetchall() - urls = [row["url"] for row in rows] - - charts = {} - for url in urls: - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(duration), 0) AS avg_duration - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.duration - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS sessions ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, "url": url, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - r = cur.fetchall() - charts[url] = helper.list_to_camel_case(r) - for i in range(len(rows)): - rows[i]["sessions"] = rows[i].pop("sessions_count") - rows[i] = helper.dict_to_camel_case(rows[i]) - rows[i]["chart"] = charts[rows[i]["url"]] + for i in range(len(rows)): + rows[i]["sessions"] = rows[i].pop("sessions_count") + rows[i] = helper.dict_to_camel_case(rows[i]) return sorted(rows, key=lambda k: k["sessions"], reverse=True) @@ -471,8 +492,6 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi location_constraints = [] img_constraints = [] request_constraints = [] - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) img_constraints_vals = {} location_constraints_vals = {} @@ -492,48 +511,84 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") + + pg_query = f"""WITH resources AS (SELECT resources.duration, resources.timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + AND resources.type = 'img' AND resources.duration>0 + {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} + ) + SELECT generated_timestamp AS timestamp, COALESCE(AVG(resources.duration),0) AS avg_image_load_time FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT resources.duration - FROM events.resources INNER JOIN public.sessions USING (session_id) + FROM resources WHERE {" AND ".join(pg_sub_query_chart)} - AND resources.type = 'img' AND resources.duration>0 - {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} ) AS resources ON (TRUE) GROUP BY timestamp ORDER BY timestamp;""" cur.execute(cur.mogrify(pg_query, {**params, **img_constraints_vals, **__get_constraint_values(args)})) rows = cur.fetchall() images = helper.list_to_camel_case(rows) - pg_query = f"""SELECT + + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") + + pg_query = f"""WITH resources AS(SELECT resources.duration, resources.timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + AND resources.type = 'fetch' AND resources.duration>0 + {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} + ) + SELECT generated_timestamp AS timestamp, COALESCE(AVG(resources.duration),0) AS avg_request_load_time FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT resources.duration - FROM events.resources INNER JOIN public.sessions USING (session_id) + FROM resources WHERE {" AND ".join(pg_sub_query_chart)} - AND resources.type = 'fetch' AND resources.duration>0 - {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} ) AS resources ON (TRUE) GROUP BY generated_timestamp ORDER BY generated_timestamp;""" cur.execute(cur.mogrify(pg_query, {**params, **request_constraints_vals, **__get_constraint_values(args)})) rows = cur.fetchall() requests = helper.list_to_camel_case(rows) - - pg_query = f"""SELECT + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="pages", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s") + pg_query = f"""WITH pages AS(SELECT pages.load_time, timestamp + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} AND pages.load_time>0 AND pages.load_time IS NOT NULL + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + ) + SELECT generated_timestamp AS timestamp, COALESCE(AVG(pages.load_time),0) AS avg_page_load_time FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT pages.load_time - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} AND pages.load_time>0 - {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} - ) AS pages ON (TRUE) + FROM pages + WHERE {" AND ".join(pg_sub_query_chart)} + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + ) AS pages ON (TRUE) GROUP BY generated_timestamp ORDER BY generated_timestamp;""" cur.execute(cur.mogrify(pg_query, {**params, **location_constraints_vals, **__get_constraint_values(args)})) @@ -595,6 +650,7 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} ORDER BY url, type ASC) AS ranked_values WHERE ranked_values.r<=5;""" + print(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text)})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text)})) rows = cur.fetchall() rows = [{"value": i["value"], "type": __get_resource_type_from_db_type(i["key"])} for i in rows] @@ -614,6 +670,9 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, FROM events.pages INNER JOIN public.sessions USING(session_id) WHERE {" AND ".join(pg_sub_query)} AND positionUTF8(url_path, %(value)s) != 0 LIMIT 10);""" + print(cur.mogrify(pg_query, {"project_id": project_id, + "value": helper.string_to_sql_like(text.lower()), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text.lower()), "platform_0": platform})) @@ -630,6 +689,10 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, FROM events.resources INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" + print(cur.mogrify(pg_query, {"project_id": project_id, + "value": helper.string_to_sql_like(text), + "resource_type": resource_type, + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "resource_type": resource_type, @@ -644,6 +707,9 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" + print(cur.mogrify(pg_query, {"project_id": project_id, + "value": helper.string_to_sql_like(text), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "platform_0": platform})) @@ -655,6 +721,9 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, FROM events.inputs INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" + print(cur.mogrify(pg_query, {"project_id": project_id, + "value": helper.string_to_sql_like(text), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "platform_0": platform})) @@ -666,6 +735,9 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, FROM events.clicks INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" + print(cur.mogrify(pg_query, {"project_id": project_id, + "value": helper.string_to_sql_like(text), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "platform_0": platform})) @@ -684,6 +756,9 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, FROM sessions WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" + print(cur.mogrify(pg_query, + {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, "platform_0": platform})) @@ -708,6 +783,10 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, AND sessions.{SESSIONS_META_FIELDS[k]} ILIKE %(value)s LIMIT 10)""") pg_query = " UNION ALL ".join(pg_query) + print(cur.mogrify(pg_query, + {"project_id": project_id, "value": helper.string_to_sql_like(text), + "key": key, + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, @@ -779,19 +858,32 @@ def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) + pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + project=False, duration=False) + pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") + + with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, - resources.url_hostpath, - COUNT(resources.session_id) AS doc_count + pg_query = f"""WITH resources AS (SELECT resources.session_id, + resources.url_hostpath, + timestamp + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT generated_timestamp AS timestamp, + resources.url_hostpath, + COUNT(resources.session_id) AS doc_count FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.session_id, - resources.url_hostpath - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) + LEFT JOIN LATERAL ( SELECT resources.session_id, + resources.url_hostpath + FROM resources + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS resources ON (TRUE) GROUP BY generated_timestamp, resources.url_hostpath ORDER BY generated_timestamp;""" cur.execute(cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id, @@ -834,24 +926,33 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days endTimestamp=TimeUTC.now(), density=19, type=None, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) + pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + project=False, duration=False) + pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") + pg_sub_query_subset.append("resources.duration>0") + pg_sub_query_subset.append("resources.duration IS NOT NULL") + if type is not None: - pg_sub_query.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'") - pg_sub_query_chart.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'") + pg_sub_query_subset.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'") if url is not None: - pg_sub_query.append(f"resources.url = %(value)s") - pg_sub_query_chart.append(f"resources.url = %(value)s") + pg_sub_query_subset.append(f"resources.url = %(value)s") with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(NULLIF(resources.duration,0)),0) AS avg + pg_query = f"""WITH resources AS (SELECT resources.duration, timestamp + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT generated_timestamp AS timestamp, + COALESCE(AVG(resources.duration), 0) AS avg FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.duration - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) + LEFT JOIN LATERAL ( SELECT resources.duration + FROM resources + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS resources ON (TRUE) GROUP BY generated_timestamp ORDER BY generated_timestamp;""" params = {"step_size": step_size, "project_id": project_id, @@ -860,14 +961,12 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days "value": url, "type": type, **__get_constraint_values(args)} cur.execute(cur.mogrify(pg_query, params)) rows = cur.fetchall() - if len(rows) > 0: - pg_query = f"""SELECT COALESCE(AVG(NULLIF(resources.duration,0)),0) AS avg - FROM events.resources INNER JOIN sessions USING(session_id) - WHERE {" AND ".join(pg_sub_query)};""" - cur.execute(cur.mogrify(pg_query, params)) - avg = cur.fetchone()["avg"] - else: - avg = 0 + pg_query = f"""SELECT COALESCE(AVG(resources.duration),0) AS avg + FROM events.resources INNER JOIN sessions USING(session_id) + WHERE {" AND ".join(pg_sub_query_subset)};""" + cur.execute(cur.mogrify(pg_query, params)) + avg = cur.fetchone()["avg"] + return {"avg": avg, "chart": rows} @@ -875,21 +974,26 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) + pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, + chart=True, data=args, main_table="pages", time_column="timestamp", + project=False, duration=False) + if url is not None: - pg_sub_query.append(f"pages.path = %(value)s") - pg_sub_query_chart.append(f"pages.path = %(value)s") - pg_sub_query.append("pages.dom_building_time>0") - pg_sub_query_chart.append("pages.dom_building_time>0") + pg_sub_query_subset.append(f"pages.path = %(value)s") + + pg_sub_query_subset.append("pages.timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("pages.timestamp<%(endTimestamp)s") + pg_sub_query_subset.append("pages.dom_building_time>0") + pg_sub_query_subset.append("pages.dom_building_time IS NOT NULL") with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT COALESCE(avg, 0) AS avg, chart - FROM (SELECT AVG(dom_building_time) - FROM public.sessions - INNER JOIN events.pages USING (session_id) - WHERE {" AND ".join(pg_sub_query)}) AS avg + pg_query = f"""WITH pages AS ( SELECT pages.dom_building_time, timestamp + FROM public.sessions + INNER JOIN events.pages USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)}) + SELECT COALESCE(avg, 0) AS avg, chart + FROM (SELECT AVG(dom_building_time) FROM pages) AS avg LEFT JOIN (SELECT jsonb_agg(chart) AS chart FROM ( @@ -897,8 +1001,7 @@ def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=- COALESCE(AVG(dom_building_time), 0) AS avg FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT pages.dom_building_time - FROM public.sessions - INNER JOIN events.pages USING (session_id) + FROM pages WHERE {" AND ".join(pg_sub_query_chart)} ) AS sessionsBD ON (TRUE) GROUP BY generated_timestamp @@ -918,24 +1021,36 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), type="all", density=19, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) + pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + project=False, duration=False) + pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") + pg_sub_query_subset.append("resources.duration>0") + pg_sub_query_subset.append("resources.duration IS NOT NULL") if type is not None and type.upper() != "ALL": sq = f"resources.type = '{__get_resource_db_type_from_type(type.upper())}'" else: sq = "resources.type != 'fetch'" pg_sub_query.append(sq) - pg_sub_query_chart.append(sq) - pg_sub_query_chart.append("resources.duration IS NOT NULL") - pg_sub_query_chart.append("resources.duration>0") + pg_sub_query_subset.append(sq) + pg_sub_query_chart.append("resources.url_hostpath ILIKE '%%' || main_list.name") + with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT * + pg_query = f"""WITH resources AS ( + SELECT resources.duration, url_hostpath, timestamp + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT * FROM (SELECT regexp_replace(resources.url_hostpath, '^.*/', '') AS name, - AVG(NULLIF(resources.duration, 0)) AS avg + AVG(resources.duration) AS avg FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} + WHERE {" AND ".join(pg_sub_query_subset)} GROUP BY name ORDER BY avg DESC LIMIT 10) AS main_list @@ -956,15 +1071,14 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT resources.duration - FROM events.resources - INNER JOIN public.sessions USING (session_id) + FROM resources WHERE {" AND ".join(pg_sub_query_chart)} - AND resources.url_hostpath ILIKE '%%' || main_list.name ) AS resources ON (TRUE) GROUP BY generated_timestamp ORDER BY generated_timestamp ) AS chart_details ) AS chart_details ON (TRUE);""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, @@ -1210,16 +1324,41 @@ def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), if value is not None: pg_sub_query.append("pages.path = %(value)s") with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT (SELECT COALESCE(AVG(pages.response_time),0) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} AND pages.response_time>0) AS avg_response_time, - (SELECT COUNT(pages.session_id) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)}) AS count_requests, - (SELECT COALESCE(AVG(pages.first_paint_time),0) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} AND pages.first_paint_time>0) AS avg_first_paint, - (SELECT COALESCE(AVG(pages.dom_content_loaded_time),0) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} AND pages.dom_content_loaded_time>0) AS avg_dom_content_loaded, - (SELECT COALESCE(AVG(pages.ttfb),0) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} AND pages.ttfb>0) AS avg_till_first_bit, - (SELECT COALESCE(AVG(pages.time_to_interactive),0) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} AND pages.time_to_interactive >0) AS avg_time_to_interactive;""" - print(cur.mogrify(pg_query, {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)})) + pg_query = f"""WITH pages AS (SELECT pages.response_time, + pages.first_paint_time, + pages.dom_content_loaded_time, + pages.ttfb, + pages.time_to_interactive + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND pages.timestamp >= %(startTimestamp)s + AND pages.timestamp < %(endTimestamp)s + AND (pages.response_time > 0 + OR pages.first_paint_time > 0 + OR pages.dom_content_loaded_time > 0 + OR pages.ttfb > 0 + OR pages.time_to_interactive > 0 + )) + SELECT (SELECT COALESCE(AVG(pages.response_time), 0) + FROM pages + WHERE pages.response_time > 0) AS avg_response_time, + (SELECT COALESCE(AVG(pages.first_paint_time), 0) + FROM pages + WHERE pages.first_paint_time > 0) AS avg_first_paint, + (SELECT COALESCE(AVG(pages.dom_content_loaded_time), 0) + FROM pages + WHERE pages.dom_content_loaded_time > 0) AS avg_dom_content_loaded, + (SELECT COALESCE(AVG(pages.ttfb), 0) + FROM pages + WHERE pages.ttfb > 0) AS avg_till_first_bit, + (SELECT COALESCE(AVG(pages.time_to_interactive), 0) + FROM pages + WHERE pages.time_to_interactive > 0) AS avg_time_to_interactive, + (SELECT COUNT(pages.session_id) + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)}) AS count_requests;""" cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, @@ -1232,31 +1371,30 @@ def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, - data=args) - pg_sub_query.append("pages.visually_complete>0") - pg_sub_query_chart.append("pages.visually_complete>0") + pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, + chart=True, data=args, main_table="pages", time_column="timestamp", + project=False, duration=False) + pg_sub_query_subset.append("pages.visually_complete>0") if url is not None: - pg_sub_query_chart.append("pages.path = %(value)s") + pg_sub_query_subset.append("pages.path = %(value)s") with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT COALESCE(avg,0) AS avg, chart - FROM (SELECT AVG(pages.visually_complete) - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)}) AS avg - LEFT JOIN - (SELECT jsonb_agg(chart) AS chart - FROM (SELECT generated_timestamp AS timestamp, - COALESCE(AVG(visually_complete), 0) AS avg - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + pg_query = f"""WITH pages AS(SELECT pages.visually_complete,pages.timestamp + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)}) + SELECT COALESCE((SELECT AVG(pages.visually_complete) FROM pages),0) AS avg, + jsonb_agg(chart) AS chart + FROM + (SELECT generated_timestamp AS timestamp, + COALESCE(AVG(visually_complete), 0) AS avg + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT pages.visually_complete - FROM events.pages INNER JOIN public.sessions USING (session_id) + FROM pages WHERE {" AND ".join(pg_sub_query_chart)} ) AS pages ON (TRUE) GROUP BY generated_timestamp - ORDER BY generated_timestamp) AS chart) AS chart ON(TRUE);""" + ORDER BY generated_timestamp) AS chart;""" params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, @@ -1493,16 +1631,24 @@ def __merge_rows_with_neutral(rows, neutral): def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, - data=args) - pg_sub_query_chart.append("resources.status/100 = %(status_code)s") + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, + data=args, main_table="resources", time_column="timestamp", project=False, + duration=False) + pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") + pg_sub_query_subset.append("resources.status/100 = %(status_code)s") with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, + pg_query = f"""WITH resources AS(SELECT resources.url_host, timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT generated_timestamp AS timestamp, COALESCE(JSONB_AGG(resources) FILTER ( WHERE resources IS NOT NULL ), '[]'::JSONB) AS keys FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.session_id) AS count - FROM events.resources INNER JOIN public.sessions USING (session_id) + LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.*) AS count + FROM resources WHERE {" AND ".join(pg_sub_query_chart)} GROUP BY url_host ORDER BY count DESC @@ -1536,16 +1682,22 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, - data=args) - pg_sub_query_chart.append("resources.status/100 = %(status_code)s") + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, + data=args, main_table="resources", time_column="timestamp", project=False, + duration=False) + pg_sub_query_subset.append("resources.status/100 = %(status_code)s") with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, + pg_query = f"""WITH resources AS (SELECT resources.url_host, timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT generated_timestamp AS timestamp, COALESCE(JSONB_AGG(resources) FILTER ( WHERE resources IS NOT NULL ), '[]'::JSONB) AS keys FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.session_id) AS count - FROM events.resources INNER JOIN public.sessions USING (session_id) + LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.url_host) AS count + FROM resources WHERE {" AND ".join(pg_sub_query_chart)} GROUP BY url_host ORDER BY count DESC @@ -1571,16 +1723,22 @@ def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) def get_domains_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, - data=args) - pg_sub_query_chart.append("resources.status/100 = %(status_code)s") + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, + data=args, main_table="resources", time_column="timestamp", project=False, + duration=False) + pg_sub_query_subset.append("resources.status/100 = %(status_code)s") with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, + pg_query = f"""WITH resources AS (SELECT resources.url_host, timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT generated_timestamp AS timestamp, COALESCE(JSONB_AGG(resources) FILTER ( WHERE resources IS NOT NULL ), '[]'::JSONB) AS keys FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.session_id) AS count - FROM events.resources INNER JOIN public.sessions USING (session_id) + LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.url_host) AS count + FROM resources WHERE {" AND ".join(pg_sub_query_chart)} GROUP BY url_host ORDER BY count DESC @@ -1785,33 +1943,55 @@ def get_calls_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) + + pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) + pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") + pg_sub_query_subset.append("resources.type != 'fetch'") + pg_sub_query_subset.append("resources.status > 200") + + pg_sub_query_subset_e = __get_constraints(project_id=project_id, data=args, duration=False, main_table="m_errors", + time_constraint=False) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, + chart=True, data=args, main_table="", time_column="timestamp", + project=False, duration=False) + pg_sub_query_subset_e.append("timestamp>=%(startTimestamp)s") + pg_sub_query_subset_e.append("timestamp<%(endTimestamp)s") with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(SUM(CASE WHEN type = 'fetch' AND status / 100 = 4 THEN 1 ELSE 0 END), 0) AS _4xx, - COALESCE(SUM(CASE WHEN type = 'fetch' AND status / 100 = 5 THEN 1 ELSE 0 END), 0) AS _5xx, - COALESCE(SUM(CASE WHEN type = 'js_exception' THEN 1 ELSE 0 END), 0) AS js, - COALESCE(SUM(CASE WHEN type != 'fetch' AND type != 'js_exception' THEN 1 ELSE 0 END), 0) AS integrations - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ((SELECT status, 'fetch' AS type - FROM events.resources - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - AND resources.timestamp >= %(startTimestamp)s - %(step_size)s - AND resources.timestamp < %(endTimestamp)s + %(step_size)s - AND resources.type = 'fetch' - AND resources.status > 200) - UNION ALL - (SELECT 0 AS status, m_errors.source::text AS type - FROM events.errors - INNER JOIN public.errors AS m_errors USING (error_id) - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - AND errors.timestamp >= %(startTimestamp)s - %(step_size)s - AND errors.timestamp < %(endTimestamp)s + %(step_size)s) - ) AS errors_partition ON (TRUE) + pg_query = f"""WITH resources AS (SELECT status, timestamp + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ), + errors_integ AS (SELECT timestamp + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + WHERE {" AND ".join(pg_sub_query_subset_e)} + AND source != 'js_exception' + ), + errors_js AS (SELECT timestamp + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + WHERE {" AND ".join(pg_sub_query_subset_e)} + AND source = 'js_exception' + ) + SELECT generated_timestamp AS timestamp, + COALESCE(SUM(CASE WHEN status / 100 = 4 THEN 1 ELSE 0 END), 0) AS _4xx, + COALESCE(SUM(CASE WHEN status / 100 = 5 THEN 1 ELSE 0 END), 0) AS _5xx, + COALESCE((SELECT COUNT(*) + FROM errors_js + WHERE {" AND ".join(pg_sub_query_chart)} + ), 0) AS js, + COALESCE((SELECT COUNT(*) + FROM errors_integ + WHERE {" AND ".join(pg_sub_query_chart)} + ), 0) AS integrations + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL (SELECT status + FROM resources + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS errors_partition ON (TRUE) GROUP BY timestamp ORDER BY timestamp;""" params = {"step_size": step_size, @@ -1828,32 +2008,49 @@ def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), e def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, + data=args, main_table="resources", time_column="timestamp", project=False, + duration=False) + pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, - COUNT(resources.session_id) AS total, + pg_query = f"""WITH resources AS(SELECT resources.type, resources.timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)}) + SELECT generated_timestamp AS timestamp, + COUNT(resources.*) AS total, SUM(CASE WHEN resources.type='fetch' THEN 1 ELSE 0 END) AS xhr FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL (SELECT resources.session_id, resources.type - FROM events.resources INNER JOIN public.sessions USING (session_id) + LEFT JOIN LATERAL (SELECT resources.type + FROM resources WHERE {" AND ".join(pg_sub_query_chart)}) AS resources ON (TRUE) GROUP BY generated_timestamp ORDER BY generated_timestamp;""" cur.execute(cur.mogrify(pg_query, params)) actions = cur.fetchall() - pg_sub_query_chart.append("pages.response_end IS NOT NULL") - pg_sub_query_chart.append("pages.response_end>0") - pg_query = f"""SELECT generated_timestamp AS timestamp, + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, + data=args, main_table="pages", time_column="timestamp", + project=False, + duration=False) + pg_sub_query_subset.append("pages.timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("pages.timestamp<%(endTimestamp)s") + pg_sub_query_subset.append("pages.response_end IS NOT NULL") + pg_sub_query_subset.append("pages.response_end>0") + pg_query = f"""WITH pages AS(SELECT pages.response_end, timestamp + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)}) + SELECT generated_timestamp AS timestamp, COALESCE(AVG(pages.response_end),0) AS avg_response_end FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL (SELECT pages.response_end - FROM events.pages INNER JOIN public.sessions USING (session_id) + FROM pages WHERE {" AND ".join(pg_sub_query_chart)}) AS pages ON(TRUE) GROUP BY generated_timestamp ORDER BY generated_timestamp;""" @@ -1878,51 +2075,60 @@ def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(de pg_sub_query_chart.append("errors.timestamp >= generated_timestamp") pg_sub_query_chart.append("errors.timestamp < generated_timestamp+ %(step_size)s") + pg_sub_query_subset = __get_constraints(project_id=project_id, data=args, duration=False, main_table="m_errors", + time_constraint=False) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, + chart=True, data=args, main_table="errors", time_column="timestamp", + project=False, duration=False) + pg_sub_query_subset.append("m_errors.source = 'js_exception'") + pg_sub_query_subset.append("errors.timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("errors.timestamp<%(endTimestamp)s") + with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT * - FROM (SELECT COUNT(DISTINCT errors.session_id) AS sessions_count - FROM events.errors - INNER JOIN public.errors AS m_errors USING (error_id) - INNER JOIN public.sessions USING(session_id) - WHERE {" AND ".join(pg_sub_query)}) AS counts - LEFT JOIN - (SELECT jsonb_agg(chart) AS chart - FROM(SELECT generated_timestamp AS timestamp, - COALESCE(COUNT(session_id), 0) AS sessions_count - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT DISTINCT errors.session_id - FROM events.errors - INNER JOIN public.errors AS m_errors USING (error_id) - INNER JOIN public.sessions USING (session_id) + pg_query = f"""WITH errors AS (SELECT DISTINCT ON (session_id,timestamp) session_id, timestamp + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT * + FROM (SELECT COUNT(DISTINCT session_id) AS sessions_count + FROM errors) AS counts + LEFT JOIN + (SELECT jsonb_agg(chart) AS chart + FROM (SELECT generated_timestamp AS timestamp, + COALESCE(COUNT(session_id), 0) AS sessions_count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT DISTINCT session_id + FROM errors WHERE {" AND ".join(pg_sub_query_chart)} - ) AS sessions ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp) AS chart) AS chart ON(TRUE);""" + ) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp) AS chart) AS chart ON (TRUE);""" cur.execute(cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)})) row_sessions = cur.fetchone() - pg_query = f"""SELECT * - FROM (SELECT COUNT(DISTINCT errors.error_id) AS errors_count - FROM events.errors - INNER JOIN public.errors AS m_errors USING (error_id) - INNER JOIN public.sessions USING(session_id) - WHERE {" AND ".join(pg_sub_query)}) AS counts - LEFT JOIN - (SELECT jsonb_agg(chart) AS chart - FROM(SELECT generated_timestamp AS timestamp, - COALESCE(COUNT(error_id), 0) AS errors_count - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT DISTINCT errors.error_id - FROM events.errors - INNER JOIN public.errors AS m_errors USING (error_id) - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS errors ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp) AS chart) AS chart ON(TRUE);""" + pg_query = f"""WITH errors AS ( SELECT DISTINCT ON(errors.error_id,timestamp) errors.error_id,timestamp + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT * + FROM (SELECT COUNT(DISTINCT errors.error_id) AS errors_count + FROM errors) AS counts + LEFT JOIN + (SELECT jsonb_agg(chart) AS chart + FROM (SELECT generated_timestamp AS timestamp, + COALESCE(COUNT(error_id), 0) AS errors_count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT DISTINCT errors.error_id + FROM errors + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS errors ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp) AS chart) AS chart ON (TRUE);""" cur.execute(cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, @@ -1939,10 +2145,23 @@ def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(de def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, + data=args, main_table="", time_column="timestamp", project=False, + duration=False) + pg_sub_query_subset.append("timestamp>=%(startTimestamp)s") + pg_sub_query_subset.append("timestamp<%(endTimestamp)s") with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, + pg_query = f"""WITH resources AS (SELECT resources.type, timestamp, session_id + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ), + pages AS (SELECT visually_complete, timestamp + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} AND pages.visually_complete > 0) + SELECT generated_timestamp AS timestamp, COALESCE(jsonb_agg(resources_avg_count_by_type) FILTER ( WHERE resources_avg_count_by_type IS NOT NULL ), '[]'::jsonb) AS types, COALESCE(AVG(total_count), 0) AS avg_count_resources, @@ -1951,17 +2170,15 @@ def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(de LEFT JOIN LATERAL (SELECT resources_count_by_session_by_type.type, avg(resources_count_by_session_by_type.count) AS avg_count, sum(resources_count_by_session_by_type.count) AS total_count - FROM (SELECT resources.type, COUNT(resources.url) AS count - FROM events.resources - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - GROUP BY resources.session_id, resources.type) AS resources_count_by_session_by_type + FROM (SELECT resources.type, COUNT(*) AS count + FROM resources + WHERE {" AND ".join(pg_sub_query_chart)} + GROUP BY resources.session_id, resources.type) AS resources_count_by_session_by_type GROUP BY resources_count_by_session_by_type.type) AS resources_avg_count_by_type ON (TRUE) LEFT JOIN LATERAL (SELECT AVG(visually_complete) AS avg_time_to_render - FROM events.pages - INNER JOIN public.sessions USING (session_id) + FROM pages WHERE {" AND ".join(pg_sub_query_chart)} - AND pages.visually_complete > 0) AS time_to_render ON (TRUE) + ) AS time_to_render ON (TRUE) GROUP BY generated_timestamp ORDER BY generated_timestamp;""" cur.execute(cur.mogrify(pg_query, {"step_size": step_size, @@ -1979,15 +2196,21 @@ def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(de def get_resources_count_by_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, + data=args, main_table="resources", time_column="timestamp", project=False, + duration=False) with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, + pg_query = f"""WITH resources AS (SELECT resources.type, timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT generated_timestamp AS timestamp, COALESCE(JSONB_AGG(t) FILTER (WHERE t IS NOT NULL), '[]'::JSONB) AS types FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL (SELECT resources.type, COUNT(resources.session_id) AS count - FROM events.resources INNER JOIN public.sessions USING (session_id) + LEFT JOIN LATERAL (SELECT resources.type, COUNT(*) AS count + FROM resources WHERE {" AND ".join(pg_sub_query_chart)} GROUP BY resources.type ) AS t ON(TRUE) @@ -2010,33 +2233,45 @@ def get_resources_count_by_type(project_id, startTimestamp=TimeUTC.now(delta_day def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, - data=args) - pg_sub_query_chart.append("resources.success = FALSE") - pg_sub_query = ["sessions.project_id =%(project_id)s", "rs.type IN ('fetch','script')"] + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") + pg_sub_query_subset.append("resources.success = FALSE") with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, + pg_query = f"""WITH resources AS ( + SELECT resources.url_host, timestamp + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT generated_timestamp AS timestamp, SUM(CASE WHEN first.url_host = sub_resources.url_host THEN 1 ELSE 0 END) AS first_party, SUM(CASE WHEN first.url_host != sub_resources.url_host THEN 1 ELSE 0 END) AS third_party FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN ( - SELECT resources.url_host, - COUNT(resources.session_id) AS count - FROM events.resources - INNER JOIN public.sessions USING (session_id) - WHERE sessions.project_id = '1' - AND resources.type IN ('fetch', 'script') - AND sessions.start_ts > (EXTRACT(EPOCH FROM now() - INTERVAL '31 days') * 1000)::BIGINT - AND resources.timestamp > (EXTRACT(EPOCH FROM now() - INTERVAL '31 days') * 1000)::BIGINT - GROUP BY resources.url_host - ORDER BY count DESC - LIMIT 1 - ) AS first ON (TRUE) - LEFT JOIN LATERAL ( - SELECT resources.url_host + SELECT resources.url_host, + COUNT(resources.session_id) AS count FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE sessions.project_id = '1' + AND resources.type IN ('fetch', 'script') + AND sessions.start_ts > (EXTRACT(EPOCH FROM now() - INTERVAL '31 days') * 1000)::BIGINT + AND sessions.start_ts < (EXTRACT(EPOCH FROM now()) * 1000)::BIGINT + AND resources.timestamp > (EXTRACT(EPOCH FROM now() - INTERVAL '31 days') * 1000)::BIGINT + AND resources.timestamp < (EXTRACT(EPOCH FROM now()) * 1000)::BIGINT + AND sessions.duration>0 + GROUP BY resources.url_host + ORDER BY count DESC + LIMIT 1 + ) AS first ON (TRUE) + LEFT JOIN LATERAL ( + SELECT resources.url_host + FROM resources WHERE {" AND ".join(pg_sub_query_chart)} ) AS sub_resources ON (TRUE) GROUP BY generated_timestamp @@ -2045,5 +2280,6 @@ def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1) "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() return rows diff --git a/scripts/helm/db/init_dbs/postgresql/1.2.0.sql b/scripts/helm/db/init_dbs/postgresql/1.2.0.sql new file mode 100644 index 000000000..321227acf --- /dev/null +++ b/scripts/helm/db/init_dbs/postgresql/1.2.0.sql @@ -0,0 +1,34 @@ +BEGIN; +CREATE INDEX pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0; +CREATE INDEX pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time > 0; +CREATE INDEX pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0; +CREATE INDEX pages_ttfb_idx ON events.pages (ttfb) WHERE ttfb > 0; +CREATE INDEX pages_time_to_interactive_idx ON events.pages (time_to_interactive) WHERE time_to_interactive > 0; +CREATE INDEX sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0; +CREATE INDEX pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL; +DROP INDEX events.resources_type_idx; +CREATE INDEX resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL; +CREATE INDEX pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL; +CREATE INDEX pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR first_paint_time > 0 OR + dom_content_loaded_time > 0 OR ttfb > 0 OR + time_to_interactive > 0; +CREATE INDEX resources_session_id_timestamp_idx ON events.resources (session_id, timestamp); +DROP INDEX events.resources_timestamp_idx; +CREATE INDEX resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type); +CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; +CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; +CREATE INDEX resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch'; +CREATE INDEX resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE; +DROP INDEX events.resources_success_idx; +CREATE INDEX resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script'); +DROP INDEX events.errors_timestamp_idx; +CREATE INDEX errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id); +CREATE INDEX errors_project_id_error_id_js_exception_idx ON public.errors (project_id, error_id) WHERE source = 'js_exception'; +CREATE INDEX errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); +CREATE INDEX errors_project_id_error_id_idx ON public.errors (project_id, error_id); +CREATE INDEX errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); +CREATE INDEX resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img'; +CREATE INDEX resources_timestamp_session_id_idx ON events.resources (timestamp, session_id); +CREATE INDEX errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception'; +CREATE INDEX errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id); +COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index b058f9dac..18c6838ce 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -57,7 +57,7 @@ CREATE TABLE users "role": "dev", "dashboard": { "cpu": true, - "fps": false, + "fps": false, "avgCpu": true, "avgFps": true, "errors": true, @@ -408,6 +408,9 @@ CREATE INDEX errors_message_gin_idx ON public.errors USING GIN (message gin_trgm CREATE INDEX errors_name_gin_idx ON public.errors USING GIN (name gin_trgm_ops); CREATE INDEX errors_project_id_idx ON public.errors (project_id); CREATE INDEX errors_project_id_status_idx ON public.errors (project_id, status); +CREATE INDEX errors_project_id_error_id_js_exception_idx ON public.errors (project_id, error_id) WHERE source = 'js_exception'; +CREATE INDEX errors_project_id_error_id_idx ON public.errors (project_id, error_id); +CREATE INDEX errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception'; CREATE TABLE user_favorite_errors ( @@ -513,6 +516,7 @@ CREATE INDEX ON sessions (project_id, user_country); CREATE INDEX ON sessions (project_id, user_browser); CREATE INDEX sessions_start_ts_idx ON public.sessions (start_ts) WHERE duration > 0; CREATE INDEX sessions_project_id_idx ON public.sessions (project_id) WHERE duration > 0; +CREATE INDEX sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0; ALTER TABLE public.sessions ADD CONSTRAINT web_browser_constraint CHECK ( (sessions.platform = 'web' AND sessions.user_browser NOTNULL) OR @@ -656,11 +660,18 @@ CREATE INDEX pages_path_idx ON events.pages (path); CREATE INDEX pages_visually_complete_idx ON events.pages (visually_complete) WHERE visually_complete > 0; CREATE INDEX pages_dom_building_time_idx ON events.pages (dom_building_time) WHERE dom_building_time > 0; CREATE INDEX pages_load_time_idx ON events.pages (load_time) WHERE load_time > 0; -CREATE INDEX pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time>0; -CREATE INDEX pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time>0; +CREATE INDEX pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0; +CREATE INDEX pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time > 0; CREATE INDEX pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0; CREATE INDEX pages_ttfb_idx ON events.pages (ttfb) WHERE ttfb > 0; CREATE INDEX pages_time_to_interactive_idx ON events.pages (time_to_interactive) WHERE time_to_interactive > 0; +CREATE INDEX pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL; +CREATE INDEX pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL; +CREATE INDEX pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR first_paint_time > 0 OR + dom_content_loaded_time > 0 OR ttfb > 0 OR + time_to_interactive > 0; +CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; +CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; CREATE TABLE events.clicks @@ -700,8 +711,10 @@ CREATE TABLE events.errors PRIMARY KEY (session_id, message_id) ); CREATE INDEX ON events.errors (session_id); -CREATE INDEX ON events.errors (timestamp); - +CREATE INDEX errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id); +CREATE INDEX errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); +CREATE INDEX errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); +CREATE INDEX errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id); CREATE TABLE events.graphql ( @@ -749,8 +762,6 @@ CREATE TABLE events.resources PRIMARY KEY (session_id, message_id) ); CREATE INDEX ON events.resources (session_id); -CREATE INDEX ON events.resources (timestamp); -CREATE INDEX ON events.resources (success); CREATE INDEX ON events.resources (status); CREATE INDEX ON events.resources (type); CREATE INDEX ON events.resources (duration) WHERE duration > 0; @@ -760,8 +771,14 @@ CREATE INDEX resources_url_gin_idx ON events.resources USING GIN (url gin_trgm_o CREATE INDEX resources_url_idx ON events.resources (url); CREATE INDEX resources_url_hostpath_gin_idx ON events.resources USING GIN (url_hostpath gin_trgm_ops); CREATE INDEX resources_url_hostpath_idx ON events.resources (url_hostpath); - - +CREATE INDEX resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL; +CREATE INDEX resources_session_id_timestamp_idx ON events.resources (session_id, timestamp); +CREATE INDEX resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type); +CREATE INDEX resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch'; +CREATE INDEX resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE; +CREATE INDEX resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script'); +CREATE INDEX resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img'; +CREATE INDEX resources_timestamp_session_id_idx ON events.resources (timestamp, session_id); CREATE TABLE events.performance (