diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index c6fc26ba6..1735a64ca 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -34,30 +34,6 @@ LeftToDb = { schemas.AlertColumn.PERFORMANCE__TIME_TO_RENDER__AVERAGE: { "table": "events.pages INNER JOIN public.sessions USING(session_id)", "formula": "AVG(NULLIF(visually_complete,0))"}, - schemas.AlertColumn.PERFORMANCE__IMAGE_LOAD_TIME__AVERAGE: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='img'"}, - schemas.AlertColumn.PERFORMANCE__REQUEST_LOAD_TIME__AVERAGE: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='fetch'"}, - schemas.AlertColumn.RESOURCES__LOAD_TIME__AVERAGE: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(resources.duration,0))"}, - schemas.AlertColumn.RESOURCES__MISSING__COUNT: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(DISTINCT url_hostpath)", "condition": "success= FALSE AND type='img'"}, - schemas.AlertColumn.ERRORS__4XX_5XX__COUNT: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(session_id)", - "condition": "status/100!=2"}, - schemas.AlertColumn.ERRORS__4XX__COUNT: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=4"}, - schemas.AlertColumn.ERRORS__5XX__COUNT: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=5"}, - schemas.AlertColumn.ERRORS__JAVASCRIPT__IMPACTED_SESSIONS__COUNT: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(DISTINCT session_id)", "condition": "success= FALSE AND type='script'"}, schemas.AlertColumn.PERFORMANCE__CRASHES__COUNT: { "table": "public.sessions", "formula": "COUNT(DISTINCT session_id)", diff --git a/api/chalicelib/core/custom_metrics_predefined.py b/api/chalicelib/core/custom_metrics_predefined.py index 62f890194..6baf7896c 100644 --- a/api/chalicelib/core/custom_metrics_predefined.py +++ b/api/chalicelib/core/custom_metrics_predefined.py @@ -7,53 +7,16 @@ from chalicelib.core import metrics logger = logging.getLogger(__name__) -def get_metric(key: Union[schemas.MetricOfWebVitals, schemas.MetricOfErrors, \ - schemas.MetricOfPerformance, schemas.MetricOfResources], project_id: int, data: dict): +def get_metric(key: Union[schemas.MetricOfWebVitals, schemas.MetricOfErrors], project_id: int, data: dict): supported = {schemas.MetricOfWebVitals.COUNT_SESSIONS: metrics.get_processed_sessions, - schemas.MetricOfWebVitals.AVG_IMAGE_LOAD_TIME: metrics.get_application_activity_avg_image_load_time, - schemas.MetricOfWebVitals.AVG_PAGE_LOAD_TIME: metrics.get_application_activity_avg_page_load_time, - schemas.MetricOfWebVitals.AVG_REQUEST_LOAD_TIME: metrics.get_application_activity_avg_request_load_time, - schemas.MetricOfWebVitals.AVG_DOM_CONTENT_LOAD_START: metrics.get_page_metrics_avg_dom_content_load_start, - schemas.MetricOfWebVitals.AVG_FIRST_CONTENTFUL_PIXEL: metrics.get_page_metrics_avg_first_contentful_pixel, schemas.MetricOfWebVitals.AVG_VISITED_PAGES: metrics.get_user_activity_avg_visited_pages, - schemas.MetricOfWebVitals.AVG_SESSION_DURATION: metrics.get_user_activity_avg_session_duration, - schemas.MetricOfWebVitals.AVG_PAGES_DOM_BUILDTIME: metrics.get_pages_dom_build_time, - schemas.MetricOfWebVitals.AVG_PAGES_RESPONSE_TIME: metrics.get_pages_response_time, - schemas.MetricOfWebVitals.AVG_RESPONSE_TIME: metrics.get_top_metrics_avg_response_time, - schemas.MetricOfWebVitals.AVG_FIRST_PAINT: metrics.get_top_metrics_avg_first_paint, - schemas.MetricOfWebVitals.AVG_DOM_CONTENT_LOADED: metrics.get_top_metrics_avg_dom_content_loaded, - schemas.MetricOfWebVitals.AVG_TILL_FIRST_BYTE: metrics.get_top_metrics_avg_till_first_bit, - schemas.MetricOfWebVitals.AVG_TIME_TO_INTERACTIVE: metrics.get_top_metrics_avg_time_to_interactive, schemas.MetricOfWebVitals.COUNT_REQUESTS: metrics.get_top_metrics_count_requests, - schemas.MetricOfWebVitals.AVG_TIME_TO_RENDER: metrics.get_time_to_render, - schemas.MetricOfWebVitals.AVG_USED_JS_HEAP_SIZE: metrics.get_memory_consumption, - schemas.MetricOfWebVitals.AVG_CPU: metrics.get_avg_cpu, - schemas.MetricOfWebVitals.AVG_FPS: metrics.get_avg_fps, schemas.MetricOfErrors.IMPACTED_SESSIONS_BY_JS_ERRORS: metrics.get_impacted_sessions_by_js_errors, schemas.MetricOfErrors.DOMAINS_ERRORS_4XX: metrics.get_domains_errors_4xx, schemas.MetricOfErrors.DOMAINS_ERRORS_5XX: metrics.get_domains_errors_5xx, schemas.MetricOfErrors.ERRORS_PER_DOMAINS: metrics.get_errors_per_domains, - schemas.MetricOfErrors.CALLS_ERRORS: metrics.get_calls_errors, schemas.MetricOfErrors.ERRORS_PER_TYPE: metrics.get_errors_per_type, schemas.MetricOfErrors.RESOURCES_BY_PARTY: metrics.get_resources_by_party, - schemas.MetricOfPerformance.SPEED_LOCATION: metrics.get_speed_index_location, - schemas.MetricOfPerformance.SLOWEST_DOMAINS: metrics.get_slowest_domains, - schemas.MetricOfPerformance.SESSIONS_PER_BROWSER: metrics.get_sessions_per_browser, - schemas.MetricOfPerformance.TIME_TO_RENDER: metrics.get_time_to_render, - schemas.MetricOfPerformance.IMPACTED_SESSIONS_BY_SLOW_PAGES: metrics.get_impacted_sessions_by_slow_pages, - schemas.MetricOfPerformance.MEMORY_CONSUMPTION: metrics.get_memory_consumption, - schemas.MetricOfPerformance.CPU: metrics.get_avg_cpu, - schemas.MetricOfPerformance.FPS: metrics.get_avg_fps, - schemas.MetricOfPerformance.CRASHES: metrics.get_crashes, - schemas.MetricOfPerformance.RESOURCES_VS_VISUALLY_COMPLETE: metrics.get_resources_vs_visually_complete, - schemas.MetricOfPerformance.PAGES_DOM_BUILDTIME: metrics.get_pages_dom_build_time, - schemas.MetricOfPerformance.PAGES_RESPONSE_TIME: metrics.get_pages_response_time, - schemas.MetricOfPerformance.PAGES_RESPONSE_TIME_DISTRIBUTION: metrics.get_pages_response_time_distribution, - schemas.MetricOfResources.MISSING_RESOURCES: metrics.get_missing_resources_trend, - schemas.MetricOfResources.SLOWEST_RESOURCES: metrics.get_slowest_resources, - schemas.MetricOfResources.RESOURCES_LOADING_TIME: metrics.get_resources_loading_time, - schemas.MetricOfResources.RESOURCE_TYPE_VS_RESPONSE_END: metrics.resource_type_vs_response_end, - schemas.MetricOfResources.RESOURCES_COUNT_BY_TYPE: metrics.get_resources_count_by_type, schemas.MetricOfWebVitals.COUNT_USERS: metrics.get_unique_users, } return supported.get(key, lambda *args: None)(project_id=project_id, **data) diff --git a/api/chalicelib/core/metrics.py b/api/chalicelib/core/metrics.py index c421fff7e..c375c2c09 100644 --- a/api/chalicelib/core/metrics.py +++ b/api/chalicelib/core/metrics.py @@ -382,213 +382,6 @@ def __get_resource_db_type_from_type(resource_type): return {v: k for k, v in RESOURCS_TYPE_TO_DB_TYPE.items()}.get(resource_type, resource_type) -def search(text, resource_type, project_id, performance=False, pages_only=False, events_only=False, - metadata=False, key=None, platform=None): - if not resource_type: - data = [] - if metadata: - resource_type = "METADATA" - elif pages_only or performance: - resource_type = "LOCATION" - else: - resource_type = "ALL" - data.extend(search(text=text, resource_type=resource_type, project_id=project_id, - performance=performance, pages_only=pages_only, events_only=events_only, key=key, - platform=platform)) - return data - - pg_sub_query = __get_constraints(project_id=project_id, time_constraint=False, duration=True, - data={} if platform is None else {"platform": platform}) - - if resource_type == "ALL" and not pages_only and not events_only: - pg_sub_query.append("url_hostpath ILIKE %(value)s") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT key, value - FROM ( SELECT DISTINCT ON (url_hostpath) ROW_NUMBER() OVER (PARTITION BY type ORDER BY url_hostpath) AS r, - url_hostpath AS value, - type AS key - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - ORDER BY url_hostpath, type ASC) AS ranked_values - WHERE ranked_values.r<=5;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text)})) - rows = cur.fetchall() - rows = [{"value": i["value"], "type": __get_resource_type_from_db_type(i["key"])} for i in rows] - elif resource_type == "ALL" and events_only: - with pg_client.PostgresClient() as cur: - pg_query = f"""(SELECT DISTINCT label AS value, 'INPUT' AS key - FROM events.inputs INNER JOIN public.sessions USING(session_id) - WHERE {" AND ".join(pg_sub_query)} AND positionUTF8(lowerUTF8(label), %(value)s) != 0 - LIMIT 10) - UNION ALL - (SELECT DISTINCT label AS value, 'CLICK' AS key - FROM events.clicks INNER JOIN public.sessions USING(session_id) - WHERE {" AND ".join(pg_sub_query)} AND positionUTF8(lowerUTF8(label), %(value)s) != 0 - LIMIT 10) - UNION ALL - (SELECT DISTINCT path AS value, 'LOCATION' AS key - FROM events.pages INNER JOIN public.sessions USING(session_id) - WHERE {" AND ".join(pg_sub_query)} AND positionUTF8(url_path, %(value)s) != 0 - LIMIT 10);""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text.lower()), - "platform_0": platform})) - rows = cur.fetchall() - rows = [{"value": i["value"], "type": i["key"]} for i in rows] - elif resource_type in ['IMG', 'REQUEST', 'STYLESHEET', 'OTHER', 'SCRIPT'] and not pages_only: - pg_sub_query.append("url_hostpath ILIKE %(value)s") - pg_sub_query.append(f"resources.type = '{__get_resource_db_type_from_type(resource_type)}'") - - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT - DISTINCT url_hostpath AS value, - %(resource_type)s AS type - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - LIMIT 10;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "resource_type": resource_type, - "platform_0": platform})) - rows = cur.fetchall() - elif resource_type == 'LOCATION': - with pg_client.PostgresClient() as cur: - pg_sub_query.append("path ILIKE %(value)s") - pg_query = f"""SELECT - DISTINCT path AS value, - 'LOCATION' AS type - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - LIMIT 10;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "platform_0": platform})) - rows = cur.fetchall() - elif resource_type == "INPUT": - with pg_client.PostgresClient() as cur: - pg_sub_query.append("label ILIKE %(value)s") - pg_query = f"""SELECT DISTINCT label AS value, 'INPUT' AS type - FROM events.inputs INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - LIMIT 10;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "platform_0": platform})) - rows = cur.fetchall() - elif resource_type == "CLICK": - with pg_client.PostgresClient() as cur: - pg_sub_query.append("label ILIKE %(value)s") - pg_query = f"""SELECT DISTINCT label AS value, 'CLICK' AS key - FROM events.clicks INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - LIMIT 10;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "platform_0": platform})) - rows = cur.fetchall() - elif resource_type == "METADATA": - if key and len(key) > 0 and key in {**METADATA_FIELDS, **SESSIONS_META_FIELDS}.keys(): - if key in METADATA_FIELDS.keys(): - column = METADATA_FIELDS[key] - pg_sub_query.append(f"{METADATA_FIELDS[key]} ILIKE %(value)s") - else: - column = SESSIONS_META_FIELDS[key] - pg_sub_query.append(f"{SESSIONS_META_FIELDS[key]} ILIKE %(value)s") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT DISTINCT {column} AS value, - %(key)s AS key - FROM sessions - WHERE {" AND ".join(pg_sub_query)} - LIMIT 10;""" - cur.execute(cur.mogrify(pg_query, - {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, - "platform_0": platform})) - rows = cur.fetchall() - else: - with pg_client.PostgresClient() as cur: - pg_query = [] - for k in METADATA_FIELDS.keys(): - pg_query.append(f"""(SELECT DISTINCT sessions.{METADATA_FIELDS[k]} AS value, - '{k}' AS key - FROM public.sessions - WHERE {" AND ".join(pg_sub_query)} - AND {METADATA_FIELDS[k]} ILIKE %(value)s - LIMIT 10)""") - for k in SESSIONS_META_FIELDS.keys(): - if k in ["platform", "country"]: - continue - pg_query.append(f"""(SELECT DISTINCT sessions.{SESSIONS_META_FIELDS[k]} AS value, - '{k}' AS key - FROM sessions - WHERE {" AND ".join(pg_sub_query)} - AND sessions.{SESSIONS_META_FIELDS[k]} ILIKE %(value)s - LIMIT 10)""") - pg_query = " UNION ALL ".join(pg_query) - cur.execute(cur.mogrify(pg_query, - {"project_id": project_id, "value": helper.string_to_sql_like(text), - "key": key, - "platform_0": platform})) - rows = cur.fetchall() - else: - return [] - return [helper.dict_to_camel_case(row) for row in rows] - - -def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, data=args) - pg_sub_query.append("resources.success = FALSE") - pg_sub_query_chart.append("resources.success = FALSE") - pg_sub_query.append("resources.type = 'img'") - pg_sub_query_chart.append("resources.type = 'img'") - - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT - resources.url_hostpath AS url, - COUNT(resources.session_id) AS sessions - FROM events.resources INNER JOIN sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - GROUP BY url_hostpath - ORDER BY sessions DESC - LIMIT 10;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - - rows = cur.fetchall() - # rows = [{"url": i["key"], "sessions": i["doc_count"]} for i in rows] - if len(rows) == 0: - return [] - pg_sub_query.append("resources.url_hostpath = %(value)s") - pg_query = f"""SELECT generated_timestamp AS timestamp, - COUNT(resources.session_id) AS count, - MAX(resources.timestamp) AS max_datatime - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.session_id, - resources.timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - for e in rows: - e["startedAt"] = startTimestamp - e["startTimestamp"] = startTimestamp - e["endTimestamp"] = endTimestamp - - cur.execute(cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": e["url"], - **__get_constraint_values(args)})) - r = cur.fetchall() - e["endedAt"] = r[-1]["max_datatime"] - e["chart"] = [{"timestamp": i["timestamp"], "count": i["count"]} for i in r] - return rows - - KEYS = { 'startTimestamp': args_transformer.int_arg, 'endTimestamp': args_transformer.int_arg, @@ -607,173 +400,6 @@ def dashboard_args(params): return args -def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=19, type=None, url=None, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, - chart=True, data=args, main_table="resources", time_column="timestamp", - project=False, duration=False) - pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") - pg_sub_query_subset.append("resources.duration>0") - pg_sub_query_subset.append("resources.duration IS NOT NULL") - - if type is not None: - pg_sub_query_subset.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'") - if url is not None: - pg_sub_query_subset.append(f"resources.url_hostpath = %(value)s") - - with pg_client.PostgresClient() as cur: - pg_query = f"""WITH resources AS (SELECT resources.duration, timestamp - FROM events.resources - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - ) - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(resources.duration), 0) AS avg - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT resources.duration - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": url, "type": type, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - pg_query = f"""SELECT COALESCE(AVG(resources.duration),0) AS avg - FROM events.resources INNER JOIN sessions USING(session_id) - WHERE {" AND ".join(pg_sub_query_subset)};""" - cur.execute(cur.mogrify(pg_query, params)) - avg = cur.fetchone()["avg"] - - return {"avg": avg, "chart": rows} - - -def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=19, url=None, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, - chart=True, data=args, main_table="pages", time_column="timestamp", - project=False, duration=False) - - if url is not None: - pg_sub_query_subset.append(f"pages.path = %(value)s") - - pg_sub_query_subset.append("pages.timestamp>=%(startTimestamp)s") - pg_sub_query_subset.append("pages.timestamp<%(endTimestamp)s") - pg_sub_query_subset.append("pages.dom_building_time>0") - pg_sub_query_subset.append("pages.dom_building_time IS NOT NULL") - - with pg_client.PostgresClient() as cur: - pg_query = f"""WITH pages AS ( SELECT pages.dom_building_time, timestamp - FROM public.sessions - INNER JOIN events.pages USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)}) - SELECT COALESCE(avg, 0) AS value, chart - FROM (SELECT AVG(dom_building_time) FROM pages) AS avg - LEFT JOIN - (SELECT jsonb_agg(chart) AS chart - FROM ( - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(dom_building_time), 0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT pages.dom_building_time - FROM pages - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS sessionsBD ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp) AS chart) AS chart ON (TRUE);""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": url, **__get_constraint_values(args)} - - cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - helper.__time_value(row) - return row - - -def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), type="all", density=19, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, - chart=True, data=args, main_table="resources", time_column="timestamp", - project=False, duration=False) - - pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") - pg_sub_query_subset.append("resources.duration>0") - pg_sub_query_subset.append("resources.duration IS NOT NULL") - if type is not None and type.upper() != "ALL": - sq = f"resources.type = '{__get_resource_db_type_from_type(type.upper())}'" - else: - sq = "resources.type != 'fetch'" - pg_sub_query.append(sq) - pg_sub_query_subset.append(sq) - pg_sub_query_chart.append("resources.url_hostpath ILIKE '%%' || main_list.name") - - with pg_client.PostgresClient() as cur: - pg_query = f"""WITH resources AS ( - SELECT resources.duration, url_hostpath, timestamp - FROM events.resources - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - ) - SELECT * - FROM (SELECT regexp_replace(resources.url_hostpath, '^.*/', '') AS name, - AVG(resources.duration) AS avg - FROM events.resources - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - GROUP BY name - ORDER BY avg DESC - LIMIT 10) AS main_list - INNER JOIN LATERAL ( - SELECT url_hostpath AS url, type - FROM events.resources - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - AND resources.url_hostpath ILIKE '%%' || main_list.name - ORDER BY resources.duration DESC - LIMIT 1 - ) AS resource_details ON (TRUE) - INNER JOIN LATERAL ( - SELECT JSONB_AGG(chart_details) AS chart - FROM ( - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(resources.duration), 0) AS avg - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT resources.duration - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp - ) AS chart_details - ) AS chart_details ON (TRUE);""" - - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "step_size": step_size, - **__get_constraint_values(args)})) - rows = cur.fetchall() - for r in rows: - r["type"] = __get_resource_type_from_db_type(r["type"]) - return rows - - def get_sessions_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -792,212 +418,6 @@ def get_sessions_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return {"count": sum(i["count"] for i in rows), "chart": helper.list_to_camel_case(rows)} -def get_speed_index_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("pages.speed_index IS NOT NULL") - pg_sub_query.append("pages.speed_index>0") - - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT sessions.user_country, AVG(pages.speed_index) AS value - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - GROUP BY sessions.user_country - ORDER BY value, sessions.user_country;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - if len(rows) > 0: - pg_query = f"""SELECT AVG(pages.speed_index) AS avg - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - cur.execute(cur.mogrify(pg_query, params)) - avg = cur.fetchone()["avg"] - else: - avg = 0 - return {"value": avg, "chart": helper.list_to_camel_case(rows), "unit": schemas.TemplatePredefinedUnits.MILLISECOND} - - -def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, url=None, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("pages.response_time IS NOT NULL") - pg_sub_query.append("pages.response_time>0") - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, - data=args) - pg_sub_query_chart.append("pages.response_time IS NOT NULL") - pg_sub_query_chart.append("pages.response_time>0") - - if url is not None: - pg_sub_query_chart.append(f"url = %(value)s") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.response_time),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT response_time - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": url, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - pg_query = f"""SELECT COALESCE(AVG(pages.response_time),0) AS avg - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - cur.execute(cur.mogrify(pg_query, params)) - avg = cur.fetchone()["avg"] - result = {"value": avg, "chart": rows} - helper.__time_value(result) - return result - - -def get_pages_response_time_distribution(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=20, **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("pages.response_time IS NOT NULL") - pg_sub_query.append("pages.response_time>0") - - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT pages.response_time AS response_time, - COUNT(pages.session_id) AS count - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - GROUP BY response_time - ORDER BY pages.response_time;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - rows = cur.fetchall() - pg_query = f"""SELECT COALESCE(AVG(pages.response_time),0) AS avg - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - avg = cur.fetchone()["avg"] - quantiles_keys = [50, 90, 95, 99] - pg_query = f"""SELECT pages.response_time AS value - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - response_times = cur.fetchall() - response_times = [i["value"] for i in response_times] - if len(response_times) > 0: - quantiles = __quantiles(a=response_times, - q=[i / 100 for i in quantiles_keys], - interpolation='higher') - else: - quantiles = [0 for i in range(len(quantiles_keys))] - result = { - "value": avg, - "total": sum(r["count"] for r in rows), - "chart": [], - "percentiles": [{ - "percentile": float(v), - "responseTime": int(quantiles[i]) - } for i, v in enumerate(quantiles_keys) - ], - "extremeValues": [{"count": 0}], - "unit": schemas.TemplatePredefinedUnits.MILLISECOND - } - rows = helper.list_to_camel_case(rows) - _99 = result["percentiles"][-1]["responseTime"] - extreme_values_first_index = -1 - for i, r in enumerate(rows): - if r["responseTime"] > _99: - extreme_values_first_index = i - break - - if extreme_values_first_index >= 0: - extreme_values_first_index += 1 - result["extremeValues"][0]["count"] = sum(r["count"] for r in rows[extreme_values_first_index:]) - rows = rows[:extreme_values_first_index] - - # ------- Merge points to reduce chart length till density - if density < len(quantiles_keys): - density = len(quantiles_keys) - - while len(rows) > density: - true_length = len(rows) - rows_partitions = [] - offset = 0 - for p in result["percentiles"]: - rows_partitions.append([]) - for r in rows[offset:]: - if r["responseTime"] < p["responseTime"]: - rows_partitions[-1].append(r) - offset += 1 - else: - break - rows_partitions.append(rows[offset:]) - - largest_partition = 0 - for i in range(len(rows_partitions)): - if len(rows_partitions[i]) > len(rows_partitions[largest_partition]): - largest_partition = i - - if len(rows_partitions[largest_partition]) <= 2: - break - # computing lowest merge diff - diff = rows[-1]["responseTime"] - for i in range(1, len(rows_partitions[largest_partition]) - 1, 1): - v1 = rows_partitions[largest_partition][i] - v2 = rows_partitions[largest_partition][i + 1] - if (v2["responseTime"] - v1["responseTime"]) < diff: - diff = v2["responseTime"] - v1["responseTime"] - - i = 1 - while i < len(rows_partitions[largest_partition]) - 1 and true_length > density - 1: - v1 = rows_partitions[largest_partition][i] - v2 = rows_partitions[largest_partition][i + 1] - if (v2["responseTime"] - v1["responseTime"]) == diff: - rows_partitions[largest_partition][i]["count"] += v2["count"] - rows_partitions[largest_partition][i]["responseTime"] = v2["responseTime"] - del rows_partitions[largest_partition][i + 1] - true_length -= 1 - else: - i += 1 - - rows = [r for rp in rows_partitions for r in rp] - - if extreme_values_first_index == len(rows): - rows.append({"count": 0, "responseTime": rows[-1]["responseTime"] + 10}) - - result["chart"] = rows - return result - - -def get_busiest_time_of_day(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT - (EXTRACT(HOUR FROM TO_TIMESTAMP(sessions.start_ts))::INTEGER / 2) * 2 AS hour, - COUNT(sessions.session_id) AS count - FROM public.sessions - WHERE {" AND ".join(pg_sub_query)} - GROUP BY hour - ORDER BY hour ASC;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - rows = cur.fetchall() - return rows - - def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), value=None, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1048,250 +468,6 @@ def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return helper.dict_to_camel_case(row) -def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, url=None, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, - chart=True, data=args, main_table="pages", time_column="timestamp", - project=False, duration=False) - pg_sub_query_subset.append("pages.visually_complete>0") - if url is not None: - pg_sub_query_subset.append("pages.path = %(value)s") - - with pg_client.PostgresClient() as cur: - pg_query = f"""WITH pages AS(SELECT pages.visually_complete,pages.timestamp - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)}) - SELECT COALESCE((SELECT AVG(pages.visually_complete) FROM pages),0) AS value, - jsonb_agg(chart) AS chart - FROM - (SELECT generated_timestamp AS timestamp, - COALESCE(AVG(visually_complete), 0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT pages.visually_complete - FROM pages - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp) AS chart;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, "value": url, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - helper.__time_value(row) - return row - - -def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, - data=args) - pg_sub_query.append("pages.response_time IS NOT NULL") - pg_sub_query_chart.append("pages.response_time IS NOT NULL") - pg_sub_query.append("pages.response_time>0") - pg_sub_query_chart.append("pages.response_time>0") - if value is not None: - pg_sub_query.append("pages.path = %(value)s") - pg_sub_query_chart.append("pages.path = %(value)s") - pg_sub_query_chart.append("avg_response_time>0") - pg_sub_query_chart.append("pages.response_time>avg_response_time*2") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, - COUNT(pages.session_id) AS count - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN ( SELECT AVG(pages.response_time) AS avg_response_time - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - ) AS avg_response_time ON (avg_response_time>0) - LEFT JOIN LATERAL ( SELECT DISTINCT pages.session_id - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)})) - rows = cur.fetchall() - return rows - - -def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) - - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(performance.avg_used_js_heap_size),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT avg_used_js_heap_size - FROM events.performance INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS performance ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp ASC;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - pg_query = f"""SELECT COALESCE(AVG(performance.avg_used_js_heap_size),0) AS avg - FROM events.performance INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - cur.execute(cur.mogrify(pg_query, params)) - avg = cur.fetchone()["avg"] - return {"value": avg, "chart": helper.list_to_camel_case(rows), "unit": schemas.TemplatePredefinedUnits.MEMORY} - - -def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) - - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(performance.avg_cpu),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT avg_cpu - FROM events.performance INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS performance ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp ASC;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - pg_query = f"""SELECT COALESCE(AVG(performance.avg_cpu),0) AS avg - FROM events.performance INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - cur.execute(cur.mogrify(pg_query, params)) - avg = cur.fetchone()["avg"] - return {"value": avg, "chart": helper.list_to_camel_case(rows), - "unit": schemas.TemplatePredefinedUnits.PERCENTAGE} - - -def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) - pg_sub_query.append("performance.avg_fps>0") - pg_sub_query_chart.append("performance.avg_fps>0") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(performance.avg_fps),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT avg_fps - FROM events.performance INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS performance ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp ASC;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - pg_query = f"""SELECT COALESCE(AVG(performance.avg_fps),0) AS avg - FROM events.performance INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - cur.execute(cur.mogrify(pg_query, params)) - avg = cur.fetchone()["avg"] - return {"value": avg, "chart": helper.list_to_camel_case(rows), "unit": schemas.TemplatePredefinedUnits.FRAME} - - -def get_crashes(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("m_issues.type = 'crash'") - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) - pg_sub_query_chart.append("m_issues.type = 'crash'") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT generated_timestamp AS timestamp, - COUNT(sessions) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT sessions.session_id - FROM public.sessions - INNER JOIN events_common.issues USING (session_id) - INNER JOIN public.issues AS m_issues USING (issue_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS sessions ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - **__get_constraint_values(args)})) - rows = cur.fetchall() - pg_query = f"""SELECT b.user_browser AS browser, - sum(bv.count) AS total, - JSONB_AGG(bv) AS versions - FROM ( - SELECT sessions.user_browser - FROM public.sessions - INNER JOIN events_common.issues USING (session_id) - INNER JOIN public.issues AS m_issues USING (issue_id) - WHERE {" AND ".join(pg_sub_query)} - GROUP BY sessions.user_browser - ORDER BY COUNT(sessions.session_id) DESC - LIMIT 3 - ) AS b - INNER JOIN LATERAL - ( - SELECT sessions.user_browser_version AS version, - COUNT(sessions.session_id) AS count - FROM sessions - INNER JOIN events_common.issues USING (session_id) - INNER JOIN public.issues AS m_issues USING (issue_id) - WHERE {" AND ".join(pg_sub_query)} - AND sessions.user_browser = b.user_browser - GROUP BY sessions.user_browser_version - ORDER BY count DESC - ) AS bv ON (TRUE) - GROUP BY b.user_browser - ORDER BY b.user_browser;""" - cur.execute(cur.mogrify(pg_query, {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - **__get_constraint_values(args)})) - browsers = cur.fetchall() - total = sum(r["total"] for r in browsers) - for r in browsers: - r["percentage"] = r["total"] / (total / 100) - versions = [] - for v in r["versions"][:3]: - versions.append({v["version"]: v["count"] / (r["total"] / 100)}) - r["versions"] = versions - - return {"chart": rows, "browsers": browsers, "unit": schemas.TemplatePredefinedUnits.COUNT} - - def __get_neutral(rows, add_All_if_empty=True): neutral = {l: 0 for l in [i for k in [list(v.keys()) for v in rows] for i in k]} if add_All_if_empty and len(neutral.keys()) <= 1: @@ -1415,37 +591,6 @@ def __nested_array_to_dict_array(rows, key="url_host", value="count"): return rows -def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("resources.duration IS NOT NULL") - pg_sub_query.append("resources.duration>0") - - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT - resources.url_host AS domain, - AVG(resources.duration) AS value - FROM events.resources INNER JOIN sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - GROUP BY resources.url_host - ORDER BY value DESC - LIMIT 5;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - if len(rows) > 0: - pg_query = f"""SELECT AVG(resources.duration) AS avg - FROM events.resources INNER JOIN sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - cur.execute(cur.mogrify(pg_query, params)) - avg = cur.fetchone()["avg"] - else: - avg = 0 - return {"value": avg, "chart": rows, "unit": schemas.TemplatePredefinedUnits.MILLISECOND} - - def get_errors_per_domains(project_id, limit, page, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1484,73 +629,6 @@ def get_errors_per_domains(project_id, limit, page, startTimestamp=TimeUTC.now(d return helper.dict_to_camel_case(row) -def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), - platform=None, **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query2 = pg_sub_query[:] - pg_sub_query2.append("sessions.user_browser = b.user_browser") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT b.user_browser AS browser, - b.count, - jsonb_agg(bv) AS versions - FROM ( - SELECT sessions.user_browser, - COUNT(sessions.session_id) AS count - FROM sessions - WHERE {" AND ".join(pg_sub_query)} - GROUP BY sessions.user_browser - ORDER BY count DESC - LIMIT 3 - ) AS b - INNER JOIN LATERAL - ( - SELECT sessions.user_browser_version, - COUNT(sessions.session_id) AS count - FROM public.sessions - WHERE {" AND ".join(pg_sub_query2)} - GROUP BY sessions.user_browser, - sessions.user_browser_version - ORDER BY count DESC - LIMIT 3 - ) AS bv ON (TRUE) - GROUP BY b.user_browser, b.count - ORDER BY b.count DESC;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - rows = cur.fetchall() - for r in rows: - for j in r["versions"]: - r[j["user_browser_version"]] = j["count"] - r.pop("versions") - return {"count": sum(i["count"] for i in rows), "chart": rows} - - -def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), - platform=None, **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("resources.type = 'fetch'") - pg_sub_query.append("resources.method IS NOT NULL") - pg_sub_query.append("resources.status/100 != 2") - - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT resources.method, - resources.url_hostpath, - COUNT(resources.session_id) AS all_requests, - SUM(CASE WHEN resources.status/100 = 4 THEN 1 ELSE 0 END) AS _4xx, - SUM(CASE WHEN resources.status/100 = 5 THEN 1 ELSE 0 END) AS _5xx - FROM events.resources INNER JOIN sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - GROUP BY resources.method, resources.url_hostpath - ORDER BY (4 + 5) DESC, 3 DESC - LIMIT 50;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - rows = cur.fetchall() - return helper.list_to_camel_case(rows) - - def __get_calls_errors_4xx_or_5xx(status, project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, **args): @@ -1655,60 +733,6 @@ def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), e return rows -def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, - data=args, main_table="resources", time_column="timestamp", project=False, - duration=False) - pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") - - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - with pg_client.PostgresClient() as cur: - pg_query = f"""WITH resources AS(SELECT resources.type, resources.timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)}) - SELECT generated_timestamp AS timestamp, - COUNT(resources.*) AS total, - SUM(CASE WHEN resources.type='fetch' THEN 1 ELSE 0 END) AS xhr - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL (SELECT resources.type - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)}) AS resources ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, params)) - actions = cur.fetchall() - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, - data=args, main_table="pages", time_column="timestamp", - project=False, - duration=False) - pg_sub_query_subset.append("pages.timestamp>=%(startTimestamp)s") - pg_sub_query_subset.append("pages.timestamp<%(endTimestamp)s") - pg_sub_query_subset.append("pages.response_end IS NOT NULL") - pg_sub_query_subset.append("pages.response_end>0") - pg_query = f"""WITH pages AS(SELECT pages.response_end, timestamp - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)}) - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.response_end),0) AS avg_response_end - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL (SELECT pages.response_end - FROM pages - WHERE {" AND ".join(pg_sub_query_chart)}) AS pages ON(TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, params)) - response_end = cur.fetchall() - return helper.list_to_camel_case(__merge_charts(response_end, actions)) - - def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1790,92 +814,6 @@ def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(de return {**row_sessions, **row_errors, "chart": chart} -def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, - data=args, main_table="", time_column="timestamp", project=False, - duration=False) - pg_sub_query_subset.append("timestamp>=%(startTimestamp)s") - pg_sub_query_subset.append("timestamp<%(endTimestamp)s") - with pg_client.PostgresClient() as cur: - pg_query = f"""WITH resources AS (SELECT resources.type, timestamp, session_id - FROM events.resources - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - ), - pages AS (SELECT visually_complete, timestamp - FROM events.pages - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} AND pages.visually_complete > 0) - SELECT generated_timestamp AS timestamp, - COALESCE(jsonb_agg(resources_avg_count_by_type) - FILTER ( WHERE resources_avg_count_by_type IS NOT NULL ), '[]'::jsonb) AS types, - COALESCE(AVG(total_count), 0) AS avg_count_resources, - COALESCE(AVG(avg_time_to_render), 0) AS avg_time_to_render - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL (SELECT resources_count_by_session_by_type.type, - avg(resources_count_by_session_by_type.count) AS avg_count, - sum(resources_count_by_session_by_type.count) AS total_count - FROM (SELECT resources.type, COUNT(*) AS count - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - GROUP BY resources.session_id, resources.type) AS resources_count_by_session_by_type - GROUP BY resources_count_by_session_by_type.type) AS resources_avg_count_by_type ON (TRUE) - LEFT JOIN LATERAL (SELECT AVG(visually_complete) AS avg_time_to_render - FROM pages - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS time_to_render ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - rows = cur.fetchall() - for r in rows: - r["types"] = {t["type"]: t["avg_count"] for t in r["types"]} - - return helper.list_to_camel_case(rows) - - -def get_resources_count_by_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, - data=args, main_table="resources", time_column="timestamp", project=False, - duration=False) - - with pg_client.PostgresClient() as cur: - pg_query = f"""WITH resources AS (SELECT resources.type, timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - ) - SELECT generated_timestamp AS timestamp, - COALESCE(JSONB_AGG(t) FILTER (WHERE t IS NOT NULL), '[]'::JSONB) AS types - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL (SELECT resources.type, COUNT(*) AS count - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - GROUP BY resources.type - ) AS t ON(TRUE) - GROUP BY timestamp - ORDER BY timestamp;""" - cur.execute(cur.mogrify(pg_query, {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - rows = cur.fetchall() - for r in rows: - for t in r["types"]: - r[t["type"]] = t["count"] - r.pop("types") - rows = __merge_rows_with_neutral(rows, {k: 0 for k in RESOURCS_TYPE_TO_DB_TYPE.keys()}) - return rows - - def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1931,111 +869,6 @@ def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1) return rows -def __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("resources.duration > 0") - pg_sub_query.append("resources.type= %(type)s") - pg_query = f"""SELECT COALESCE(AVG(resources.duration),0) AS value - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - row = cur.fetchone() - return row - - -def get_application_activity_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with pg_client.PostgresClient() as cur: - row = __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - results = row - results["chart"] = get_performance_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - row = __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - previous = helper.dict_to_camel_case(row) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - -def get_performance_avg_image_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=19, **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) - img_constraints = [] - - img_constraints_vals = {} - - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, - chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, - chart=True, data=args, main_table="resources", time_column="timestamp", - duration=False) - pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") - - pg_query = f"""WITH resources AS (SELECT resources.duration, resources.timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - AND resources.type = 'img' AND resources.duration>0 - {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} - ) - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(resources.duration),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT resources.duration - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) - GROUP BY timestamp - ORDER BY timestamp;""" - cur.execute(cur.mogrify(pg_query, {**params, **img_constraints_vals, **__get_constraint_values(args)})) - rows = cur.fetchall() - rows = helper.list_to_camel_case(rows) - - return rows - - -def __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("pages.timestamp >= %(startTimestamp)s") - pg_sub_query.append("pages.timestamp > %(endTimestamp)s") - pg_sub_query.append("pages.load_time > 0") - pg_sub_query.append("pages.load_time IS NOT NULL") - pg_query = f"""SELECT COALESCE(AVG(pages.load_time) ,0) AS value - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, - **__get_constraint_values(args)} - - cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - helper.__time_value(row) - return row - - -def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with pg_client.PostgresClient() as cur: - row = __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - results = row - results["chart"] = get_performance_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - row = __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - previous = helper.dict_to_camel_case(row) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - def get_performance_avg_page_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, **args): @@ -2071,218 +904,6 @@ def get_performance_avg_page_load_time(cur, project_id, startTimestamp=TimeUTC.n return rows -def __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("resources.duration > 0") - pg_sub_query.append("resources.type= %(type)s") - pg_query = f"""SELECT COALESCE(AVG(resources.duration),0) AS value - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)};""" - - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - - row = cur.fetchone() - helper.__time_value(row) - return row - - -def get_application_activity_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with pg_client.PostgresClient() as cur: - row = __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - results = row - results["chart"] = get_performance_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - row = __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - previous = helper.dict_to_camel_case(row) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - -def get_performance_avg_request_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=19, **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) - request_constraints = [] - request_constraints_vals = {} - - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, - chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, - chart=True, data=args, main_table="resources", time_column="timestamp", - duration=False) - pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") - - pg_query = f"""WITH resources AS(SELECT resources.duration, resources.timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - AND resources.type = 'fetch' AND resources.duration>0 - {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} - ) - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(resources.duration),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT resources.duration - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {**params, **request_constraints_vals, **__get_constraint_values(args)})) - rows = cur.fetchall() - - return rows - - -def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with pg_client.PostgresClient() as cur: - row = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args) - results = helper.dict_to_camel_case(row) - results["chart"] = __get_page_metrics_avg_dom_content_load_start_chart(cur, project_id, startTimestamp, - endTimestamp, **args) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - row = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args) - previous = helper.dict_to_camel_case(row) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - -def __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("pages.timestamp>=%(startTimestamp)s") - pg_sub_query.append("pages.timestamp<%(endTimestamp)s") - pg_sub_query.append("pages.dom_content_loaded_time > 0") - pg_query = f"""SELECT COALESCE(AVG(pages.dom_content_loaded_time), 0) AS value - FROM (SELECT pages.dom_content_loaded_time - FROM events.pages - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - ) AS pages;""" - params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, - **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - return row - - -def __get_page_metrics_avg_dom_content_load_start_chart(cur, project_id, startTimestamp, endTimestamp, density=19, - **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, - chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, - chart=True, data=args, main_table="pages", time_column="timestamp", - duration=False) - pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s") - pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s") - pg_sub_query_subset.append("pages.dom_content_loaded_time > 0") - - pg_query = f"""WITH pages AS(SELECT pages.dom_content_loaded_time, pages.timestamp - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - ) - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.dom_content_loaded_time),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT pages.dom_content_loaded_time - FROM pages - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {**params, **__get_constraint_values(args)})) - rows = cur.fetchall() - return rows - - -def get_page_metrics_avg_first_contentful_pixel(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with pg_client.PostgresClient() as cur: - rows = __get_page_metrics_avg_first_contentful_pixel(cur, project_id, startTimestamp, endTimestamp, **args) - if len(rows) > 0: - results = helper.dict_to_camel_case(rows[0]) - results["chart"] = __get_page_metrics_avg_first_contentful_pixel_chart(cur, project_id, startTimestamp, - endTimestamp, **args) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - rows = __get_page_metrics_avg_first_contentful_pixel(cur, project_id, startTimestamp, endTimestamp, **args) - if len(rows) > 0: - previous = helper.dict_to_camel_case(rows[0]) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - -def __get_page_metrics_avg_first_contentful_pixel(cur, project_id, startTimestamp, endTimestamp, **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("pages.timestamp>=%(startTimestamp)s") - pg_sub_query.append("pages.timestamp<%(endTimestamp)s") - pg_sub_query.append("pages.first_contentful_paint_time > 0") - pg_query = f"""SELECT COALESCE(AVG(pages.first_contentful_paint_time), 0) AS value - FROM (SELECT pages.first_contentful_paint_time - FROM events.pages - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - ) AS pages;""" - params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, - **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - return rows - - -def __get_page_metrics_avg_first_contentful_pixel_chart(cur, project_id, startTimestamp, endTimestamp, density=20, - **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, - chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, - chart=True, data=args, main_table="pages", time_column="timestamp", - duration=False) - pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s") - pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s") - pg_sub_query_subset.append("pages.first_contentful_paint_time > 0") - - pg_query = f"""WITH pages AS(SELECT pages.first_contentful_paint_time, pages.timestamp - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - ) - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.first_contentful_paint_time),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT pages.first_contentful_paint_time - FROM pages - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {**params, **__get_constraint_values(args)})) - rows = cur.fetchall() - return rows - - def get_user_activity_avg_visited_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: @@ -2346,279 +967,6 @@ def __get_user_activity_avg_visited_pages_chart(cur, project_id, startTimestamp, return rows -def get_user_activity_avg_session_duration(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with pg_client.PostgresClient() as cur: - row = __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args) - results = helper.dict_to_camel_case(row) - results["chart"] = __get_user_activity_avg_session_duration_chart(cur, project_id, startTimestamp, - endTimestamp, **args) - - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - row = __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args) - - previous = helper.dict_to_camel_case(row) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - -def __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args): - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query.append("sessions.duration IS NOT NULL") - pg_sub_query.append("sessions.duration > 0") - pg_query = f"""SELECT COALESCE(AVG(sessions.duration),0) AS value - FROM public.sessions - WHERE {" AND ".join(pg_sub_query)};""" - params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, - **__get_constraint_values(args)} - - cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - return row - - -def __get_user_activity_avg_session_duration_chart(cur, project_id, startTimestamp, endTimestamp, density=20, **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - pg_sub_query_subset = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, - chart=True, data=args, main_table="sessions", time_column="start_ts", - duration=False) - pg_sub_query_subset.append("sessions.duration IS NOT NULL") - pg_sub_query_subset.append("sessions.duration > 0") - - pg_query = f"""WITH sessions AS(SELECT sessions.duration, sessions.start_ts - FROM public.sessions - WHERE {" AND ".join(pg_sub_query_subset)} - ) - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(sessions.duration),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT sessions.duration - FROM sessions - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS sessions ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {**params, **__get_constraint_values(args)})) - rows = cur.fetchall() - return rows - - -def get_top_metrics_avg_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=20, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) - - if value is not None: - pg_sub_query.append("pages.path = %(value)s") - pg_sub_query_chart.append("pages.path = %(value)s") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT COALESCE(AVG(pages.response_time), 0) AS value - FROM events.pages - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - AND pages.timestamp >= %(startTimestamp)s - AND pages.timestamp < %(endTimestamp)s - AND pages.response_time > 0;""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.response_time),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT response_time - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} AND pages.response_time > 0 - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp ASC;""" - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - row["chart"] = helper.list_to_camel_case(rows) - helper.__time_value(row) - return helper.dict_to_camel_case(row) - - -def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=20, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) - - if value is not None: - pg_sub_query.append("pages.path = %(value)s") - pg_sub_query_chart.append("pages.path = %(value)s") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT COALESCE(AVG(pages.first_paint_time), 0) AS value - FROM events.pages - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - AND pages.timestamp >= %(startTimestamp)s - AND pages.timestamp < %(endTimestamp)s - AND pages.first_paint_time > 0;""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.first_paint_time),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT first_paint_time - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} AND pages.first_paint_time > 0 - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp ASC;""" - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - row["chart"] = helper.list_to_camel_case(rows) - helper.__time_value(row) - return helper.dict_to_camel_case(row) - - -def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=19, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) - pg_sub_query.append("pages.dom_content_loaded_time>0") - pg_sub_query_chart.append("pages.dom_content_loaded_time>0") - if value is not None: - pg_sub_query.append("pages.path = %(value)s") - pg_sub_query_chart.append("pages.path = %(value)s") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT COALESCE(AVG(pages.dom_content_loaded_time), 0) AS value - FROM events.pages - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - AND pages.timestamp >= %(startTimestamp)s - AND pages.timestamp < %(endTimestamp)s - AND pages.dom_content_loaded_time > 0;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.dom_content_loaded_time),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT dom_content_loaded_time - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp ASC;""" - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - row["chart"] = helper.list_to_camel_case(rows) - helper.__time_value(row) - return helper.dict_to_camel_case(row) - - -def get_top_metrics_avg_till_first_bit(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=20, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) - - if value is not None: - pg_sub_query.append("pages.path = %(value)s") - pg_sub_query_chart.append("pages.path = %(value)s") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT COALESCE(AVG(pages.ttfb), 0) AS value - FROM events.pages - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - AND pages.timestamp >= %(startTimestamp)s - AND pages.timestamp < %(endTimestamp)s - AND pages.ttfb > 0;""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.ttfb),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT ttfb - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} AND pages.ttfb > 0 - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp ASC;""" - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - row["chart"] = helper.list_to_camel_case(rows) - helper.__time_value(row) - return helper.dict_to_camel_case(row) - - -def get_top_metrics_avg_time_to_interactive(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=20, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) - pg_sub_query = __get_constraints(project_id=project_id, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, - chart=True, data=args) - - pg_sub_query.append("pages.time_to_interactive > 0") - pg_sub_query_chart.append("pages.time_to_interactive > 0") - if value is not None: - pg_sub_query.append("pages.path = %(value)s") - pg_sub_query_chart.append("pages.path = %(value)s") - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT COALESCE(AVG(pages.time_to_interactive), 0) AS value - FROM events.pages - INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - AND pages.timestamp >= %(startTimestamp)s - AND pages.timestamp < %(endTimestamp)s;""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)} - cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.time_to_interactive),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT time_to_interactive - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp ASC;""" - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - row["chart"] = helper.list_to_camel_case(rows) - helper.__time_value(row) - return helper.dict_to_camel_case(row) - - def get_top_metrics_count_requests(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), value=None, density=20, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) diff --git a/api/chalicelib/core/resources.py b/api/chalicelib/core/resources.py deleted file mode 100644 index e56c0cf74..000000000 --- a/api/chalicelib/core/resources.py +++ /dev/null @@ -1,32 +0,0 @@ -from chalicelib.utils import helper, pg_client -from decouple import config - - -def get_by_session_id(session_id, project_id, start_ts, duration): - with pg_client.PostgresClient() as cur: - if duration is None or (type(duration) != 'int' and type(duration) != 'float') or duration < 0: - duration = 0 - delta = config("events_ts_delta", cast=int, default=60 * 60) * 1000 - ch_query = """\ - SELECT - timestamp AS datetime, - url, - type, - resources.duration AS duration, - ttfb, - header_size, - encoded_body_size, - decoded_body_size, - success, - COALESCE(CASE WHEN status=0 THEN NULL ELSE status END, CASE WHEN success THEN 200 END) AS status - FROM events.resources INNER JOIN sessions USING (session_id) - WHERE session_id = %(session_id)s - AND project_id= %(project_id)s - AND sessions.start_ts=%(start_ts)s - AND resources.timestamp>=%(res_start_ts)s - AND resources.timestamp<=%(res_end_ts)s;""" - params = {"session_id": session_id, "project_id": project_id, "start_ts": start_ts, "duration": duration, - "res_start_ts": start_ts - delta, "res_end_ts": start_ts + duration + delta, } - cur.execute(cur.mogrify(ch_query, params)) - rows = cur.fetchall() - return helper.list_to_camel_case(rows) diff --git a/api/chalicelib/core/sessions_replay.py b/api/chalicelib/core/sessions_replay.py index 8011a8643..9512a0fa3 100644 --- a/api/chalicelib/core/sessions_replay.py +++ b/api/chalicelib/core/sessions_replay.py @@ -1,6 +1,6 @@ import schemas from chalicelib.core import events, metadata, events_mobile, \ - sessions_mobs, issues, resources, assist, sessions_devtool, canvas, user_testing + sessions_mobs, issues, assist, sessions_devtool, canvas, user_testing from chalicelib.utils import errors_helper from chalicelib.utils import pg_client, helper @@ -121,8 +121,6 @@ def get_events(project_id, session_id): if e['source'] == "js_exception"][:500] data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, session_id=session_id) - data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, - start_ts=s_data["startTs"], duration=s_data["duration"]) data['userTesting'] = user_testing.get_test_signals(session_id=session_id, project_id=project_id) data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) diff --git a/api/schemas/schemas.py b/api/schemas/schemas.py index 775303429..8215dabf2 100644 --- a/api/schemas/schemas.py +++ b/api/schemas/schemas.py @@ -393,14 +393,6 @@ class AlertColumn(str, Enum): PERFORMANCE__PAGE_RESPONSE_TIME__AVERAGE = "performance.page_response_time.average" PERFORMANCE__TTFB__AVERAGE = "performance.ttfb.average" PERFORMANCE__TIME_TO_RENDER__AVERAGE = "performance.time_to_render.average" - PERFORMANCE__IMAGE_LOAD_TIME__AVERAGE = "performance.image_load_time.average" - PERFORMANCE__REQUEST_LOAD_TIME__AVERAGE = "performance.request_load_time.average" - RESOURCES__LOAD_TIME__AVERAGE = "resources.load_time.average" - RESOURCES__MISSING__COUNT = "resources.missing.count" - ERRORS__4XX_5XX__COUNT = "errors.4xx_5xx.count" - ERRORS__4XX__COUNT = "errors.4xx.count" - ERRORS__5XX__COUNT = "errors.5xx.count" - ERRORS__JAVASCRIPT__IMPACTED_SESSIONS__COUNT = "errors.javascript.impacted_sessions.count" PERFORMANCE__CRASHES__COUNT = "performance.crashes.count" ERRORS__JAVASCRIPT__COUNT = "errors.javascript.count" ERRORS__BACKEND__COUNT = "errors.backend.count" @@ -945,7 +937,6 @@ class MetricType(str, Enum): class MetricOfErrors(str, Enum): - CALLS_ERRORS = "callsErrors" DOMAINS_ERRORS_4XX = "domainsErrors4xx" DOMAINS_ERRORS_5XX = "domainsErrors5xx" ERRORS_PER_DOMAINS = "errorsPerDomains" @@ -954,47 +945,8 @@ class MetricOfErrors(str, Enum): RESOURCES_BY_PARTY = "resourcesByParty" -class MetricOfPerformance(str, Enum): - CPU = "cpu" - CRASHES = "crashes" - FPS = "fps" - IMPACTED_SESSIONS_BY_SLOW_PAGES = "impactedSessionsBySlowPages" - MEMORY_CONSUMPTION = "memoryConsumption" - PAGES_DOM_BUILDTIME = "pagesDomBuildtime" - PAGES_RESPONSE_TIME = "pagesResponseTime" - PAGES_RESPONSE_TIME_DISTRIBUTION = "pagesResponseTimeDistribution" - RESOURCES_VS_VISUALLY_COMPLETE = "resourcesVsVisuallyComplete" - SESSIONS_PER_BROWSER = "sessionsPerBrowser" - SLOWEST_DOMAINS = "slowestDomains" - SPEED_LOCATION = "speedLocation" - TIME_TO_RENDER = "timeToRender" - - -class MetricOfResources(str, Enum): - MISSING_RESOURCES = "missingResources" - RESOURCES_COUNT_BY_TYPE = "resourcesCountByType" - RESOURCES_LOADING_TIME = "resourcesLoadingTime" - RESOURCE_TYPE_VS_RESPONSE_END = "resourceTypeVsResponseEnd" - SLOWEST_RESOURCES = "slowestResources" - - class MetricOfWebVitals(str, Enum): - AVG_CPU = "avgCpu" - AVG_DOM_CONTENT_LOADED = "avgDomContentLoaded" - AVG_DOM_CONTENT_LOAD_START = "avgDomContentLoadStart" - AVG_FIRST_CONTENTFUL_PIXEL = "avgFirstContentfulPixel" - AVG_FIRST_PAINT = "avgFirstPaint" - AVG_FPS = "avgFps" - AVG_IMAGE_LOAD_TIME = "avgImageLoadTime" - AVG_PAGE_LOAD_TIME = "avgPageLoadTime" - AVG_PAGES_DOM_BUILDTIME = "avgPagesDomBuildtime" - AVG_PAGES_RESPONSE_TIME = "avgPagesResponseTime" - AVG_REQUEST_LOAD_TIME = "avgRequestLoadTime" - AVG_RESPONSE_TIME = "avgResponseTime" AVG_SESSION_DURATION = "avgSessionDuration" - AVG_TILL_FIRST_BYTE = "avgTillFirstByte" - AVG_TIME_TO_INTERACTIVE = "avgTimeToInteractive" - AVG_TIME_TO_RENDER = "avgTimeToRender" AVG_USED_JS_HEAP_SIZE = "avgUsedJsHeapSize" AVG_VISITED_PAGES = "avgVisitedPages" COUNT_REQUESTS = "countRequests" @@ -1225,43 +1177,9 @@ class CardErrors(__CardSchema): return self -class CardPerformance(__CardSchema): - metric_type: Literal[MetricType.PERFORMANCE] - metric_of: MetricOfPerformance = Field(default=MetricOfPerformance.CPU) - view_type: MetricOtherViewType = Field(...) - - @model_validator(mode="before") - @classmethod - def __enforce_default(cls, values): - values["series"] = [] - return values - - @model_validator(mode="after") - def __transform(self): - self.metric_of = MetricOfPerformance(self.metric_of) - return self - - -class CardResources(__CardSchema): - metric_type: Literal[MetricType.RESOURCES] - metric_of: MetricOfResources = Field(default=MetricOfResources.MISSING_RESOURCES) - view_type: MetricOtherViewType = Field(...) - - @model_validator(mode="before") - @classmethod - def __enforce_default(cls, values): - values["series"] = [] - return values - - @model_validator(mode="after") - def __transform(self): - self.metric_of = MetricOfResources(self.metric_of) - return self - - class CardWebVital(__CardSchema): metric_type: Literal[MetricType.WEB_VITAL] - metric_of: MetricOfWebVitals = Field(default=MetricOfWebVitals.AVG_CPU) + metric_of: MetricOfWebVitals = Field(default=MetricOfWebVitals.AVG_VISITED_PAGES) view_type: MetricOtherViewType = Field(...) @model_validator(mode="before") @@ -1390,7 +1308,7 @@ class CardPathAnalysis(__CardSchema): # Union of cards-schemas that doesn't change between FOSS and EE __cards_union_base = Union[ CardTimeSeries, CardTable, CardFunnel, - CardErrors, CardPerformance, CardResources, + CardErrors, CardWebVital, CardHeatMap, CardPathAnalysis] CardSchema = ORUnion(Union[__cards_union_base, CardInsights], discriminator='metric_type') diff --git a/ee/api/chalicelib/core/__init__.py b/ee/api/chalicelib/core/__init__.py index 3d11f704d..88c8528a7 100644 --- a/ee/api/chalicelib/core/__init__.py +++ b/ee/api/chalicelib/core/__init__.py @@ -44,6 +44,3 @@ if config("EXP_FUNNELS", cast=bool, default=False): from . import significance_exp as significance else: from . import significance as significance - -if config("EXP_RESOURCES", cast=bool, default=False): - logging.info(">>> Using experimental resources for session-replay") diff --git a/ee/api/chalicelib/core/alerts_processor.py b/ee/api/chalicelib/core/alerts_processor.py index 915766b9f..629a18a37 100644 --- a/ee/api/chalicelib/core/alerts_processor.py +++ b/ee/api/chalicelib/core/alerts_processor.py @@ -40,30 +40,6 @@ LeftToDb = { schemas.AlertColumn.PERFORMANCE__TIME_TO_RENDER__AVERAGE: { "table": "events.pages INNER JOIN public.sessions USING(session_id)", "formula": "AVG(NULLIF(visually_complete,0))"}, - schemas.AlertColumn.PERFORMANCE__IMAGE_LOAD_TIME__AVERAGE: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='img'"}, - schemas.AlertColumn.PERFORMANCE__REQUEST_LOAD_TIME__AVERAGE: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='fetch'"}, - schemas.AlertColumn.RESOURCES__LOAD_TIME__AVERAGE: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(resources.duration,0))"}, - schemas.AlertColumn.RESOURCES__MISSING__COUNT: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(DISTINCT url_hostpath)", "condition": "success= FALSE AND type='img'"}, - schemas.AlertColumn.ERRORS__4XX_5XX__COUNT: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(session_id)", - "condition": "status/100!=2"}, - schemas.AlertColumn.ERRORS__4XX__COUNT: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=4"}, - schemas.AlertColumn.ERRORS__5XX__COUNT: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=5"}, - schemas.AlertColumn.ERRORS__JAVASCRIPT__IMPACTED_SESSIONS__COUNT: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(DISTINCT session_id)", "condition": "success= FALSE AND type='script'"}, schemas.AlertColumn.PERFORMANCE__CRASHES__COUNT: { "table": "public.sessions", "formula": "COUNT(DISTINCT session_id)", diff --git a/ee/api/chalicelib/core/alerts_processor_exp.py b/ee/api/chalicelib/core/alerts_processor_exp.py index 76c8dc4b5..13e047206 100644 --- a/ee/api/chalicelib/core/alerts_processor_exp.py +++ b/ee/api/chalicelib/core/alerts_processor_exp.py @@ -52,49 +52,6 @@ LeftToDb = { "formula": "AVG(NULLIF(visually_complete,0))", "eventType": "LOCATION" }, - schemas.AlertColumn.PERFORMANCE__IMAGE_LOAD_TIME__AVERAGE: { - "table": lambda timestamp: f"{exp_ch_helper.get_main_resources_table(timestamp)} AS resources", - "formula": "AVG(NULLIF(resources.duration,0))", - "condition": "type='img'" - }, - schemas.AlertColumn.PERFORMANCE__REQUEST_LOAD_TIME__AVERAGE: { - "table": lambda timestamp: f"{exp_ch_helper.get_main_resources_table(timestamp)} AS resources", - "formula": "AVG(NULLIF(resources.duration,0))", - "condition": "type='fetch'" - }, - schemas.AlertColumn.RESOURCES__LOAD_TIME__AVERAGE: { - "table": lambda timestamp: f"{exp_ch_helper.get_main_resources_table(timestamp)} AS resources", - "formula": "AVG(NULLIF(resources.duration,0))" - }, - schemas.AlertColumn.RESOURCES__MISSING__COUNT: { - "table": lambda timestamp: f"{exp_ch_helper.get_main_resources_table(timestamp)} AS resources", - "formula": "COUNT(DISTINCT url_hostpath)", - "condition": "success= FALSE AND type='img'" - }, - schemas.AlertColumn.ERRORS__4XX_5XX__COUNT: { - "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS requests", - "eventType": "REQUEST", - "formula": "COUNT(1)", - "condition": "intDiv(requests.status, 100)!=2" - }, - schemas.AlertColumn.ERRORS__4XX__COUNT: { - "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS requests", - "eventType": "REQUEST", - "formula": "COUNT(1)", - "condition": "intDiv(requests.status, 100)==4" - }, - schemas.AlertColumn.ERRORS__5XX__COUNT: { - "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS requests", - "eventType": "REQUEST", - "formula": "COUNT(1)", - "condition": "intDiv(requests.status, 100)==5" - }, - schemas.AlertColumn.ERRORS__JAVASCRIPT__IMPACTED_SESSIONS__COUNT: { - "table": lambda timestamp: f"{exp_ch_helper.get_main_events_table(timestamp)} AS errors", - "eventType": "ERROR", - "formula": "COUNT(DISTINCT session_id)", - "condition": "source='js_exception'" - }, schemas.AlertColumn.PERFORMANCE__CRASHES__COUNT: { "table": lambda timestamp: f"{exp_ch_helper.get_main_sessions_table(timestamp)} AS sessions", "formula": "COUNT(DISTINCT session_id)", diff --git a/ee/api/chalicelib/core/metrics.py b/ee/api/chalicelib/core/metrics.py index 4bb5fe984..fc0a96580 100644 --- a/ee/api/chalicelib/core/metrics.py +++ b/ee/api/chalicelib/core/metrics.py @@ -422,207 +422,6 @@ def __get_resource_db_type_from_type(resource_type): return {v: k for k, v in RESOURCS_TYPE_TO_DB_TYPE.items()}.get(resource_type, resource_type) -def search(text, resource_type, project_id, performance=False, pages_only=False, events_only=False, - metadata=False, key=None, platform=None): - if text.startswith("^"): - text = text[1:] - if not resource_type: - data = [] - if metadata: - resource_type = "METADATA" - elif pages_only or performance: - resource_type = "LOCATION" - else: - resource_type = "ALL" - data.extend(search(text=text, resource_type=resource_type, project_id=project_id, - performance=performance, pages_only=pages_only, events_only=events_only, key=key, - platform=platform)) - return data - - ch_sub_query = __get_basic_constraints(time_constraint=False, - data={} if platform is None else {"platform": platform}) - - if resource_type == "ALL" and not pages_only and not events_only: - ch_sub_query.append("positionUTF8(url_path,%(value)s)!=0") - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT arrayJoin(arraySlice(arrayReverseSort(arrayDistinct(groupArray(url_path))), 1, 5)) AS value, - type AS key - FROM {exp_ch_helper.get_main_resources_table(0)} AS resources - WHERE {" AND ".join(ch_sub_query)} - GROUP BY type - ORDER BY type ASC;""" - # print(ch.format(query=ch_query, - # params={"project_id": project_id, - # "value": text})) - rows = ch.execute(query=ch_query, - params={"project_id": project_id, - "value": text}) - rows = [{"value": i["value"], "type": __get_resource_type_from_db_type(i["key"])} for i in rows] - elif resource_type == "ALL" and events_only: - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT DISTINCT value AS value, type AS key - FROM {exp_ch_helper.get_autocomplete_table(0)} autocomplete - WHERE {" AND ".join(ch_sub_query)} - AND positionUTF8(lowerUTF8(value), %(value)s) != 0 - AND type IN ('LOCATION','INPUT','CLICK') - ORDER BY type, value - LIMIT 10 BY type;""" - rows = ch.execute(query=ch_query, - params={"project_id": project_id, - "value": text.lower(), - "platform_0": platform}) - rows = [{"value": i["value"], "type": i["key"]} for i in rows] - elif resource_type in ['IMG', 'REQUEST', 'STYLESHEET', 'OTHER', 'SCRIPT'] and not pages_only: - ch_sub_query.append("positionUTF8(url_path,%(value)s)!=0") - ch_sub_query.append(f"resources.type = '{__get_resource_db_type_from_type(resource_type)}'") - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT DISTINCT url_path AS value, - %(resource_type)s AS key - FROM {exp_ch_helper.get_main_resources_table(0)} AS resources - WHERE {" AND ".join(ch_sub_query)} - LIMIT 10;""" - rows = ch.execute(query=ch_query, - params={"project_id": project_id, - "value": text, - "resource_type": resource_type, - "platform_0": platform}) - rows = [{"value": i["value"], "type": i["key"]} for i in rows] - elif resource_type == 'LOCATION': - with ch_client.ClickHouseClient() as ch: - ch_sub_query.append("type='LOCATION'") - ch_sub_query.append("positionUTF8(value,%(value)s)!=0") - ch_query = f"""SELECT - DISTINCT value AS value, - 'LOCATION' AS key - FROM {exp_ch_helper.get_autocomplete_table(0)} AS autocomplete - WHERE {" AND ".join(ch_sub_query)} - LIMIT 10;""" - rows = ch.execute(query=ch_query, - params={"project_id": project_id, - "value": text, - "platform_0": platform}) - rows = [{"value": i["value"], "type": i["key"]} for i in rows] - elif resource_type == "INPUT": - with ch_client.ClickHouseClient() as ch: - ch_sub_query.append("positionUTF8(lowerUTF8(value), %(value)s) != 0") - ch_sub_query.append("type='INPUT") - ch_query = f"""SELECT DISTINCT label AS value, 'INPUT' AS key - FROM {exp_ch_helper.get_autocomplete_table(0)} AS autocomplete - WHERE {" AND ".join(ch_sub_query)} - LIMIT 10;""" - rows = ch.execute(query=ch_query, - params={"project_id": project_id, - "value": text.lower(), - "platform_0": platform}) - rows = [{"value": i["value"], "type": i["key"]} for i in rows] - elif resource_type == "CLICK": - with ch_client.ClickHouseClient() as ch: - ch_sub_query.append("positionUTF8(lowerUTF8(value), %(value)s) != 0") - ch_sub_query.append("type='CLICK'") - ch_query = f"""SELECT DISTINCT value AS value, 'CLICK' AS key - FROM {exp_ch_helper.get_autocomplete_table(0)} AS autocomplete - WHERE {" AND ".join(ch_sub_query)} - LIMIT 10;""" - rows = ch.execute(query=ch_query, - params={"project_id": project_id, - "value": text.lower(), - "platform_0": platform}) - rows = [{"value": i["value"], "type": i["key"]} for i in rows] - elif resource_type == "METADATA": - if key and len(key) > 0 and key in {**METADATA_FIELDS, **SESSIONS_META_FIELDS}.keys(): - if key in METADATA_FIELDS.keys(): - ch_sub_query.append( - f"positionCaseInsensitiveUTF8(sessions.{METADATA_FIELDS[key]},%(value)s)!=0") - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT DISTINCT sessions.{METADATA_FIELDS[key]} AS value, - %(key)s AS key - FROM {exp_ch_helper.get_main_sessions_table(0)} AS sessions - WHERE {" AND ".join(ch_sub_query)} - LIMIT 10;""" - rows = ch.execute(query=ch_query, - params={"project_id": project_id, "value": text, "key": key, - "platform_0": platform}) - else: - ch_sub_query.append(f"positionCaseInsensitiveUTF8(sessions.{SESSIONS_META_FIELDS[key]},%(value)s)>0") - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT DISTINCT sessions.{SESSIONS_META_FIELDS[key]} AS value, - '{key}' AS key - FROM {exp_ch_helper.get_main_sessions_table(0)} AS sessions - WHERE {" AND ".join(ch_sub_query)} - LIMIT 10;""" - rows = ch.execute(query=ch_query, params={"project_id": project_id, "value": text, "key": key, - "platform_0": platform}) - else: - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT DISTINCT value AS value, - type AS key - FROM {exp_ch_helper.get_autocomplete_table(0)} AS autocomplete - WHERE project_id = toUInt16(2460) - AND positionCaseInsensitiveUTF8(value, %(value)s) != 0 - LIMIT 10 BY type""" - - # print(ch.format(query=ch_query, params={"project_id": project_id, "value": text, "key": key, - # "platform_0": platform})) - rows = ch.execute(query=ch_query, params={"project_id": project_id, "value": text, "key": key, - "platform_0": platform}) - else: - return [] - return [helper.dict_to_camel_case(row) for row in rows] - - -def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query = __get_basic_constraints(table_name="resources", data=args) - ch_sub_query.append("resources.success = 0") - ch_sub_query.append("resources.type = 'img'") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT resources.url_path AS key, - COUNT(1) AS doc_count - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query)} - GROUP BY url_path - ORDER BY doc_count DESC - LIMIT 10;""" - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - # print(ch.format(query=ch_query, params=params)) - rows = ch.execute(query=ch_query, params=params) - - rows = [{"url": i["key"], "sessions": i["doc_count"]} for i in rows] - if len(rows) == 0: - return [] - ch_sub_query.append("resources.url_path = %(value)s") - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COUNT(1) AS doc_count, - toUnixTimestamp(MAX(resources.datetime))*1000 AS max_datatime - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query)} - GROUP BY timestamp - ORDER BY timestamp;""" - for e in rows: - e["startedAt"] = startTimestamp - e["startTimestamp"] = startTimestamp - e["endTimestamp"] = endTimestamp - params["value"] = e["url"] - r = ch.execute(query=ch_query, params=params) - - e["endedAt"] = r[-1]["max_datatime"] - e["chart"] = [{"timestamp": i["timestamp"], "count": i["doc_count"]} for i in - __complete_missing_steps(rows=r, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"doc_count": 0})] - return rows - - KEYS = { 'startTimestamp': args_transformer.int_arg, 'endTimestamp': args_transformer.int_arg, @@ -641,146 +440,6 @@ def dashboard_args(params): return args -def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=19, type=None, url=None, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - if type is not None: - ch_sub_query_chart.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'") - if url is not None: - ch_sub_query_chart.append(f"resources.url_hostpath = %(value)s") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - ch_sub_query_chart.append("resources.duration>0") - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(resources.duration),0) AS avg - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": url, "type": type, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - ch_query = f"""SELECT COALESCE(avgOrNull(resources.duration),0) AS avg - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query_chart)};""" - avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avg": avg, "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"avg": 0})} - - -def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=19, url=None, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.event_type='LOCATION'") - if url is not None: - ch_sub_query_chart.append(f"pages.url_path = %(value)s") - ch_sub_query_chart.append("isNotNull(pages.dom_building_time)") - ch_sub_query_chart.append("pages.dom_building_time>0") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(pages.dom_building_time),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": url, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - ch_query = f"""SELECT COALESCE(avgOrNull(pages.dom_building_time),0) AS avg - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)};""" - avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - - results = {"value": avg, - "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"value": 0})} - helper.__time_value(results) - return results - - -def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), type="all", density=19, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query = __get_basic_constraints(table_name="resources", data=args) - ch_sub_query.append("isNotNull(resources.name)") - ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - ch_sub_query_chart += meta_condition - - if type is not None and type.upper() != "ALL": - sq = f"resources.type = '{__get_resource_db_type_from_type(type.upper())}'" - else: - sq = "resources.type != 'fetch'" - ch_sub_query.append(sq) - ch_sub_query_chart.append(sq) - ch_sub_query_chart.append("isNotNull(resources.duration)") - ch_sub_query_chart.append("resources.duration>0") - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT any(url_hostpath) AS url, any(type) AS type, name, - COALESCE(avgOrNull(NULLIF(resources.duration,0)),0) AS avg - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query)} - GROUP BY name - ORDER BY avg DESC - LIMIT 10;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - # print(ch.format(query=ch_query, params=params)) - rows = ch.execute(query=ch_query, params=params) - if len(rows) == 0: - return [] - ch_sub_query.append(ch_sub_query_chart[-1]) - results = [] - names = [r["name"] for r in rows] - ch_query = f"""SELECT name, - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(resources.duration),0) AS avg - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query_chart)} - AND name IN %(names)s - GROUP BY name,timestamp - ORDER BY name,timestamp;""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "names": names, **__get_constraint_values(args)} - # print(ch.format(query=ch_query, params=params)) - charts = ch.execute(query=ch_query, params=params) - for r in rows: - sub_chart = [] - for c in charts: - if c["name"] == r["name"]: - cc = dict(c) - cc.pop("name") - sub_chart.append(cc) - elif len(sub_chart) > 0: - break - r["chart"] = __complete_missing_steps(rows=sub_chart, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"avg": 0}) - r["type"] = __get_resource_type_from_db_type(r["type"]) - results.append(r) - - return results - - def get_sessions_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): ch_sub_query = __get_basic_constraints(table_name="sessions", data=args) @@ -800,210 +459,6 @@ def get_sessions_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return {"count": sum(i["count"] for i in rows), "chart": helper.list_to_camel_case(rows)} -def get_speed_index_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - ch_sub_query.append("isNotNull(pages.speed_index)") - ch_sub_query.append("pages.speed_index>0") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT sessions.user_country, COALESCE(avgOrNull(pages.speed_index),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - INNER JOIN {exp_ch_helper.get_main_sessions_table(startTimestamp)} AS sessions USING (session_id) - WHERE {" AND ".join(ch_sub_query)} - GROUP BY sessions.user_country - ORDER BY value ,sessions.user_country;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - # print(ch.format(query=ch_query, params=params)) - rows = ch.execute(query=ch_query, params=params) - ch_query = f"""SELECT COALESCE(avgOrNull(pages.speed_index),0) AS avg - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)};""" - avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"value": avg, "chart": helper.list_to_camel_case(rows), "unit": schemas.TemplatePredefinedUnits.MILLISECOND} - - -def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, url=None, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.event_type='LOCATION'") - ch_sub_query_chart.append("isNotNull(pages.response_time)") - ch_sub_query_chart.append("pages.response_time>0") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - if url is not None: - ch_sub_query_chart.append(f"url_path = %(value)s") - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COALESCE(avgOrNull(pages.response_time),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": url, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - ch_query = f"""SELECT COALESCE(avgOrNull(pages.response_time),0) AS avg - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)};""" - avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - results = {"value": avg, - "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"value": 0})} - helper.__time_value(results) - return results - - -def get_pages_response_time_distribution(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=20, **args): - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - ch_sub_query.append("isNotNull(pages.response_time)") - ch_sub_query.append("pages.response_time>0") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT pages.response_time AS response_time, - COUNT(1) AS count - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)} - GROUP BY response_time - ORDER BY response_time;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - ch_query = f"""SELECT COALESCE(avgOrNull(pages.response_time),0) AS avg - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)};""" - avg = ch.execute(query=ch_query, params=params)[0]["avg"] - quantiles_keys = [50, 90, 95, 99] - ch_query = f"""SELECT quantilesExact({",".join([str(i / 100) for i in quantiles_keys])})(pages.response_time) AS values - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)};""" - quantiles = ch.execute(query=ch_query, params=params) - result = { - "value": avg, - "total": sum(r["count"] for r in rows), - "chart": [], - "percentiles": [{ - "percentile": v, - "responseTime": ( - quantiles[0]["values"][i] if quantiles[0]["values"][i] is not None and not math.isnan( - quantiles[0]["values"][i]) else 0)} for i, v in enumerate(quantiles_keys) - ], - "extremeValues": [{"count": 0}], - "unit": schemas.TemplatePredefinedUnits.MILLISECOND - } - if len(rows) > 0: - rows = helper.list_to_camel_case(rows) - _99 = result["percentiles"][-1]["responseTime"] - extreme_values_first_index = -1 - for i, r in enumerate(rows): - if r["responseTime"] > _99: - extreme_values_first_index = i - break - - if extreme_values_first_index >= 0: - extreme_values_first_index += 1 - result["extremeValues"][0]["count"] = sum(r["count"] for r in rows[extreme_values_first_index:]) - # result["extremeValues"][0]["responseTime"] = rows[extreme_values_first_index]["responseTime"] - - rows = rows[:extreme_values_first_index] - - # ------- Merge points to reduce chart length till density - if density < len(quantiles_keys): - density = len(quantiles_keys) - - while len(rows) > density: - true_length = len(rows) - rows_partitions = [] - offset = 0 - for p in result["percentiles"]: - rows_partitions.append([]) - for r in rows[offset:]: - if r["responseTime"] < p["responseTime"]: - rows_partitions[-1].append(r) - offset += 1 - else: - break - rows_partitions.append(rows[offset:]) - # print(f"len rows partition: {len(rows_partitions)}") - # for r in rows_partitions: - # print(f"{r[0]} => {sum(v['count'] for v in r)}") - - largest_partition = 0 - for i in range(len(rows_partitions)): - if len(rows_partitions[i]) > len(rows_partitions[largest_partition]): - largest_partition = i - # print(f"largest partition: {len(rows_partitions[largest_partition])}") - - if len(rows_partitions[largest_partition]) <= 2: - break - # computing lowest merge diff - diff = rows[-1]["responseTime"] - for i in range(1, len(rows_partitions[largest_partition]) - 1, 1): - v1 = rows_partitions[largest_partition][i] - v2 = rows_partitions[largest_partition][i + 1] - if (v2["responseTime"] - v1["responseTime"]) < diff: - diff = v2["responseTime"] - v1["responseTime"] - # print(f"lowest merge diff: {diff}") - i = 1 - while i < len(rows_partitions[largest_partition]) - 1 and true_length > density - 1: - v1 = rows_partitions[largest_partition][i] - v2 = rows_partitions[largest_partition][i + 1] - if (v2["responseTime"] - v1["responseTime"]) == diff: - rows_partitions[largest_partition][i]["count"] += v2["count"] - rows_partitions[largest_partition][i]["responseTime"] = v2["responseTime"] - del rows_partitions[largest_partition][i + 1] - true_length -= 1 - else: - i += 1 - - rows = [r for rp in rows_partitions for r in rp] - - if extreme_values_first_index == len(rows): - rows.append({"count": 0, "responseTime": rows[-1]["responseTime"] + 10}) - - result["chart"] = rows - - return result - - -def get_busiest_time_of_day(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - ch_sub_query = __get_basic_constraints(table_name="sessions", data=args) - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT intDiv(toHour(sessions.datetime),2)*2 AS hour, - COUNT(1) AS count - FROM {exp_ch_helper.get_main_sessions_table(startTimestamp)} AS sessions - WHERE {" AND ".join(ch_sub_query)} - GROUP BY hour - ORDER BY hour ASC;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - return __complete_missing_steps(rows=rows, start_time=0, end_time=24000, density=12, - neutral={"count": 0}, - time_key="hour", time_coefficient=1) - - def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), value=None, **args): ch_sub_query = __get_basic_constraints(table_name="pages", data=args) @@ -1035,244 +490,6 @@ def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return helper.dict_to_camel_case(rows[0]) -def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, url=None, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.event_type='LOCATION'") - ch_sub_query_chart.append("isNotNull(pages.visually_complete)") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - if url is not None: - ch_sub_query_chart.append("pages.url_path = %(value)s") - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COALESCE(avgOrNull(pages.visually_complete),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, "value": url, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - ch_query = f"""SELECT COALESCE(avgOrNull(pages.visually_complete),0) AS avg - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)};""" - avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - results = {"value": avg, "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, density=density, - neutral={"value": 0})} - helper.__time_value(results) - return results - - -def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - ch_sub_query.append("isNotNull(pages.response_time)") - ch_sub_query.append("pages.response_time>0") - sch_sub_query = ch_sub_query[:] - if value is not None: - ch_sub_query.append("pages.url_path = %(value)s") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(DISTINCT pages.session_id) AS count - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)} - AND (pages.response_time)>(SELECT COALESCE(avgOrNull(pages.response_time),0) - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(sch_sub_query)})*2 - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, - params={"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)}) - return __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, density=density, - neutral={"count": 0}) - - -def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="performance", round_start=True, - data=args) - ch_sub_query_chart.append("performance.event_type='PERFORMANCE'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(performance.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COALESCE(avgOrNull(performance.avg_used_js_heap_size),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS performance - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp ASC;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - ch_query = f"""SELECT COALESCE(avgOrNull(performance.avg_used_js_heap_size),0) AS avg - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS performance - WHERE {" AND ".join(ch_sub_query_chart)};""" - avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"value": avg, - "chart": helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"value": 0})), - "unit": schemas.TemplatePredefinedUnits.MEMORY} - - -def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="performance", round_start=True, - data=args) - ch_sub_query_chart.append("performance.event_type='PERFORMANCE'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(performance.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COALESCE(avgOrNull(performance.avg_cpu),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS performance - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp ASC;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - ch_query = f"""SELECT COALESCE(avgOrNull(performance.avg_cpu),0) AS avg - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS performance - WHERE {" AND ".join(ch_sub_query_chart)};""" - avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"value": avg, - "chart": helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"value": 0})), - "unit": schemas.TemplatePredefinedUnits.PERCENTAGE} - - -def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="performance", round_start=True, - data=args) - ch_sub_query_chart.append("performance.event_type='PERFORMANCE'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(performance.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COALESCE(avgOrNull(performance.avg_fps),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS performance - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp ASC;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - ch_query = f"""SELECT COALESCE(avgOrNull(performance.avg_fps),0) AS avg - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS performance - WHERE {" AND ".join(ch_sub_query_chart)};""" - avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"value": avg, - "chart": helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"value": 0})), - "unit": schemas.TemplatePredefinedUnits.FRAME} - - -def get_crashes(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query = __get_basic_constraints(table_name="sessions", round_start=True, data=args) - ch_sub_query.append("has(sessions.issue_types,'crash')") - ch_sub_query_chart = __get_basic_constraints(table_name="sessions", round_start=True, - data=args) - ch_sub_query_chart.append("has(sessions.issue_types,'crash')") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - ch_sub_query_chart += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(sessions.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(1) AS value - FROM {exp_ch_helper.get_main_sessions_table(startTimestamp)} AS sessions - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - if len(rows) == 0: - browsers = [] - else: - ch_query = f"""SELECT b.user_browser AS browser, - sum(bv.count) AS total, - groupArray([bv.user_browser_version, toString(bv.count)]) AS versions - FROM ( - SELECT sessions.user_browser - FROM {exp_ch_helper.get_main_sessions_table(startTimestamp)} AS sessions - WHERE {" AND ".join(ch_sub_query)} - GROUP BY sessions.user_browser - ORDER BY COUNT(1) DESC - LIMIT 3 - ) AS b - INNER JOIN - ( - SELECT sessions.user_browser, - sessions.user_browser_version, - COUNT(1) AS count - FROM {exp_ch_helper.get_main_sessions_table(startTimestamp)} AS sessions - WHERE {" AND ".join(ch_sub_query)} - GROUP BY sessions.user_browser, - sessions.user_browser_version - ORDER BY count DESC - ) AS bv USING (user_browser) - GROUP BY b.user_browser - ORDER BY b.user_browser;""" - browsers = ch.execute(query=ch_query, params=params) - total = sum(r["total"] for r in browsers) - for r in browsers: - r["percentage"] = r["total"] / (total / 100) - versions = [] - for i in range(len(r["versions"][:3])): - versions.append({r["versions"][i][0]: int(r["versions"][i][1]) / (r["total"] / 100)}) - r["versions"] = versions - - result = {"chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"value": 0}), - "browsers": browsers, - "unit": schemas.TemplatePredefinedUnits.COUNT} - return result - - def __get_domains_errors_neutral(rows): neutral = {l: 0 for l in [i for k in [list(v.keys()) for v in rows] for i in k]} if len(neutral.keys()) == 0: @@ -1386,33 +603,6 @@ def __nested_array_to_dict_array(rows): return rows -def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - ch_sub_query = __get_basic_constraints(table_name="resources", data=args) - ch_sub_query.append("isNotNull(resources.duration)") - ch_sub_query.append("resources.duration>0") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT resources.url_host AS domain, - COALESCE(avgOrNull(resources.duration),0) AS value - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query)} - GROUP BY resources.url_host - ORDER BY value DESC - LIMIT 5;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - ch_query = f"""SELECT COALESCE(avgOrNull(resources.duration),0) AS avg - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query)};""" - avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"value": avg, "chart": rows, "unit": schemas.TemplatePredefinedUnits.MILLISECOND} - - def get_errors_per_domains(project_id, limit, page, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): ch_sub_query = __get_basic_constraints(table_name="requests", data=args) @@ -1453,81 +643,6 @@ def get_errors_per_domains(project_id, limit, page, startTimestamp=TimeUTC.now(d return response -def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), - platform=None, **args): - ch_sub_query = __get_basic_constraints(table_name="sessions", data=args) - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT b.user_browser AS browser, - b.count, - groupArray([bv.user_browser_version, toString(bv.count)]) AS versions - FROM - ( - SELECT sessions.user_browser, - COUNT(1) AS count - FROM {exp_ch_helper.get_main_sessions_table(startTimestamp)} AS sessions - WHERE {" AND ".join(ch_sub_query)} - GROUP BY sessions.user_browser - ORDER BY count DESC - LIMIT 3 - ) AS b - INNER JOIN - ( - SELECT sessions.user_browser, - sessions.user_browser_version, - COUNT(1) AS count - FROM {exp_ch_helper.get_main_sessions_table(startTimestamp)} AS sessions - WHERE {" AND ".join(ch_sub_query)} - GROUP BY - sessions.user_browser, - sessions.user_browser_version - ORDER BY count DESC - LIMIT 3 - ) AS bv USING (user_browser) - GROUP BY - b.user_browser, b.count - ORDER BY b.count DESC;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - for i, r in enumerate(rows): - versions = {} - for j in range(len(r["versions"])): - versions[r["versions"][j][0]] = int(r["versions"][j][1]) - r.pop("versions") - rows[i] = {**r, **versions} - return {"count": sum(i["count"] for i in rows), "chart": rows} - - -def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), - platform=None, **args): - ch_sub_query = __get_basic_constraints(table_name="requests", data=args) - ch_sub_query.append("requests.event_type = 'REQUEST'") - ch_sub_query.append("intDiv(requests.status, 100) != 2") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT requests.method, - requests.url_hostpath, - COUNT(1) AS all_requests, - SUM(if(intDiv(requests.status, 100) == 4, 1, 0)) AS _4xx, - SUM(if(intDiv(requests.status, 100) == 5, 1, 0)) AS _5xx - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS requests - WHERE {" AND ".join(ch_sub_query)} - GROUP BY requests.method, requests.url_hostpath - ORDER BY (_4xx + _5xx) DESC, all_requests DESC - LIMIT 50;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - return helper.list_to_camel_case(rows) - - def __get_calls_errors_4xx_or_5xx(status, project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, **args): @@ -1602,50 +717,6 @@ def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), e neutral={"4xx": 0, "5xx": 0, "js": 0, "integrations": 0}) -def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - ch_sub_query_chart_response_end = __get_basic_constraints(table_name="pages", round_start=True, - data=args) - ch_sub_query_chart_response_end.append("pages.event_type='LOCATION'") - ch_sub_query_chart_response_end.append("isNotNull(pages.response_end)") - ch_sub_query_chart_response_end.append("pages.response_end>0") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - ch_sub_query_chart_response_end += meta_condition - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(1) AS total, - SUM(if(resources.type='fetch',1,0)) AS xhr - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - # print(ch.format(query=ch_query, params=params)) - actions = ch.execute(query=ch_query, params=params) - actions = __complete_missing_steps(rows=actions, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"total": 0, "xhr": 0}) - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COALESCE(avgOrNull(pages.response_end),0) AS avg_response_end - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart_response_end)} - GROUP BY timestamp - ORDER BY timestamp;""" - response_end = ch.execute(query=ch_query, params=params) - response_end = __complete_missing_steps(rows=response_end, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"avg_response_end": 0}) - return helper.list_to_camel_case(__merge_charts(response_end, actions)) - - def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) @@ -1686,102 +757,6 @@ def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(de "errors_count": 0}))} -# TODO: super slow (try using sampling) -def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query = __get_basic_constraints(table_name="resources", data=args) - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(s.base_datetime, toIntervalSecond(%(step_size)s))) * 1000 AS timestamp, - COALESCE(avgOrNull(NULLIF(s.count,0)),0) AS avg, - groupArray([toString(t.type), toString(t.xavg)]) AS types - FROM - ( SELECT resources.session_id, - MIN(resources.datetime) AS base_datetime, - COUNT(1) AS count - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query)} - GROUP BY resources.session_id - ) AS s - INNER JOIN - (SELECT session_id, - type, - COALESCE(avgOrNull(NULLIF(count,0)),0) AS xavg - FROM (SELECT resources.session_id, resources.type, COUNT(1) AS count - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query)} - GROUP BY resources.session_id, resources.type) AS ss - GROUP BY ss.session_id, ss.type) AS t USING (session_id) - GROUP BY timestamp - ORDER BY timestamp ASC;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - # print(">>>>>>>>>>>>>>") - # print(ch.format(query=ch_query, params=params)) - # print(">>>>>>>>>>>>>>") - rows = ch.execute(query=ch_query, params=params) - for r in rows: - types = {} - for i in range(len(r["types"])): - if r["types"][i][0] not in types: - types[r["types"][i][0]] = [] - types[r["types"][i][0]].append(float(r["types"][i][1])) - for i in types: - types[i] = sum(types[i]) / len(types[i]) - r["types"] = types - resources = __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"avg": 0, "types": {}}) - time_to_render = get_time_to_render(project_id=project_id, startTimestamp=startTimestamp, - endTimestamp=endTimestamp, density=density, - **args) - - return helper.list_to_camel_case( - __merge_charts( - [{"timestamp": i["timestamp"], "avgCountResources": i["avg"], "types": i["types"]} for i in resources], - [{"timestamp": i["timestamp"], "avgTimeToRender": i["value"]} for i in time_to_render["chart"]])) - - -def get_resources_count_by_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), density=7, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT timestamp, - groupArray([toString(t.type), toString(t.count)]) AS types - FROM(SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - resources.type, - COUNT(1) AS count - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp,resources.type - ORDER BY timestamp) AS t - GROUP BY timestamp;""" - params = {"step_size": step_size, - "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} - # print(ch.format(query=ch_query, params=params)) - rows = ch.execute(query=ch_query, params=params) - for r in rows: - for t in r["types"]: - r[t[0]] = t[1] - r.pop("types") - return __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={k: 0 for k in RESOURCS_TYPE_TO_DB_TYPE.keys()}) - - def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) @@ -1829,42 +804,6 @@ def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1) "third_party": 0})) -def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with ch_client.ClickHouseClient() as ch: - row = __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args) - results = helper.dict_to_camel_case(row) - results["chart"] = get_performance_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - row = __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args) - previous = helper.dict_to_camel_case(row) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - -def __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args): - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - ch_sub_query.append("pages.load_event_end>0") - ch_query = f"""SELECT COALESCE(avgOrNull(pages.load_event_end),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)};""" - params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, - **__get_constraint_values(args)} - # print(ch.format(query=ch_query, params=params)) - row = ch.execute(query=ch_query, params=params)[0] - result = row - for k in result: - if result[k] is None: - result[k] = 0 - return result - - def get_performance_avg_page_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, resources=None, **args): @@ -1909,281 +848,6 @@ def get_performance_avg_page_load_time(ch, project_id, startTimestamp=TimeUTC.no return pages -def get_application_activity_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with ch_client.ClickHouseClient() as ch: - row = __get_application_activity_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args) - results = helper.dict_to_camel_case(row) - results["chart"] = get_performance_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - row = __get_application_activity_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args) - previous = helper.dict_to_camel_case(row) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - -def __get_application_activity_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args): - ch_sub_query = __get_basic_constraints(table_name="resources", data=args) - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - ch_sub_query.append("resources.type= %(type)s") - ch_sub_query.append("resources.duration>0") - ch_query = f"""\ - SELECT COALESCE(avgOrNull(resources.duration),0) AS value - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query)};""" - row = ch.execute(query=ch_query, - params={"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})[0] - result = row - # for k in result: - # if result[k] is None: - # result[k] = 0 - return result - - -def get_performance_avg_image_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=19, resources=None, **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) - img_constraints = [] - ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - img_constraints_vals = {} - - if resources and len(resources) > 0: - for r in resources: - if r["type"] == "IMG": - img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s") - img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value'] - - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - ch_sub_query_chart.append("resources.duration>0") - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(resources.duration),0) AS value - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query_chart)} - AND resources.type = 'img' - {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, params={**params, **img_constraints_vals, **__get_constraint_values(args)}) - images = __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"value": 0}) - - # for s in images: - # for k in s: - # if s[k] is None: - # s[k] = 0 - return images - - -def get_application_activity_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with ch_client.ClickHouseClient() as ch: - row = __get_application_activity_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args) - results = helper.dict_to_camel_case(row) - results["chart"] = get_performance_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - row = __get_application_activity_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args) - previous = helper.dict_to_camel_case(row) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - -def __get_application_activity_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args): - ch_sub_query = __get_basic_constraints(table_name="resources", data=args) - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - ch_sub_query.append("resources.type= %(type)s") - ch_sub_query.append("resources.duration>0") - ch_query = f"""SELECT COALESCE(avgOrNull(resources.duration),0) AS value - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query)};""" - row = ch.execute(query=ch_query, - params={"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})[0] - result = row - # for k in result: - # if result[k] is None: - # result[k] = 0 - return result - - -def get_performance_avg_request_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=19, resources=None, **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) - request_constraints = [] - ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - request_constraints_vals = {} - - if resources and len(resources) > 0: - for r in resources: - if r["type"] != "IMG" and r["type"] == "LOCATION": - request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s") - request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - ch_sub_query_chart.append("resources.duration>0") - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(resources.duration),0) AS value - FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources - WHERE {" AND ".join(ch_sub_query_chart)} - AND resources.type = 'fetch' - {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, - params={**params, **request_constraints_vals, **__get_constraint_values(args)}) - requests = __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, density=density, - neutral={"value": 0}) - - # for s in requests: - # for k in s: - # if s[k] is None: - # s[k] = 0 - return requests - - -def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with ch_client.ClickHouseClient() as ch: - results = {} - rows = __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args) - if len(rows) > 0: - results = helper.dict_to_camel_case(rows[0]) - results["chart"] = __get_page_metrics_avg_dom_content_load_start_chart(ch, project_id, startTimestamp, - endTimestamp, **args) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - rows = __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args) - if len(rows) > 0: - previous = helper.dict_to_camel_case(rows[0]) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - -def __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args): - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - ch_sub_query.append("pages.dom_content_loaded_event_end>0") - ch_query = f"""SELECT COALESCE(avgOrNull(pages.dom_content_loaded_event_end),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)};""" - params = {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, - **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - return rows - - -def __get_page_metrics_avg_dom_content_load_start_chart(ch, project_id, startTimestamp, endTimestamp, density=19, - **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.event_type='LOCATION'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - ch_sub_query_chart.append("pages.dom_content_loaded_event_end>0") - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(pages.dom_content_loaded_event_end),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, params={**params, **__get_constraint_values(args)}) - rows = __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"value": 0}) - - # for s in rows: - # for k in s: - # if s[k] is None: - # s[k] = 0 - return rows - - -def get_page_metrics_avg_first_contentful_pixel(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - with ch_client.ClickHouseClient() as ch: - rows = __get_page_metrics_avg_first_contentful_pixel(ch, project_id, startTimestamp, endTimestamp, **args) - if len(rows) > 0: - results = helper.dict_to_camel_case(rows[0]) - results["chart"] = __get_page_metrics_avg_first_contentful_pixel_chart(ch, project_id, startTimestamp, - endTimestamp, **args) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - rows = __get_page_metrics_avg_first_contentful_pixel(ch, project_id, startTimestamp, endTimestamp, **args) - if len(rows) > 0: - previous = helper.dict_to_camel_case(rows[0]) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - -def __get_page_metrics_avg_first_contentful_pixel(ch, project_id, startTimestamp, endTimestamp, **args): - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - ch_sub_query.append("pages.first_contentful_paint_time>0") - # changed dom_content_loaded_event_start to dom_content_loaded_event_end - ch_query = f"""\ - SELECT COALESCE(avgOrNull(pages.first_contentful_paint_time),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)};""" - params = {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, - **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - return rows - - -def __get_page_metrics_avg_first_contentful_pixel_chart(ch, project_id, startTimestamp, endTimestamp, density=20, - **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.event_type='LOCATION'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - ch_sub_query_chart.append("pages.first_contentful_paint_time>0") - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(pages.first_contentful_paint_time),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, params={**params, **__get_constraint_values(args)}) - rows = __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"value": 0}) - return rows - - def get_user_activity_avg_visited_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): results = {} @@ -2256,112 +920,6 @@ def __get_user_activity_avg_visited_pages_chart(ch, project_id, startTimestamp, return rows -def get_user_activity_avg_session_duration(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): - results = {} - - with ch_client.ClickHouseClient() as ch: - rows = __get_user_activity_avg_session_duration(ch, project_id, startTimestamp, endTimestamp, **args) - if len(rows) > 0: - results = helper.dict_to_camel_case(rows[0]) - for key in results: - if isnan(results[key]): - results[key] = 0 - results["chart"] = __get_user_activity_avg_session_duration_chart(ch, project_id, startTimestamp, - endTimestamp, **args) - diff = endTimestamp - startTimestamp - endTimestamp = startTimestamp - startTimestamp = endTimestamp - diff - rows = __get_user_activity_avg_session_duration(ch, project_id, startTimestamp, endTimestamp, **args) - - if len(rows) > 0: - previous = helper.dict_to_camel_case(rows[0]) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - helper.__time_value(results) - return results - - -def __get_user_activity_avg_session_duration(ch, project_id, startTimestamp, endTimestamp, **args): - ch_sub_query = __get_basic_constraints(table_name="sessions", data=args) - meta_condition = __get_meta_constraint(args) - ch_sub_query += meta_condition - ch_sub_query.append("isNotNull(sessions.duration)") - ch_sub_query.append("sessions.duration>0") - - ch_query = f"""SELECT COALESCE(avgOrNull(sessions.duration),0) AS value - FROM {exp_ch_helper.get_main_sessions_table(startTimestamp)} AS sessions - WHERE {" AND ".join(ch_sub_query)};""" - params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, - **__get_constraint_values(args)} - - rows = ch.execute(query=ch_query, params=params) - - return rows - - -def __get_user_activity_avg_session_duration_chart(ch, project_id, startTimestamp, endTimestamp, density=20, **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) - ch_sub_query_chart = __get_basic_constraints(table_name="sessions", round_start=True, data=args) - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - ch_sub_query_chart.append("isNotNull(sessions.duration)") - ch_sub_query_chart.append("sessions.duration>0") - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp} - - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(sessions.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(sessions.duration),0) AS value - FROM {exp_ch_helper.get_main_sessions_table(startTimestamp)} AS sessions - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - - rows = ch.execute(query=ch_query, params={**params, **__get_constraint_values(args)}) - rows = __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"value": 0}) - return rows - - -def get_top_metrics_avg_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=20, **args): - step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.event_type='LOCATION'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - ch_sub_query += meta_condition - - if value is not None: - ch_sub_query.append("pages.url_path = %(value)s") - ch_sub_query_chart.append("pages.url_path = %(value)s") - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT COALESCE(avgOrNull(pages.response_time),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.response_time) AND pages.response_time>0;""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - results = rows[0] - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COALESCE(avgOrNull(pages.response_time),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)} AND isNotNull(pages.response_time) AND pages.response_time>0 - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, params={**params, **__get_constraint_values(args)}) - rows = __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"value": 0}) - results["chart"] = rows - helper.__time_value(results) - return helper.dict_to_camel_case(results) - - def get_top_metrics_count_requests(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), value=None, density=20, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) @@ -2401,179 +959,6 @@ def get_top_metrics_count_requests(project_id, startTimestamp=TimeUTC.now(delta_ return helper.dict_to_camel_case(result) -def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=20, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.event_type='LOCATION'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - ch_sub_query += meta_condition - - if value is not None: - ch_sub_query.append("pages.url_path = %(value)s") - ch_sub_query_chart.append("pages.url_path = %(value)s") - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT COALESCE(avgOrNull(pages.first_paint),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.first_paint) AND pages.first_paint>0;""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - results = rows[0] - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COALESCE(avgOrNull(pages.first_paint),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)} AND isNotNull(pages.first_paint) AND pages.first_paint>0 - GROUP BY timestamp - ORDER BY timestamp;;""" - rows = ch.execute(query=ch_query, params=params) - results["chart"] = helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"value": 0})) - - helper.__time_value(results) - return helper.dict_to_camel_case(results) - - -def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=19, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.event_type='LOCATION'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - ch_sub_query += meta_condition - - if value is not None: - ch_sub_query.append("pages.url_path = %(value)s") - ch_sub_query_chart.append("pages.url_path = %(value)s") - ch_sub_query.append("isNotNull(pages.dom_content_loaded_event_time)") - ch_sub_query.append("pages.dom_content_loaded_event_time>0") - ch_sub_query_chart.append("isNotNull(pages.dom_content_loaded_event_time)") - ch_sub_query_chart.append("pages.dom_content_loaded_event_time>0") - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT COALESCE(avgOrNull(pages.dom_content_loaded_event_time),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)};""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - results = helper.dict_to_camel_case(rows[0]) - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COALESCE(avgOrNull(pages.dom_content_loaded_event_time),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, params=params) - results["chart"] = helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"value": 0})) - helper.__time_value(results) - return results - - -def get_top_metrics_avg_till_first_bit(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=20, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.event_type='LOCATION'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - ch_sub_query += meta_condition - - if value is not None: - ch_sub_query.append("pages.url_path = %(value)s") - ch_sub_query_chart.append("pages.url_path = %(value)s") - ch_sub_query.append("isNotNull(pages.ttfb)") - ch_sub_query.append("pages.ttfb>0") - ch_sub_query_chart.append("isNotNull(pages.ttfb)") - ch_sub_query_chart.append("pages.ttfb>0") - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT COALESCE(avgOrNull(pages.ttfb),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)};""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - results = rows[0] - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COALESCE(avgOrNull(pages.ttfb),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, params=params) - results["chart"] = helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"value": 0})) - helper.__time_value(results) - return helper.dict_to_camel_case(results) - - -def get_top_metrics_avg_time_to_interactive(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, density=20, **args): - step_size = __get_step_size(startTimestamp, endTimestamp, density) - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) - ch_sub_query_chart.append("pages.event_type='LOCATION'") - meta_condition = __get_meta_constraint(args) - ch_sub_query_chart += meta_condition - - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) - ch_sub_query.append("pages.event_type='LOCATION'") - ch_sub_query += meta_condition - - if value is not None: - ch_sub_query.append("pages.url_path = %(value)s") - ch_sub_query_chart.append("pages.url_path = %(value)s") - ch_sub_query.append("isNotNull(pages.time_to_interactive)") - ch_sub_query.append("pages.time_to_interactive >0") - ch_sub_query_chart.append("isNotNull(pages.time_to_interactive)") - ch_sub_query_chart.append("pages.time_to_interactive >0") - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT COALESCE(avgOrNull(pages.time_to_interactive),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query)};""" - params = {"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)} - rows = ch.execute(query=ch_query, params=params) - results = rows[0] - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COALESCE(avgOrNull(pages.time_to_interactive),0) AS value - FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS pages - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, params=params) - results["chart"] = helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, - neutral={"value": 0})) - helper.__time_value(results) - return helper.dict_to_camel_case(results) - - def get_unique_users(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): diff --git a/ee/api/chalicelib/core/resources.py b/ee/api/chalicelib/core/resources.py deleted file mode 100644 index 8bae47d10..000000000 --- a/ee/api/chalicelib/core/resources.py +++ /dev/null @@ -1,45 +0,0 @@ -from chalicelib.utils import helper, exp_ch_helper -from chalicelib.utils import ch_client -from chalicelib.utils.TimeUTC import TimeUTC -from decouple import config - - -def get_by_session_id(session_id, project_id, start_ts, duration): - with ch_client.ClickHouseClient() as ch: - if duration is None or (type(duration) != 'int' and type(duration) != 'float') or duration < 0: - duration = 0 - delta = config("events_ts_delta", cast=int, default=60 * 60) * 1000 - if config("EXP_RESOURCES", cast=bool, default=False): - ch_query = f"""SELECT - datetime,url,type,duration,ttfb,header_size, - encoded_body_size,decoded_body_size,success, - if(success, 200, 400) AS status - FROM {exp_ch_helper.get_main_resources_table(start_ts)} - WHERE session_id = toUInt64(%(session_id)s) - AND project_id = toUInt16(%(project_id)s) - AND datetime >= toDateTime(%(res_start_ts)s / 1000) - AND datetime <= toDateTime(%(res_end_ts)s / 1000);""" - else: - ch_query = """SELECT - datetime,url,type,duration,ttfb,header_size, - encoded_body_size,decoded_body_size,success, - coalesce(status,if(success, 200, status)) AS status - FROM resources - WHERE session_id = toUInt64(%(session_id)s) - AND project_id = toUInt16(%(project_id)s) - AND datetime >= toDateTime(%(res_start_ts)s / 1000) - AND datetime <= toDateTime(%(res_end_ts)s / 1000);""" - params = {"session_id": session_id, "project_id": project_id, "start_ts": start_ts, "duration": duration, - "res_start_ts": start_ts - delta, "res_end_ts": start_ts + duration + delta, } - rows = ch.execute(query=ch_query, params=params) - results = [] - for r in rows: - r["datetime"] = TimeUTC.datetime_to_timestamp(r["datetime"]) - # TODO: remove this once the tracker is fixed - if isinstance(r["url"], bytes): - try: - r["url"] = r["url"].decode("utf-8") - except UnicodeDecodeError: - continue - results.append(r) - return helper.list_to_camel_case(results) diff --git a/ee/api/chalicelib/core/sessions_replay.py b/ee/api/chalicelib/core/sessions_replay.py index 4bc4beac2..042ff92b0 100644 --- a/ee/api/chalicelib/core/sessions_replay.py +++ b/ee/api/chalicelib/core/sessions_replay.py @@ -1,6 +1,6 @@ import schemas from chalicelib.core import events, metadata, events_mobile, \ - sessions_mobs, issues, resources, assist, sessions_devtool, sessions_notes, canvas, user_testing + sessions_mobs, issues, assist, sessions_devtool, canvas, user_testing from chalicelib.utils import errors_helper from chalicelib.utils import pg_client, helper @@ -125,8 +125,6 @@ def get_events(project_id, session_id): if e['source'] == "js_exception"][:500] data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, session_id=session_id) - data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, - start_ts=s_data["startTs"], duration=s_data["duration"]) data['userTesting'] = user_testing.get_test_signals(session_id=session_id, project_id=project_id) data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) diff --git a/ee/api/chalicelib/utils/exp_ch_helper.py b/ee/api/chalicelib/utils/exp_ch_helper.py index 7ce033cde..31115d3f5 100644 --- a/ee/api/chalicelib/utils/exp_ch_helper.py +++ b/ee/api/chalicelib/utils/exp_ch_helper.py @@ -26,12 +26,6 @@ def get_main_sessions_table(timestamp=0): and timestamp and timestamp >= TimeUTC.now(delta_days=-7) else "experimental.sessions" -def get_main_resources_table(timestamp=0): - return "experimental.resources_l7d_mv" \ - if config("EXP_7D_MV", cast=bool, default=True) \ - and timestamp and timestamp >= TimeUTC.now(delta_days=-7) else "experimental.resources" - - def get_autocomplete_table(timestamp=0): return "experimental.autocomplete" diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/1.21.0/1.21.0.sql b/ee/scripts/schema/db/init_dbs/clickhouse/1.21.0/1.21.0.sql index c60511f00..b7639787c 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/1.21.0/1.21.0.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/1.21.0/1.21.0.sql @@ -1 +1,6 @@ CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.21.0-ee'; + + +DROP TABLE IF EXISTS experimental.resources_l7d_mv; + +DROP TABLE IF EXISTS experimental.resources; \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index 14d225848..6cb9078ef 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -90,32 +90,7 @@ CREATE TABLE IF NOT EXISTS experimental.events ORDER BY (project_id, datetime, event_type, session_id, message_id) TTL datetime + INTERVAL 3 MONTH; -CREATE TABLE IF NOT EXISTS experimental.resources -( - session_id UInt64, - project_id UInt16, - datetime DateTime, - url String, - url_host String MATERIALIZED lower(domain(url)), - url_path String, - url_hostpath String MATERIALIZED concat(url_host, url_path), - type Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4), - name Nullable(String) MATERIALIZED if(type = 'fetch', null, - coalesce(nullIf(splitByChar('/', url_path)[-1], ''), - nullIf(splitByChar('/', url_path)[-2], ''))), - duration Nullable(UInt16), - ttfb Nullable(UInt16), - header_size Nullable(UInt16), - encoded_body_size Nullable(UInt32), - decoded_body_size Nullable(UInt32), - compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), - success Nullable(UInt8) COMMENT 'currently available for type=img only', - message_id UInt64 DEFAULT 0, - _timestamp DateTime DEFAULT now() -) ENGINE = ReplacingMergeTree(_timestamp) - PARTITION BY toYYYYMM(datetime) - ORDER BY (project_id, datetime, type, session_id, message_id) - TTL datetime + INTERVAL 3 MONTH; + CREATE TABLE IF NOT EXISTS experimental.sessions ( @@ -289,33 +264,6 @@ SELECT session_id, FROM experimental.events WHERE datetime >= now() - INTERVAL 7 DAY; -CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.resources_l7d_mv - ENGINE = ReplacingMergeTree(_timestamp) - PARTITION BY toYYYYMMDD(datetime) - ORDER BY (project_id, datetime, type, session_id, message_id) - TTL datetime + INTERVAL 7 DAY - POPULATE -AS -SELECT session_id, - project_id, - datetime, - url, - url_host, - url_path, - url_hostpath, - type, - name, - duration, - ttfb, - header_size, - encoded_body_size, - decoded_body_size, - compression_ratio, - success, - message_id, - _timestamp -FROM experimental.resources -WHERE datetime >= now() - INTERVAL 7 DAY; CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.sessions_l7d_mv ENGINE = ReplacingMergeTree(_timestamp) diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.21.0/1.21.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.21.0/1.21.0.sql index 1ece9429f..6b38d333c 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.21.0/1.21.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.21.0/1.21.0.sql @@ -35,6 +35,14 @@ CREATE TABLE IF NOT EXISTS public.session_integrations PRIMARY KEY (session_id, project_id, provider) ); +ALTER TABLE IF EXISTS public.metrics + ALTER COLUMN user_id DROP NOT NULL, + ALTER COLUMN project_id SET NOT NULL; + +ALTER TABLE IF EXISTS public.dashboards + ALTER COLUMN user_id DROP NOT NULL, + ALTER COLUMN project_id SET NOT NULL; + COMMIT; \elif :is_next diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index 9cf444947..8a95fd4bc 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -758,44 +758,6 @@ CREATE TABLE events.state_actions CREATE INDEX state_actions_name_gin_idx ON events.state_actions USING GIN (name gin_trgm_ops); CREATE INDEX state_actions_timestamp_idx ON events.state_actions (timestamp); -CREATE TYPE events.resource_type AS ENUM ('other', 'script', 'stylesheet', 'fetch', 'img', 'media'); -CREATE TYPE events.resource_method AS ENUM ('GET' , 'HEAD' , 'POST' , 'PUT' , 'DELETE' , 'CONNECT' , 'OPTIONS' , 'TRACE' , 'PATCH' ); -CREATE TABLE events.resources -( - session_id bigint NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE, - message_id bigint NOT NULL, - timestamp bigint NOT NULL, - duration bigint NULL, - type events.resource_type NOT NULL, - url text NOT NULL, - url_host text NOT NULL, - url_hostpath text NOT NULL, - success boolean NOT NULL, - status smallint NULL, - method events.resource_method NULL, - ttfb bigint NULL, - header_size bigint NULL, - encoded_body_size integer NULL, - decoded_body_size integer NULL, - PRIMARY KEY (session_id, message_id, timestamp) -); -CREATE INDEX resources_session_id_idx ON events.resources (session_id); -CREATE INDEX resources_status_idx ON events.resources (status); -CREATE INDEX resources_type_idx ON events.resources (type); -CREATE INDEX resources_url_host_idx ON events.resources (url_host); -CREATE INDEX resources_timestamp_idx ON events.resources (timestamp); -CREATE INDEX resources_success_idx ON events.resources (success); - -CREATE INDEX resources_url_hostpath_gin_idx ON events.resources USING GIN (url_hostpath gin_trgm_ops); -CREATE INDEX resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL; -CREATE INDEX resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type); -CREATE INDEX resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch'; -CREATE INDEX resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE; -CREATE INDEX resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script'); -CREATE INDEX resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img'; -CREATE INDEX resources_timestamp_session_id_idx ON events.resources (timestamp, session_id); -CREATE INDEX resources_timestamp_duration_durationgt0NN_idx ON events.resources (timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL; - CREATE TABLE events.performance ( session_id bigint NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE, @@ -875,8 +837,8 @@ CREATE INDEX jobs_project_id_idx ON public.jobs (project_id); CREATE TABLE public.metrics ( metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NULL REFERENCES public.projects (project_id) ON DELETE CASCADE, - user_id integer REFERENCES public.users (user_id) ON DELETE SET NULL, + project_id integer NOT NULL REFERENCES public.projects (project_id) ON DELETE CASCADE, + user_id integer NULL REFERENCES public.users (user_id) ON DELETE SET NULL, name text NOT NULL, is_public boolean NOT NULL DEFAULT TRUE, created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), @@ -915,7 +877,7 @@ CREATE TABLE public.dashboards ( dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES public.projects (project_id) ON DELETE CASCADE, - user_id integer REFERENCES public.users (user_id) ON DELETE SET NULL, + user_id integer NULL REFERENCES public.users (user_id) ON DELETE SET NULL, name text NOT NULL, description text NOT NULL DEFAULT '', is_public boolean NOT NULL DEFAULT TRUE, diff --git a/ee/scripts/schema/db/rollback_dbs/clickhouse/1.21.0/1.21.0.sql b/ee/scripts/schema/db/rollback_dbs/clickhouse/1.21.0/1.21.0.sql index 890fbd6ec..01b5b615c 100644 --- a/ee/scripts/schema/db/rollback_dbs/clickhouse/1.21.0/1.21.0.sql +++ b/ee/scripts/schema/db/rollback_dbs/clickhouse/1.21.0/1.21.0.sql @@ -1 +1,56 @@ -CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.20.0-ee'; \ No newline at end of file +CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.20.0-ee'; + +CREATE TABLE IF NOT EXISTS experimental.resources +( + session_id UInt64, + project_id UInt16, + datetime DateTime, + url String, + url_host String MATERIALIZED lower(domain(url)), + url_path String, + url_hostpath String MATERIALIZED concat(url_host, url_path), + type Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4), + name Nullable(String) MATERIALIZED if(type = 'fetch', null, + coalesce(nullIf(splitByChar('/', url_path)[-1], ''), + nullIf(splitByChar('/', url_path)[-2], ''))), + duration Nullable(UInt16), + ttfb Nullable(UInt16), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(UInt8) COMMENT 'currently available for type=img only', + message_id UInt64 DEFAULT 0, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, type, session_id, message_id) + TTL datetime + INTERVAL 3 MONTH; + +CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.resources_l7d_mv + ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, type, session_id, message_id) + TTL datetime + INTERVAL 7 DAY + POPULATE +AS +SELECT session_id, + project_id, + datetime, + url, + url_host, + url_path, + url_hostpath, + type, + name, + duration, + ttfb, + header_size, + encoded_body_size, + decoded_body_size, + compression_ratio, + success, + message_id, + _timestamp +FROM experimental.resources +WHERE datetime >= now() - INTERVAL 7 DAY; \ No newline at end of file diff --git a/scripts/schema/db/init_dbs/postgresql/1.21.0/1.21.0.sql b/scripts/schema/db/init_dbs/postgresql/1.21.0/1.21.0.sql index 4ab6e29d6..485423658 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.21.0/1.21.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.21.0/1.21.0.sql @@ -31,6 +31,46 @@ CREATE TABLE IF NOT EXISTS public.session_integrations PRIMARY KEY (session_id, project_id, provider) ); +ALTER TABLE IF EXISTS public.metrics + ALTER COLUMN user_id DROP NOT NULL, + ALTER COLUMN project_id SET NOT NULL; + +ALTER TABLE IF EXISTS public.dashboards + ALTER COLUMN user_id DROP NOT NULL, + ALTER COLUMN project_id SET NOT NULL; + +DELETE +FROM public.metrics +WHERE metric_of IN ('avgCpu', 'avgDomContentLoaded', + 'avgDomContentLoadStart', 'avgFirstContentfulPixel', + 'avgFirstPaint', + 'avgFps', 'avgImageLoadTime', + 'avgPageLoadTime', 'avgRequestLoadTime', + 'avgResponseTime', 'avgSessionDuration', + 'avgTillFirstByte', 'avgTimeToRender') + or metric_of IN ('timeToRender', 'cpu','crashes' + 'fps', 'avgTimeToInteractive', + 'avgPagesResponseTime', 'avgUsedJsHeapSize', + 'memoryConsumption', 'pagesResponseTime', + 'pagesDomBuildtime', 'pagesResponseTimeDistribution', + 'resourcesVsVisuallyComplete', 'sessionsPerBrowser', + 'slowestDomains', 'speedLocation', 'impactedSessionsBySlowPages', + 'avgPagesDomBuildtime') + or metric_of IN ('missingResources', 'resourcesLoadingTime', + 'slowestResources', 'callsErrors','resourceTypeVsResponseEnd', + 'resourcesCountByType'); + +DELETE +FROM public.alerts +WHERE query ->> 'left' IN ('performance.image_load_time.average', 'performance.request_load_time.average', + 'resources.load_time.average', 'resources.missing.count', + 'errors.4xx_5xx.count', 'errors.4xx.count', + 'errors.5xx.count', 'errors.javascript.impacted_sessions.count'); + +DROP TABLE IF EXISTS events.resources; +DROP TYPE IF EXISTS events.resource_type; +DROP TYPE IF EXISTS events.resource_method; + COMMIT; \elif :is_next diff --git a/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/scripts/schema/db/init_dbs/postgresql/init_schema.sql index 52f003886..bc5d1c01b 100644 --- a/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -719,44 +719,6 @@ CREATE TABLE events.state_actions CREATE INDEX state_actions_name_gin_idx ON events.state_actions USING GIN (name gin_trgm_ops); CREATE INDEX state_actions_timestamp_idx ON events.state_actions (timestamp); -CREATE TYPE events.resource_type AS ENUM ('other', 'script', 'stylesheet', 'fetch', 'img', 'media'); -CREATE TYPE events.resource_method AS ENUM ('GET' , 'HEAD' , 'POST' , 'PUT' , 'DELETE' , 'CONNECT' , 'OPTIONS' , 'TRACE' , 'PATCH' ); -CREATE TABLE events.resources -( - session_id bigint NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE, - message_id bigint NOT NULL, - timestamp bigint NOT NULL, - duration bigint NULL, - type events.resource_type NOT NULL, - url text NOT NULL, - url_host text NOT NULL, - url_hostpath text NOT NULL, - success boolean NOT NULL, - status smallint NULL, - method events.resource_method NULL, - ttfb bigint NULL, - header_size bigint NULL, - encoded_body_size integer NULL, - decoded_body_size integer NULL, - PRIMARY KEY (session_id, message_id, timestamp) -); -CREATE INDEX resources_session_id_idx ON events.resources (session_id); -CREATE INDEX resources_status_idx ON events.resources (status); -CREATE INDEX resources_type_idx ON events.resources (type); -CREATE INDEX resources_url_host_idx ON events.resources (url_host); -CREATE INDEX resources_timestamp_idx ON events.resources (timestamp); -CREATE INDEX resources_success_idx ON events.resources (success); - -CREATE INDEX resources_url_hostpath_gin_idx ON events.resources USING GIN (url_hostpath gin_trgm_ops); -CREATE INDEX resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL; -CREATE INDEX resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type); -CREATE INDEX resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch'; -CREATE INDEX resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE; -CREATE INDEX resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script'); -CREATE INDEX resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img'; -CREATE INDEX resources_timestamp_session_id_idx ON events.resources (timestamp, session_id); -CREATE INDEX resources_timestamp_duration_durationgt0NN_idx ON events.resources (timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL; - CREATE TABLE events.performance ( session_id bigint NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE, @@ -836,8 +798,8 @@ CREATE INDEX jobs_project_id_idx ON public.jobs (project_id); CREATE TABLE public.metrics ( metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NULL REFERENCES public.projects (project_id) ON DELETE CASCADE, - user_id integer REFERENCES public.users (user_id) ON DELETE SET NULL, + project_id integer NOT NULL REFERENCES public.projects (project_id) ON DELETE CASCADE, + user_id integer NULL REFERENCES public.users (user_id) ON DELETE SET NULL, name text NOT NULL, is_public boolean NOT NULL DEFAULT TRUE, created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), @@ -876,7 +838,7 @@ CREATE TABLE public.dashboards ( dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES public.projects (project_id) ON DELETE CASCADE, - user_id integer REFERENCES public.users (user_id) ON DELETE SET NULL, + user_id integer NULL REFERENCES public.users (user_id) ON DELETE SET NULL, name text NOT NULL, description text NOT NULL DEFAULT '', is_public boolean NOT NULL DEFAULT TRUE,