jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Api metrics (#81)

Browse source 
* feat(api): changed initial connetion pool s

* feat(api): FOS metrics optimization
feat(db): Indexes change & delta

* feat(api): more metrics optimizations
feat(db): new indexs and removed resources_type_idx for better DB engine planning

* feat(api): metrics optimization
feat(db): changed indexes

* feat(api): optimized metrics queries
feat(db): changed indexes

* feat(api): optimized metrics
feat(db): created new indexes

* feat(api): changed metrics-autocomplete behaviour

* feat(db): pg-delta v1.2.0
This commit is contained in:
Kraiem Taha Yassine 2021-07-08 15:28:10 +02:00 • committed by GitHub
parent 441f38afec
commit 14327ab9c8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 633 additions and 401 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
api/chalicelib/blueprints/subs/bp_dashboard.py
View file

@ -130,8 +130,9 @@ def get_network_widget(projectId, context):
@app.route('/{projectId}/dashboard/{widget}/search', methods=['GET'])
def get_dashboard_autocomplete(projectId, widget, context):
params = app.current_request.query_params
if params is None:
if params is None or params.get('q') is None or len(params.get('q')) == 0:
return {"data": []}
params['q'] = '^' + params['q']
if widget in ['performance']:
data = dashboard.search(params.get('q', ''), params.get('type', ''), project_id=projectId,
@ -547,59 +548,3 @@ def get_dashboard_group(projectId, context):
*helper.explode_widget(dashboard.get_avg_cpu(project_id=projectId, **{**data, **args})),
*helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **{**data, **args})),
]}
@app.route('/{projectId}/dashboard/errors_crashes', methods=['GET', 'POST'])
def get_dashboard_group(projectId, context):
data = app.current_request.json_body
if data is None:
data = {}
params = app.current_request.query_params
args = dashboard.dashboard_args(params)
return {"data": [
{"key": "errors",
"data": dashboard.get_errors(project_id=projectId, **{**data, **args})},
{"key": "errors_trend",
"data": dashboard.get_errors_trend(project_id=projectId, **{**data, **args})},
{"key": "crashes",
"data": dashboard.get_crashes(project_id=projectId, **{**data, **args})},
{"key": "domains_errors",
"data": dashboard.get_domains_errors(project_id=projectId, **{**data, **args})},
{"key": "errors_per_domains",
"data": dashboard.get_errors_per_domains(project_id=projectId, **{**data, **args})},
{"key": "calls_errors",
"data": dashboard.get_calls_errors(project_id=projectId, **{**data, **args})},
{"key": "errors_per_type",
"data": dashboard.get_errors_per_type(project_id=projectId, **{**data, **args})},
{"key": "impacted_sessions_by_js_errors",
"data": dashboard.get_impacted_sessions_by_js_errors(project_id=projectId, **{**data, **args})}
]}
@app.route('/{projectId}/dashboard/resources', methods=['GET', 'POST'])
def get_dashboard_group(projectId, context):
data = app.current_request.json_body
if data is None:
data = {}
params = app.current_request.query_params
args = dashboard.dashboard_args(params)
return {"data": [
{"key": "slowest_images",
"data": dashboard.get_slowest_images(project_id=projectId, **{**data, **args})},
{"key": "missing_resources",
"data": dashboard.get_missing_resources_trend(project_id=projectId, **{**data, **args})},
{"key": "slowest_resources",
"data": dashboard.get_slowest_resources(project_id=projectId, type='all', **{**data, **args})},
{"key": "resources_loading_time",
"data": dashboard.get_resources_loading_time(project_id=projectId, **{**data, **args})},
{"key": "resources_by_party",
"data": dashboard.get_resources_by_party(project_id=projectId, **{**data, **args})},
{"key": "resource_type_vs_response_end",
"data": dashboard.resource_type_vs_response_end(project_id=projectId, **{**data, **args})},
{"key": "resources_vs_visually_complete",
"data": dashboard.get_resources_vs_visually_complete(project_id=projectId, **{**data, **args})},
{"key": "resources_count_by_type",
"data": dashboard.get_resources_count_by_type(project_id=projectId, **{**data, **args})}
]}

906
api/chalicelib/core/dashboard.py
View file

File diff suppressed because it is too large Load diff

34
scripts/helm/db/init_dbs/postgresql/1.2.0.sql Normal file
View file

@ -0,0 +1,34 @@
BEGIN;
CREATE INDEX pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0;
CREATE INDEX pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time > 0;
CREATE INDEX pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0;
CREATE INDEX pages_ttfb_idx ON events.pages (ttfb) WHERE ttfb > 0;
CREATE INDEX pages_time_to_interactive_idx ON events.pages (time_to_interactive) WHERE time_to_interactive > 0;
CREATE INDEX sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0;
CREATE INDEX pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL;
DROP INDEX events.resources_type_idx;
CREATE INDEX resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL;
CREATE INDEX pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL;
CREATE INDEX pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR first_paint_time > 0 OR
dom_content_loaded_time > 0 OR ttfb > 0 OR
time_to_interactive > 0;
CREATE INDEX resources_session_id_timestamp_idx ON events.resources (session_id, timestamp);
DROP INDEX events.resources_timestamp_idx;
CREATE INDEX resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type);
CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL;
CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL;
CREATE INDEX resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch';
CREATE INDEX resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE;
DROP INDEX events.resources_success_idx;
CREATE INDEX resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script');
DROP INDEX events.errors_timestamp_idx;
CREATE INDEX errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id);
CREATE INDEX errors_project_id_error_id_js_exception_idx ON public.errors (project_id, error_id) WHERE source = 'js_exception';
CREATE INDEX errors_error_id_timestamp_idx ON events.errors (error_id, timestamp);
CREATE INDEX errors_project_id_error_id_idx ON public.errors (project_id, error_id);
CREATE INDEX errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id);
CREATE INDEX resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img';
CREATE INDEX resources_timestamp_session_id_idx ON events.resources (timestamp, session_id);
CREATE INDEX errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception';
CREATE INDEX errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id);
COMMIT;

35
scripts/helm/db/init_dbs/postgresql/init_schema.sql
View file

@ -57,7 +57,7 @@ CREATE TABLE users
"role": "dev",
"dashboard": {
"cpu": true,
"fps": false,
"fps": false,
"avgCpu": true,
"avgFps": true,
"errors": true,
@ -408,6 +408,9 @@ CREATE INDEX errors_message_gin_idx ON public.errors USING GIN (message gin_trgm
CREATE INDEX errors_name_gin_idx ON public.errors USING GIN (name gin_trgm_ops);
CREATE INDEX errors_project_id_idx ON public.errors (project_id);
CREATE INDEX errors_project_id_status_idx ON public.errors (project_id, status);
CREATE INDEX errors_project_id_error_id_js_exception_idx ON public.errors (project_id, error_id) WHERE source = 'js_exception';
CREATE INDEX errors_project_id_error_id_idx ON public.errors (project_id, error_id);
CREATE INDEX errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception';
CREATE TABLE user_favorite_errors
(
@ -513,6 +516,7 @@ CREATE INDEX ON sessions (project_id, user_country);
CREATE INDEX ON sessions (project_id, user_browser);
CREATE INDEX sessions_start_ts_idx ON public.sessions (start_ts) WHERE duration > 0;
CREATE INDEX sessions_project_id_idx ON public.sessions (project_id) WHERE duration > 0;
CREATE INDEX sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0;
ALTER TABLE public.sessions
ADD CONSTRAINT web_browser_constraint CHECK ( (sessions.platform = 'web' AND sessions.user_browser NOTNULL) OR
@ -656,11 +660,18 @@ CREATE INDEX pages_path_idx ON events.pages (path);
CREATE INDEX pages_visually_complete_idx ON events.pages (visually_complete) WHERE visually_complete > 0;
CREATE INDEX pages_dom_building_time_idx ON events.pages (dom_building_time) WHERE dom_building_time > 0;
CREATE INDEX pages_load_time_idx ON events.pages (load_time) WHERE load_time > 0;
CREATE INDEX pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time>0;
CREATE INDEX pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time>0;
CREATE INDEX pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0;
CREATE INDEX pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time > 0;
CREATE INDEX pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0;
CREATE INDEX pages_ttfb_idx ON events.pages (ttfb) WHERE ttfb > 0;
CREATE INDEX pages_time_to_interactive_idx ON events.pages (time_to_interactive) WHERE time_to_interactive > 0;
CREATE INDEX pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL;
CREATE INDEX pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL;
CREATE INDEX pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR first_paint_time > 0 OR
dom_content_loaded_time > 0 OR ttfb > 0 OR
time_to_interactive > 0;
CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL;
CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL;
CREATE TABLE events.clicks
@ -700,8 +711,10 @@ CREATE TABLE events.errors
PRIMARY KEY (session_id, message_id)
);
CREATE INDEX ON events.errors (session_id);
CREATE INDEX ON events.errors (timestamp);
CREATE INDEX errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id);
CREATE INDEX errors_error_id_timestamp_idx ON events.errors (error_id, timestamp);
CREATE INDEX errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id);
CREATE INDEX errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id);
CREATE TABLE events.graphql
(
@ -749,8 +762,6 @@ CREATE TABLE events.resources
PRIMARY KEY (session_id, message_id)
);
CREATE INDEX ON events.resources (session_id);
CREATE INDEX ON events.resources (timestamp);
CREATE INDEX ON events.resources (success);
CREATE INDEX ON events.resources (status);
CREATE INDEX ON events.resources (type);
CREATE INDEX ON events.resources (duration) WHERE duration > 0;
@ -760,8 +771,14 @@ CREATE INDEX resources_url_gin_idx ON events.resources USING GIN (url gin_trgm_o
CREATE INDEX resources_url_idx ON events.resources (url);
CREATE INDEX resources_url_hostpath_gin_idx ON events.resources USING GIN (url_hostpath gin_trgm_ops);
CREATE INDEX resources_url_hostpath_idx ON events.resources (url_hostpath);
CREATE INDEX resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL;
CREATE INDEX resources_session_id_timestamp_idx ON events.resources (session_id, timestamp);
CREATE INDEX resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type);
CREATE INDEX resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch';
CREATE INDEX resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE;
CREATE INDEX resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script');
CREATE INDEX resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img';
CREATE INDEX resources_timestamp_session_id_idx ON events.resources (timestamp, session_id);
CREATE TABLE events.performance
(