From 9675da07e1aa6bc837ccb2c943564e81858adc72 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 22 Mar 2022 17:16:08 +0100 Subject: [PATCH 001/125] feat(api): dashboards 1/5 --- api/chalicelib/core/dashboards2.py | 60 +++++++++++++++++++ api/routers/subs/dashboard.py | 27 ++++++++- api/schemas.py | 9 +++ .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 43 +++++++++++++ 4 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 api/chalicelib/core/dashboards2.py create mode 100644 scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py new file mode 100644 index 000000000..5f2bced2d --- /dev/null +++ b/api/chalicelib/core/dashboards2.py @@ -0,0 +1,60 @@ +import schemas +from chalicelib.utils import helper +from chalicelib.utils import pg_client + + +def create_dashboard(project_id, user_id, data: schemas.CreateDashboardSchema): + with pg_client.PostgresClient() as cur: + pg_query = f"""INSERT INTO dashboards(project_id, user_id, name, is_public, is_pinned) + VALUES(%(projectId)s, %(userId)s, %(name)s, %(is_public)s, %(is_pinned)s) + RETURNING *;""" + params = {"userId": user_id, "projectId": project_id, **data.dict()} + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def get_dashboards(project_id, user_id): + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT * + FROM dashboards + WHERE deleted_at ISNULL + AND project_id = %(projectId)s + AND (user_id = %(userId)s OR is_public);""" + params = {"userId": user_id, "projectId": project_id} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + return helper.list_to_camel_case(rows) + + +def get_dashboard(project_id, user_id, dashboard_id): + with pg_client.PostgresClient() as cur: + pg_query = """SELECT dashboards.*, all_widgets.* + FROM dashboards + LEFT JOIN LATERAL (SELECT COALESCE(ARRAY_AGG(widgets), '{}') AS widgets + FROM widgets + INNER JOIN dashboard_widgets USING (widget_id) + WHERE dashboard_widgets.dashboard_id = dashboards.dashboard_id + AND widgets.deleted_at ISNULL + AND (widgets.project_id ISNULL OR widgets.project_id = %(projectId)s) + ) AS all_widgets ON (TRUE) + WHERE dashboards.deleted_at ISNULL + AND dashboards.project_id = %(projectId)s + AND dashboard_id = %(dashboard_id)s + AND (dashboards.user_id = %(userId)s OR is_public);""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id} + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def get_widgets(project_id): + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT * + FROM widgets + WHERE deleted_at ISNULL + AND project_id = %(projectId)s;""" + params = {"projectId": project_id} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + return helper.list_to_camel_case(rows) diff --git a/api/routers/subs/dashboard.py b/api/routers/subs/dashboard.py index 169893693..1f5e827e4 100644 --- a/api/routers/subs/dashboard.py +++ b/api/routers/subs/dashboard.py @@ -1,9 +1,10 @@ -from fastapi import Body +from fastapi import Body, Depends import schemas -from chalicelib.core import dashboard +from chalicelib.core import dashboard, dashboards2 from chalicelib.core import metadata from chalicelib.utils import helper +from or_dependencies import OR_context from routers.base import get_routers public_app, app, app_apikey = get_routers() @@ -344,3 +345,25 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body *helper.explode_widget(dashboard.get_avg_cpu(project_id=projectId, **data.dict())), *helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **data.dict())), ]} + + +@app.post('/{projectId}/dashboards', tags=["dashboard", "metrics"]) +@app.put('/{projectId}/dashboards', tags=["dashboard", "metrics"]) +def create_dashboards(projectId: int, data: schemas.CreateDashboardSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.create_dashboard(project_id=projectId, user_id=context.user_id, data=data)} + + +@app.get('/{projectId}/dashboards', tags=["dashboard", "metrics"]) +def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.get_dashboards(project_id=projectId, user_id=context.user_id)} + + +@app.get('/{projectId}/dashboards/{dashboardId}', tags=["dashboard", "metrics"]) +def get_dashboards(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.get_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId)} + + +@app.get('/{projectId}/widgets', tags=["dashboard", "metrics"]) +def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.get_widgets(project_id=projectId)} diff --git a/api/schemas.py b/api/schemas.py index 77cb78c05..61b1f4056 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -875,3 +875,12 @@ class UpdateCustomMetricsStatusSchema(BaseModel): class SavedSearchSchema(FunnelSchema): filter: FlatSessionsSearchPayloadSchema = Field([]) + + +class CreateDashboardSchema(BaseModel): + name: str = Field(...) + is_public: bool = Field(default=False) + is_pinned: bool = Field(default=False) + + class Config: + alias_generator = attribute_to_camel_case diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql new file mode 100644 index 000000000..a8bb8aafd --- /dev/null +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -0,0 +1,43 @@ +BEGIN; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.5.5' +$$ LANGUAGE sql IMMUTABLE; + + +CREATE TABLE dashboards +( + dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT TRUE, + is_pinned boolean NOT NULL DEFAULT FALSE, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + deleted_at timestamp NULL DEFAULT NULL +); + + +CREATE TABLE widgets +( + widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_template boolean NOT NULL DEFAULT FALSE, + predefined boolean NOT NULL DEFAULT FALSE, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + deleted_at timestamp NULL DEFAULT NULL +); + +CREATE TABLE dashboard_widgets +( + dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, + widget_id integer NOT NULL REFERENCES widgets (widget_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + configuration jsonb NOT NULL DEFAULT '{}'::jsonb +); + +COMMIT; \ No newline at end of file From fb3c6a739a181a7f3fd98fff5cd5a0990c79fe27 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 22 Mar 2022 17:51:25 +0100 Subject: [PATCH 002/125] feat(api): dashboards insights --- api/app.py | 4 ++-- api/chalicelib/core/insights.py | 7 ++----- ee/api/app.py | 4 ++-- ee/api/chalicelib/core/insights.py | 32 +++++++++++++++--------------- 4 files changed, 22 insertions(+), 25 deletions(-) diff --git a/api/app.py b/api/app.py index d261dadac..4d3697e92 100644 --- a/api/app.py +++ b/api/app.py @@ -12,7 +12,7 @@ from routers import core, core_dynamic from routers.app import v1_api from routers.crons import core_crons from routers.crons import core_dynamic_crons -from routers.subs import dashboard +from routers.subs import dashboard, insights app = FastAPI() @@ -54,7 +54,7 @@ app.include_router(core_dynamic.public_app) app.include_router(core_dynamic.app) app.include_router(core_dynamic.app_apikey) app.include_router(dashboard.app) -# app.include_router(insights.app) +app.include_router(insights.app) app.include_router(v1_api.app_apikey) Schedule = AsyncIOScheduler() diff --git a/api/chalicelib/core/insights.py b/api/chalicelib/core/insights.py index 08adfd3ca..5b3894606 100644 --- a/api/chalicelib/core/insights.py +++ b/api/chalicelib/core/insights.py @@ -1,11 +1,8 @@ import schemas -from chalicelib.core import sessions_metas +from chalicelib.core.dashboard import __get_constraints, __get_constraint_values from chalicelib.utils import helper, dev from chalicelib.utils import pg_client from chalicelib.utils.TimeUTC import TimeUTC -from chalicelib.utils.metrics_helper import __get_step_size -import math -from chalicelib.core.dashboard import __get_constraints, __get_constraint_values def __transform_journey(rows): @@ -930,4 +927,4 @@ def search(text, feature_type, project_id, platform=None): rows = cur.fetchall() else: return [] - return [helper.dict_to_camel_case(row) for row in rows] \ No newline at end of file + return [helper.dict_to_camel_case(row) for row in rows] diff --git a/ee/api/app.py b/ee/api/app.py index fdf7f60b8..deae9d78d 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -14,7 +14,7 @@ from routers import core, core_dynamic, ee, saml from routers.app import v1_api, v1_api_ee from routers.crons import core_crons from routers.crons import core_dynamic_crons -from routers.subs import dashboard +from routers.subs import dashboard, insights app = FastAPI() @@ -65,7 +65,7 @@ app.include_router(saml.public_app) app.include_router(saml.app) app.include_router(saml.app_apikey) app.include_router(dashboard.app) -# app.include_router(insights.app) +app.include_router(insights.app) app.include_router(v1_api.app_apikey) app.include_router(v1_api_ee.app_apikey) diff --git a/ee/api/chalicelib/core/insights.py b/ee/api/chalicelib/core/insights.py index 387029fd4..ff9d4dad4 100644 --- a/ee/api/chalicelib/core/insights.py +++ b/ee/api/chalicelib/core/insights.py @@ -1,9 +1,9 @@ -from chalicelib.core import sessions_metas -from chalicelib.utils import helper, dev -from chalicelib.utils import ch_client -from chalicelib.utils.TimeUTC import TimeUTC -from chalicelib.core.dashboard import __get_constraint_values, __complete_missing_steps +import schemas from chalicelib.core.dashboard import __get_basic_constraints, __get_meta_constraint +from chalicelib.core.dashboard import __get_constraint_values, __complete_missing_steps +from chalicelib.utils import ch_client +from chalicelib.utils import helper, dev +from chalicelib.utils.TimeUTC import TimeUTC def __transform_journey(rows): @@ -42,7 +42,7 @@ def journey(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp= elif f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append(f"sessions_metadata.project_id = %(project_id)s") meta_condition.append(f"sessions_metadata.datetime >= toDateTime(%(startTimestamp)s / 1000)") @@ -303,7 +303,7 @@ def feature_retention(project_id, startTimestamp=TimeUTC.now(delta_days=-70), en elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.user_id IS NOT NULL") meta_condition.append("not empty(sessions_metadata.user_id)") @@ -404,7 +404,7 @@ def feature_acquisition(project_id, startTimestamp=TimeUTC.now(delta_days=-70), elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.user_id IS NOT NULL") meta_condition.append("not empty(sessions_metadata.user_id)") @@ -512,7 +512,7 @@ def feature_popularity_frequency(project_id, startTimestamp=TimeUTC.now(delta_da if f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.user_id IS NOT NULL") meta_condition.append("not empty(sessions_metadata.user_id)") @@ -586,7 +586,7 @@ def feature_adoption(project_id, startTimestamp=TimeUTC.now(delta_days=-70), end elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.user_id IS NOT NULL") meta_condition.append("not empty(sessions_metadata.user_id)") @@ -672,7 +672,7 @@ def feature_adoption_top_users(project_id, startTimestamp=TimeUTC.now(delta_days elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("user_id IS NOT NULL") meta_condition.append("not empty(sessions_metadata.user_id)") @@ -742,7 +742,7 @@ def feature_adoption_daily_usage(project_id, startTimestamp=TimeUTC.now(delta_da elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.project_id = %(project_id)s") meta_condition.append("sessions_metadata.datetime >= toDateTime(%(startTimestamp)s/1000)") @@ -807,7 +807,7 @@ def feature_intensity(project_id, startTimestamp=TimeUTC.now(delta_days=-70), en if f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.project_id = %(project_id)s") meta_condition.append("sessions_metadata.datetime >= toDateTime(%(startTimestamp)s/1000)") @@ -847,7 +847,7 @@ def users_active(project_id, startTimestamp=TimeUTC.now(delta_days=-70), endTime for f in filters: if f["type"] == "PERIOD" and f["value"] in ["DAY", "WEEK"]: period = f["value"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") extra_values["user_id"] = f["value"] period_function = PERIOD_TO_FUNCTION[period] @@ -940,7 +940,7 @@ def users_slipping(project_id, startTimestamp=TimeUTC.now(delta_days=-70), endTi elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.project_id = %(project_id)s") meta_condition.append("sessions_metadata.datetime >= toDateTime(%(startTimestamp)s/1000)") @@ -1044,4 +1044,4 @@ def search(text, feature_type, project_id, platform=None): rows = ch.execute(ch_query, params) else: return [] - return [helper.dict_to_camel_case(row) for row in rows] \ No newline at end of file + return [helper.dict_to_camel_case(row) for row in rows] From cc08060e308cec2bbec6f9e4d07aeccb7b28c812 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 25 Mar 2022 20:18:31 +0100 Subject: [PATCH 003/125] feat(api): dashboard 2/5 --- api/app.py | 3 +- api/chalicelib/core/dashboards2.py | 65 +++++++++++---- api/chalicelib/core/templates.py | 19 +++++ api/requirements.txt | 2 +- api/routers/core.py | 72 ---------------- api/routers/subs/dashboard.py | 27 +----- api/routers/subs/metrics.py | 129 +++++++++++++++++++++++++++++ api/schemas.py | 17 ++++ 8 files changed, 217 insertions(+), 117 deletions(-) create mode 100644 api/chalicelib/core/templates.py create mode 100644 api/routers/subs/metrics.py diff --git a/api/app.py b/api/app.py index 4d3697e92..b1ddca682 100644 --- a/api/app.py +++ b/api/app.py @@ -12,7 +12,7 @@ from routers import core, core_dynamic from routers.app import v1_api from routers.crons import core_crons from routers.crons import core_dynamic_crons -from routers.subs import dashboard, insights +from routers.subs import dashboard, insights, metrics app = FastAPI() @@ -54,6 +54,7 @@ app.include_router(core_dynamic.public_app) app.include_router(core_dynamic.app) app.include_router(core_dynamic.app_apikey) app.include_router(dashboard.app) +app.include_router(metrics.app) app.include_router(insights.app) app.include_router(v1_api.app_apikey) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 5f2bced2d..782532718 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -1,3 +1,5 @@ +import json + import schemas from chalicelib.utils import helper from chalicelib.utils import pg_client @@ -29,32 +31,59 @@ def get_dashboards(project_id, user_id): def get_dashboard(project_id, user_id, dashboard_id): with pg_client.PostgresClient() as cur: - pg_query = """SELECT dashboards.*, all_widgets.* + pg_query = """SELECT dashboards.*, all_template_widgets.widgets AS template_widgets, all_metric_widgets.widgets AS metric_widgets FROM dashboards - LEFT JOIN LATERAL (SELECT COALESCE(ARRAY_AGG(widgets), '{}') AS widgets - FROM widgets - INNER JOIN dashboard_widgets USING (widget_id) + LEFT JOIN LATERAL (SELECT COALESCE(JSONB_AGG(templates), '[]'::jsonb) AS widgets + FROM templates + INNER JOIN dashboard_widgets USING (template_id) WHERE dashboard_widgets.dashboard_id = dashboards.dashboard_id - AND widgets.deleted_at ISNULL - AND (widgets.project_id ISNULL OR widgets.project_id = %(projectId)s) - ) AS all_widgets ON (TRUE) + ) AS all_template_widgets ON (TRUE) + LEFT JOIN LATERAL (SELECT COALESCE(JSONB_AGG(raw_metrics), '[]') AS widgets + FROM (SELECT metrics.*, metric_series.series + FROM metrics + INNER JOIN dashboard_widgets USING (metric_id) + LEFT JOIN LATERAL (SELECT JSONB_AGG(metric_series.* ORDER BY index) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + WHERE dashboard_widgets.dashboard_id = dashboards.dashboard_id + AND metrics.deleted_at ISNULL + AND metrics.project_id = %(projectId)s) AS raw_metrics + ) AS all_metric_widgets ON (TRUE) WHERE dashboards.deleted_at ISNULL AND dashboards.project_id = %(projectId)s AND dashboard_id = %(dashboard_id)s AND (dashboards.user_id = %(userId)s OR is_public);""" params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id} + print(cur.mogrify(pg_query, params)) + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + row["widgets"] = row.pop("template_widgets") + row.pop("metric_widgets") + return helper.dict_to_camel_case(row) + + +def add_widget(project_id, user_id, dashboard_id, data: schemas.AddWidgetToDashboardPayloadSchema): + ref_key = "metric_id" + if data.template_id is not None: + ref_key = "template_id" + with pg_client.PostgresClient() as cur: + pg_query = f"""INSERT INTO dashboard_widgets(dashboard_id, {ref_key}, user_id, configuration, name) + VALUES (%(dashboard_id)s, %({ref_key})s, %(userId)s, %(configuration)s::jsonb, %(name)s) + RETURNING *;""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, **data.dict()} + params["configuration"] = json.dumps(params.get("configuration", {})) cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() return helper.dict_to_camel_case(row) - -def get_widgets(project_id): - with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT * - FROM widgets - WHERE deleted_at ISNULL - AND project_id = %(projectId)s;""" - params = {"projectId": project_id} - cur.execute(cur.mogrify(pg_query, params)) - rows = cur.fetchall() - return helper.list_to_camel_case(rows) +# def get_widgets(project_id): +# with pg_client.PostgresClient() as cur: +# pg_query = f"""SELECT * +# FROM widgets +# WHERE deleted_at ISNULL +# AND project_id = %(projectId)s;""" +# params = {"projectId": project_id} +# cur.execute(cur.mogrify(pg_query, params)) +# rows = cur.fetchall() +# return helper.list_to_camel_case(rows) diff --git a/api/chalicelib/core/templates.py b/api/chalicelib/core/templates.py new file mode 100644 index 000000000..8141bdd35 --- /dev/null +++ b/api/chalicelib/core/templates.py @@ -0,0 +1,19 @@ +from chalicelib.utils import helper +from chalicelib.utils import pg_client + +CATEGORY_DESCRIPTION = { + 'categ1': 'lorem', +} + + +def get_templates(): + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT category, jsonb_agg(templates ORDER BY name) AS widgets + FROM templates + GROUP BY category + ORDER BY category;""" + cur.execute(pg_query) + rows = cur.fetchall() + for r in rows: + r["description"] = CATEGORY_DESCRIPTION.get(r["category"], "") + return helper.list_to_camel_case(rows) diff --git a/api/requirements.txt b/api/requirements.txt index 4af962f4f..e5672d80a 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -8,7 +8,7 @@ jira==2.0.0 -fastapi==0.74.1 +fastapi==0.75.0 uvicorn[standard]==0.17.5 python-decouple==3.6 pydantic[email]==1.8.2 diff --git a/api/routers/core.py b/api/routers/core.py index 97a749429..53fdba667 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -1065,78 +1065,6 @@ def change_client_password(data: schemas.EditUserPasswordSchema = Body(...), user_id=context.user_id) -@app.post('/{projectId}/custom_metrics/try', tags=["customMetrics"]) -@app.put('/{projectId}/custom_metrics/try', tags=["customMetrics"]) -def try_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.merged_live(project_id=projectId, data=data)} - - -@app.post('/{projectId}/custom_metrics', tags=["customMetrics"]) -@app.put('/{projectId}/custom_metrics', tags=["customMetrics"]) -def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - return custom_metrics.create(project_id=projectId, user_id=context.user_id, data=data) - - -@app.get('/{projectId}/custom_metrics', tags=["customMetrics"]) -def get_custom_metrics(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.get_all(project_id=projectId, user_id=context.user_id)} - - -@app.get('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) -def get_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): - data = custom_metrics.get(project_id=projectId, user_id=context.user_id, metric_id=metric_id) - if data is None: - return {"errors": ["custom metric not found"]} - return {"data": data} - - -@app.post('/{projectId}/custom_metrics/{metric_id}/sessions', tags=["customMetrics"]) -def get_custom_metric_sessions(projectId: int, metric_id: int, - data: schemas.CustomMetricSessionsPayloadSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - data = custom_metrics.get_sessions(project_id=projectId, user_id=context.user_id, metric_id=metric_id, data=data) - if data is None: - return {"errors": ["custom metric not found"]} - return {"data": data} - - -@app.post('/{projectId}/custom_metrics/{metric_id}/chart', tags=["customMetrics"]) -def get_custom_metric_chart(projectId: int, metric_id: int, data: schemas.CustomMetricChartPayloadSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - data = custom_metrics.make_chart(project_id=projectId, user_id=context.user_id, metric_id=metric_id, - data=data) - if data is None: - return {"errors": ["custom metric not found"]} - return {"data": data} - - -@app.post('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) -@app.put('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) -def update_custom_metric(projectId: int, metric_id: int, data: schemas.UpdateCustomMetricsSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - data = custom_metrics.update(project_id=projectId, user_id=context.user_id, metric_id=metric_id, data=data) - if data is None: - return {"errors": ["custom metric not found"]} - return {"data": data} - - -@app.post('/{projectId}/custom_metrics/{metric_id}/status', tags=["customMetrics"]) -@app.put('/{projectId}/custom_metrics/{metric_id}/status', tags=["customMetrics"]) -def update_custom_metric_state(projectId: int, metric_id: int, - data: schemas.UpdateCustomMetricsStatusSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - return { - "data": custom_metrics.change_state(project_id=projectId, user_id=context.user_id, metric_id=metric_id, - status=data.active)} - - -@app.delete('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) -def delete_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.delete(project_id=projectId, user_id=context.user_id, metric_id=metric_id)} - - @app.post('/{projectId}/saved_search', tags=["savedSearch"]) @app.put('/{projectId}/saved_search', tags=["savedSearch"]) def add_saved_search(projectId: int, data: schemas.SavedSearchSchema = Body(...), diff --git a/api/routers/subs/dashboard.py b/api/routers/subs/dashboard.py index 1f5e827e4..169893693 100644 --- a/api/routers/subs/dashboard.py +++ b/api/routers/subs/dashboard.py @@ -1,10 +1,9 @@ -from fastapi import Body, Depends +from fastapi import Body import schemas -from chalicelib.core import dashboard, dashboards2 +from chalicelib.core import dashboard from chalicelib.core import metadata from chalicelib.utils import helper -from or_dependencies import OR_context from routers.base import get_routers public_app, app, app_apikey = get_routers() @@ -345,25 +344,3 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body *helper.explode_widget(dashboard.get_avg_cpu(project_id=projectId, **data.dict())), *helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **data.dict())), ]} - - -@app.post('/{projectId}/dashboards', tags=["dashboard", "metrics"]) -@app.put('/{projectId}/dashboards', tags=["dashboard", "metrics"]) -def create_dashboards(projectId: int, data: schemas.CreateDashboardSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - return {"data": dashboards2.create_dashboard(project_id=projectId, user_id=context.user_id, data=data)} - - -@app.get('/{projectId}/dashboards', tags=["dashboard", "metrics"]) -def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": dashboards2.get_dashboards(project_id=projectId, user_id=context.user_id)} - - -@app.get('/{projectId}/dashboards/{dashboardId}', tags=["dashboard", "metrics"]) -def get_dashboards(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": dashboards2.get_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId)} - - -@app.get('/{projectId}/widgets', tags=["dashboard", "metrics"]) -def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": dashboards2.get_widgets(project_id=projectId)} diff --git a/api/routers/subs/metrics.py b/api/routers/subs/metrics.py new file mode 100644 index 000000000..9c0bb64ba --- /dev/null +++ b/api/routers/subs/metrics.py @@ -0,0 +1,129 @@ +from fastapi import Body, Depends + +import schemas +from chalicelib.core import dashboards2, templates, custom_metrics +from or_dependencies import OR_context +from routers.base import get_routers + +public_app, app, app_apikey = get_routers() + + +@app.post('/{projectId}/dashboards', tags=["dashboard", "metrics"]) +@app.put('/{projectId}/dashboards', tags=["dashboard", "metrics"]) +def create_dashboards(projectId: int, data: schemas.CreateDashboardSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.create_dashboard(project_id=projectId, user_id=context.user_id, data=data)} + + +@app.get('/{projectId}/dashboards', tags=["dashboard", "metrics"]) +def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.get_dashboards(project_id=projectId, user_id=context.user_id)} + + +@app.get('/{projectId}/dashboards/{dashboardId}', tags=["dashboard", "metrics"]) +def get_dashboards(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.get_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId)} + + +@app.post('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard", "metrics"]) +@app.put('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard", "metrics"]) +def add_widget_to_dashboards(projectId: int, dashboardId: int, + data: schemas.AddWidgetToDashboardPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.add_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, + data=data)} + + +# @app.get('/{projectId}/widgets', tags=["dashboard", "metrics"]) +# def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): +# return {"data": dashboards2.get_widgets(project_id=projectId)} + + +@app.get('/{projectId}/metrics/templates', tags=["dashboard", "metrics"]) +def get_templates(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": templates.get_templates()} + + +@app.post('/{projectId}/metrics/try', tags=["dashboard"]) +@app.put('/{projectId}/metrics/try', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics/try', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics/try', tags=["customMetrics"]) +def try_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.merged_live(project_id=projectId, data=data)} + + +@app.post('/{projectId}/metrics', tags=["dashboard"]) +@app.put('/{projectId}/metrics', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics', tags=["customMetrics"]) +def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return custom_metrics.create(project_id=projectId, user_id=context.user_id, data=data) + + +@app.get('/{projectId}/metrics', tags=["dashboard"]) +@app.get('/{projectId}/custom_metrics', tags=["customMetrics"]) +def get_custom_metrics(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.get_all(project_id=projectId, user_id=context.user_id)} + + +@app.get('/{projectId}/metrics/{metric_id}', tags=["dashboard"]) +@app.get('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def get_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): + data = custom_metrics.get(project_id=projectId, user_id=context.user_id, metric_id=metric_id) + if data is None: + return {"errors": ["custom metric not found"]} + return {"data": data} + + +@app.post('/{projectId}/metrics/{metric_id}/sessions', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics/{metric_id}/sessions', tags=["customMetrics"]) +def get_custom_metric_sessions(projectId: int, metric_id: int, + data: schemas.CustomMetricSessionsPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + data = custom_metrics.get_sessions(project_id=projectId, user_id=context.user_id, metric_id=metric_id, data=data) + if data is None: + return {"errors": ["custom metric not found"]} + return {"data": data} + + +@app.post('/{projectId}/metrics/{metric_id}/chart', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics/{metric_id}/chart', tags=["customMetrics"]) +def get_custom_metric_chart(projectId: int, metric_id: int, data: schemas.CustomMetricChartPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + data = custom_metrics.make_chart(project_id=projectId, user_id=context.user_id, metric_id=metric_id, + data=data) + if data is None: + return {"errors": ["custom metric not found"]} + return {"data": data} + + +@app.post('/{projectId}/metrics/{metric_id}', tags=["dashboard"]) +@app.put('/{projectId}/metrics/{metric_id}', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def update_custom_metric(projectId: int, metric_id: int, data: schemas.UpdateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + data = custom_metrics.update(project_id=projectId, user_id=context.user_id, metric_id=metric_id, data=data) + if data is None: + return {"errors": ["custom metric not found"]} + return {"data": data} + + +@app.post('/{projectId}/metrics/{metric_id}/status', tags=["dashboard"]) +@app.put('/{projectId}/metrics/{metric_id}/status', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics/{metric_id}/status', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics/{metric_id}/status', tags=["customMetrics"]) +def update_custom_metric_state(projectId: int, metric_id: int, + data: schemas.UpdateCustomMetricsStatusSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return { + "data": custom_metrics.change_state(project_id=projectId, user_id=context.user_id, metric_id=metric_id, + status=data.active)} + + +@app.delete('/{projectId}/metrics/{metric_id}', tags=["dashboard"]) +@app.delete('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def delete_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.delete(project_id=projectId, user_id=context.user_id, metric_id=metric_id)} diff --git a/api/schemas.py b/api/schemas.py index 61b1f4056..bb111d61b 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -884,3 +884,20 @@ class CreateDashboardSchema(BaseModel): class Config: alias_generator = attribute_to_camel_case + + +class AddWidgetToDashboardPayloadSchema(BaseModel): + template_id: Optional[int] = Field(default=None) + metric_id: Optional[int] = Field(default=None) + name: Optional[str] = Field(default=None) + configuration: dict = Field(default={}) + + @root_validator + def validator(cls, values): + assert bool(values.get("template_id") is not None) != bool(values.get("metric_id") is not None), \ + f"templateId or metricId should be provided, but not both at the same time" + + return values + + class Config: + alias_generator = attribute_to_camel_case From 8e1f93630ce44c0f7606761e47860ee0212824df Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 28 Mar 2022 19:55:06 +0200 Subject: [PATCH 004/125] feat(api): dashboard re-think 1/5 --- api/app.py | 3 +- api/chalicelib/core/dashboards2.py | 24 +++-- api/chalicelib/core/templates.py | 9 +- api/routers/app/__init__.py | 0 api/routers/subs/metrics.py | 18 +++- api/routers/{app => subs}/v1_api.py | 0 .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 102 +++++++++++++----- .../db/init_dbs/postgresql/init_schema.sql | 14 ++- 8 files changed, 119 insertions(+), 51 deletions(-) delete mode 100644 api/routers/app/__init__.py rename api/routers/{app => subs}/v1_api.py (100%) diff --git a/api/app.py b/api/app.py index b1ddca682..ae7f051bf 100644 --- a/api/app.py +++ b/api/app.py @@ -9,10 +9,9 @@ from starlette.responses import StreamingResponse from chalicelib.utils import helper from chalicelib.utils import pg_client from routers import core, core_dynamic -from routers.app import v1_api from routers.crons import core_crons from routers.crons import core_dynamic_crons -from routers.subs import dashboard, insights, metrics +from routers.subs import dashboard, insights, metrics, v1_api app = FastAPI() diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 782532718..c36293059 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -77,13 +77,17 @@ def add_widget(project_id, user_id, dashboard_id, data: schemas.AddWidgetToDashb row = cur.fetchone() return helper.dict_to_camel_case(row) -# def get_widgets(project_id): -# with pg_client.PostgresClient() as cur: -# pg_query = f"""SELECT * -# FROM widgets -# WHERE deleted_at ISNULL -# AND project_id = %(projectId)s;""" -# params = {"projectId": project_id} -# cur.execute(cur.mogrify(pg_query, params)) -# rows = cur.fetchall() -# return helper.list_to_camel_case(rows) +def remove_widget(project_id, user_id, dashboard_id,widget_id): + ref_key = "metric_id" + if data.template_id is not None: + ref_key = "template_id" + with pg_client.PostgresClient() as cur: + pg_query = f"""INSERT INTO dashboard_widgets(dashboard_id, {ref_key}, user_id, configuration, name) + VALUES (%(dashboard_id)s, %({ref_key})s, %(userId)s, %(configuration)s::jsonb, %(name)s) + RETURNING *;""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, **data.dict()} + params["configuration"] = json.dumps(params.get("configuration", {})) + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + diff --git a/api/chalicelib/core/templates.py b/api/chalicelib/core/templates.py index 8141bdd35..8e42668d3 100644 --- a/api/chalicelib/core/templates.py +++ b/api/chalicelib/core/templates.py @@ -6,12 +6,13 @@ CATEGORY_DESCRIPTION = { } -def get_templates(): +def get_templates(project_id, user_id): with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT category, jsonb_agg(templates ORDER BY name) AS widgets - FROM templates + pg_query = cur.mogrify(f"""SELECT category, jsonb_agg(metrics ORDER BY name) AS widgets + FROM metrics + WHERE project_id ISNULL OR (project_id = %(project_id)s AND (is_public OR user_id= %(userId)s)) GROUP BY category - ORDER BY category;""" + ORDER BY category;""", {"project_id": project_id, "userId": user_id}) cur.execute(pg_query) rows = cur.fetchall() for r in rows: diff --git a/api/routers/app/__init__.py b/api/routers/app/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/api/routers/subs/metrics.py b/api/routers/subs/metrics.py index 9c0bb64ba..56b486977 100644 --- a/api/routers/subs/metrics.py +++ b/api/routers/subs/metrics.py @@ -21,15 +21,23 @@ def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_ @app.get('/{projectId}/dashboards/{dashboardId}', tags=["dashboard", "metrics"]) -def get_dashboards(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): +def get_dashboard(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): return {"data": dashboards2.get_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId)} @app.post('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard", "metrics"]) @app.put('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard", "metrics"]) -def add_widget_to_dashboards(projectId: int, dashboardId: int, - data: schemas.AddWidgetToDashboardPayloadSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): +def add_widget_to_dashboard(projectId: int, dashboardId: int, + data: schemas.AddWidgetToDashboardPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.add_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, + data=data)} + + +@app.delete('/{projectId}/dashboards/{dashboardId}/metrics/{metricId}', tags=["dashboard", "metrics"]) +def remove_widget_from_dashboard(projectId: int, dashboardId: int, metricId: int, + data: schemas.AddWidgetToDashboardPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): return {"data": dashboards2.add_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, data=data)} @@ -41,7 +49,7 @@ def add_widget_to_dashboards(projectId: int, dashboardId: int, @app.get('/{projectId}/metrics/templates', tags=["dashboard", "metrics"]) def get_templates(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": templates.get_templates()} + return {"data": templates.get_templates(project_id=projectId, user_id=context.user_id)} @app.post('/{projectId}/metrics/try', tags=["dashboard"]) diff --git a/api/routers/app/v1_api.py b/api/routers/subs/v1_api.py similarity index 100% rename from api/routers/app/v1_api.py rename to api/routers/subs/v1_api.py diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index a8bb8aafd..d0d31ecce 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -6,38 +6,86 @@ SELECT 'v1.5.5' $$ LANGUAGE sql IMMUTABLE; -CREATE TABLE dashboards +-- CREATE TABLE dashboards +-- ( +-- dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, +-- project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, +-- user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, +-- name text NOT NULL, +-- is_public boolean NOT NULL DEFAULT TRUE, +-- is_pinned boolean NOT NULL DEFAULT FALSE, +-- created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), +-- deleted_at timestamp NULL DEFAULT NULL +-- ); + +-- CREATE TABLE templates +-- ( +-- template_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, +-- template_key text, +-- name text NOT NULL, +-- category text NOT NULL, +-- series jsonb NOT NULL, +-- config jsonb NOT NULL, +-- predefined boolean DEFAULT TRUE +-- ); + +-- CREATE TABLE dashboard_widgets +-- ( +-- widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, +-- dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, +-- metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, +-- -- template_id integer NOT NULL REFERENCES templates (template_id) ON DELETE CASCADE, +-- user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, +-- created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), +-- configuration jsonb NOT NULL DEFAULT '{}'::jsonb, +-- name text +-- ); + +-- INSERT INTO public.templates (name, category, series, config, predefined, template_key) +-- VALUES ('captured sessions', 'overview', '[]', '{}', true, 'count_sessions'), +-- ('request load time', 'overview', '[]', '{}', true, 'avg_request_load_time'), +-- ('page load time', 'overview', '[]', '{}', true, 'avg_page_load_time'), +-- ('image load time', 'overview', '[]', '{}', true, 'avg_image_load_time'); + +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; + +ALTER TABLE IF EXISTS metrics + DROP CONSTRAINT IF EXISTS null_project_id_for_template_only; + +ALTER TABLE IF EXISTS metrics + ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', + ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS key text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{}'::jsonb, + ALTER COLUMN project_id DROP NOT NULL, + ADD CONSTRAINT null_project_id_for_template_only + CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ); + + + +CREATE TABLE IF NOT EXISTS dashboard_widgets ( - dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, + metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, - name text NOT NULL, - is_public boolean NOT NULL DEFAULT TRUE, - is_pinned boolean NOT NULL DEFAULT FALSE, created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - deleted_at timestamp NULL DEFAULT NULL + config jsonb NOT NULL DEFAULT '{}'::jsonb, + name text ); +-- INSERT INTO public.templates (name, category, series, config, predefined, template_key) +-- VALUES ('captured sessions', 'overview', '[]', '{}', true, 'count_sessions'), +-- ('request load time', 'overview', '[]', '{}', true, 'avg_request_load_time'), +-- ('page load time', 'overview', '[]', '{}', true, 'avg_page_load_time'), +-- ('image load time', 'overview', '[]', '{}', true, 'avg_image_load_time'); -CREATE TABLE widgets -( - widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, - name text NOT NULL, - is_template boolean NOT NULL DEFAULT FALSE, - predefined boolean NOT NULL DEFAULT FALSE, - created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - deleted_at timestamp NULL DEFAULT NULL -); +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) +VALUES ('captured sessions', 'overview', '{}', true, true, true, 'count_sessions'), + ('request load time', 'overview', '{}', true, true, true, 'avg_request_load_time'), + ('page load time', 'overview', '{}', true, true, true, 'avg_page_load_time'), + ('image load time', 'overview', '{}', true, true, true, 'avg_image_load_time'); -CREATE TABLE dashboard_widgets -( - dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, - widget_id integer NOT NULL REFERENCES widgets (widget_id) ON DELETE CASCADE, - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, - created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - configuration jsonb NOT NULL DEFAULT '{}'::jsonb -); -COMMIT; \ No newline at end of file +COMMIT \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 834a79968..52f367e8e 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -936,11 +936,11 @@ $$ CREATE INDEX jobs_project_id_idx ON jobs (project_id); CREATE TYPE metric_type AS ENUM ('timeseries','table'); - CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart'); + CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart'); CREATE TABLE metrics ( metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE, user_id integer REFERENCES users (user_id) ON DELETE SET NULL, name text NOT NULL, is_public boolean NOT NULL DEFAULT FALSE, @@ -951,8 +951,16 @@ $$ view_type metric_view_type NOT NULL DEFAULT 'lineChart', metric_of text NOT NULL DEFAULT 'sessionCount', metric_value text[] NOT NULL DEFAULT '{}'::text[], - metric_format text + metric_format text, + category text NULL DEFAULT 'custom', + is_predefined boolean NOT NULL DEFAULT FALSE, + is_template boolean NOT NULL DEFAULT FALSE, + key text NULL DEFAULT NULL, + config jsonb NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT null_project_id_for_template_only + CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ) ); + CREATE INDEX metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); CREATE TABLE metric_series ( From fe53ee07d7fbe9aec3140c4a9387f92cd6e43011 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 29 Mar 2022 18:33:41 +0200 Subject: [PATCH 005/125] feat(api): dashboards 3/5 --- api/chalicelib/core/dashboards2.py | 108 ++++++++++++++---- api/chalicelib/core/templates.py | 20 ---- api/routers/subs/metrics.py | 59 +++++++--- api/schemas.py | 5 +- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 1 + .../db/init_dbs/postgresql/init_schema.sql | 1 + 6 files changed, 129 insertions(+), 65 deletions(-) delete mode 100644 api/chalicelib/core/templates.py diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index c36293059..215a4f9ea 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -4,6 +4,24 @@ import schemas from chalicelib.utils import helper from chalicelib.utils import pg_client +CATEGORY_DESCRIPTION = { + 'categ1': 'lorem', +} + + +def get_templates(project_id, user_id): + with pg_client.PostgresClient() as cur: + pg_query = cur.mogrify(f"""SELECT category, jsonb_agg(metrics ORDER BY name) AS widgets + FROM metrics + WHERE deleted_at IS NULL AND (project_id ISNULL OR (project_id = %(project_id)s AND (is_public OR user_id= %(userId)s))) + GROUP BY category + ORDER BY category;""", {"project_id": project_id, "userId": user_id}) + cur.execute(pg_query) + rows = cur.fetchall() + for r in rows: + r["description"] = CATEGORY_DESCRIPTION.get(r["category"], "") + return helper.list_to_camel_case(rows) + def create_dashboard(project_id, user_id, data: schemas.CreateDashboardSchema): with pg_client.PostgresClient() as cur: @@ -31,15 +49,10 @@ def get_dashboards(project_id, user_id): def get_dashboard(project_id, user_id, dashboard_id): with pg_client.PostgresClient() as cur: - pg_query = """SELECT dashboards.*, all_template_widgets.widgets AS template_widgets, all_metric_widgets.widgets AS metric_widgets + pg_query = """SELECT dashboards.*, all_metric_widgets.widgets AS widgets FROM dashboards - LEFT JOIN LATERAL (SELECT COALESCE(JSONB_AGG(templates), '[]'::jsonb) AS widgets - FROM templates - INNER JOIN dashboard_widgets USING (template_id) - WHERE dashboard_widgets.dashboard_id = dashboards.dashboard_id - ) AS all_template_widgets ON (TRUE) LEFT JOIN LATERAL (SELECT COALESCE(JSONB_AGG(raw_metrics), '[]') AS widgets - FROM (SELECT metrics.*, metric_series.series + FROM (SELECT dashboard_widgets.*, metrics.*, metric_series.series FROM metrics INNER JOIN dashboard_widgets USING (metric_id) LEFT JOIN LATERAL (SELECT JSONB_AGG(metric_series.* ORDER BY index) AS series @@ -59,35 +72,84 @@ def get_dashboard(project_id, user_id, dashboard_id): print(cur.mogrify(pg_query, params)) cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() - row["widgets"] = row.pop("template_widgets") + row.pop("metric_widgets") + return helper.dict_to_camel_case(row) + + +def delete_dashboard(project_id, user_id, dashboard_id): + with pg_client.PostgresClient() as cur: + pg_query = """UPDATE dashboards + SET deleted_at = timezone('utc'::text, now()) + WHERE dashboards.project_id = %(projectId)s + AND dashboard_id = %(dashboard_id)s + AND (dashboards.user_id = %(userId)s OR is_public);""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id} + cur.execute(cur.mogrify(pg_query, params)) + return {"data": {"success": True}} + + +def update_dashboard(project_id, user_id, dashboard_id, data: schemas.CreateDashboardSchema): + with pg_client.PostgresClient() as cur: + pg_query = """UPDATE dashboards + SET name = %(name)s, is_pinned = %(is_pinned)s, is_public = %(is_public)s + WHERE dashboards.project_id = %(projectId)s + AND dashboard_id = %(dashboard_id)s + AND (dashboards.user_id = %(userId)s OR is_public) + RETURNING *;""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, **data.dict()} + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() return helper.dict_to_camel_case(row) def add_widget(project_id, user_id, dashboard_id, data: schemas.AddWidgetToDashboardPayloadSchema): - ref_key = "metric_id" - if data.template_id is not None: - ref_key = "template_id" with pg_client.PostgresClient() as cur: - pg_query = f"""INSERT INTO dashboard_widgets(dashboard_id, {ref_key}, user_id, configuration, name) - VALUES (%(dashboard_id)s, %({ref_key})s, %(userId)s, %(configuration)s::jsonb, %(name)s) + pg_query = """INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config, name) + SELECT %(dashboard_id)s AS dashboard_id, %(metric_id)s AS metric_id, + %(userId)s AS user_id, %(config)s::jsonb AS config, %(name)s AS name + WHERE EXISTS(SELECT 1 FROM dashboards + WHERE dashboards.deleted_at ISNULL AND dashboards.project_id = %(projectId)s + AND dashboard_id = %(dashboard_id)s + AND (dashboards.user_id = %(userId)s OR is_public)) RETURNING *;""" params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, **data.dict()} - params["configuration"] = json.dumps(params.get("configuration", {})) + params["config"] = json.dumps(data.config) cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() return helper.dict_to_camel_case(row) -def remove_widget(project_id, user_id, dashboard_id,widget_id): - ref_key = "metric_id" - if data.template_id is not None: - ref_key = "template_id" + +def update_widget(project_id, user_id, dashboard_id, widget_id, data: schemas.AddWidgetToDashboardPayloadSchema): with pg_client.PostgresClient() as cur: - pg_query = f"""INSERT INTO dashboard_widgets(dashboard_id, {ref_key}, user_id, configuration, name) - VALUES (%(dashboard_id)s, %({ref_key})s, %(userId)s, %(configuration)s::jsonb, %(name)s) - RETURNING *;""" - params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, **data.dict()} - params["configuration"] = json.dumps(params.get("configuration", {})) + pg_query = """UPDATE dashboard_widgets + SET name= %(name)s, config= %(config)s + WHERE dashboard_id=%(dashboard_id)s AND widget_id=%(widget_id)s + RETURNINIG *;""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, + "widget_id": widget_id, **data.dict()} cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() return helper.dict_to_camel_case(row) + +def remove_widget(project_id, user_id, dashboard_id, widget_id): + with pg_client.PostgresClient() as cur: + pg_query = """DELETE FROM dashboard_widgets + WHERE dashboard_id=%(dashboard_id)s AND widget_id=%(widget_id)s;""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, "widget_id": widget_id} + cur.execute(cur.mogrify(pg_query, params)) + return {"data": {"success": True}} + + +def pin_dashboard(project_id, user_id, dashboard_id): + with pg_client.PostgresClient() as cur: + pg_query = """UPDATE dashboards + SET is_pinned = FALSE + WHERE dashboard_id=%(dashboard_id)s AND project_id=%(project_id)s; + UPDATE dashboards + SET is_pinned = True + WHERE dashboard_id=%(dashboard_id)s AND project_id=%(project_id)s AND deleted_at ISNULL + RETURNING *;""" + params = {"userId": user_id, "project_id": project_id, "dashboard_id": dashboard_id} + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return helper.dict_to_camel_case(row) diff --git a/api/chalicelib/core/templates.py b/api/chalicelib/core/templates.py deleted file mode 100644 index 8e42668d3..000000000 --- a/api/chalicelib/core/templates.py +++ /dev/null @@ -1,20 +0,0 @@ -from chalicelib.utils import helper -from chalicelib.utils import pg_client - -CATEGORY_DESCRIPTION = { - 'categ1': 'lorem', -} - - -def get_templates(project_id, user_id): - with pg_client.PostgresClient() as cur: - pg_query = cur.mogrify(f"""SELECT category, jsonb_agg(metrics ORDER BY name) AS widgets - FROM metrics - WHERE project_id ISNULL OR (project_id = %(project_id)s AND (is_public OR user_id= %(userId)s)) - GROUP BY category - ORDER BY category;""", {"project_id": project_id, "userId": user_id}) - cur.execute(pg_query) - rows = cur.fetchall() - for r in rows: - r["description"] = CATEGORY_DESCRIPTION.get(r["category"], "") - return helper.list_to_camel_case(rows) diff --git a/api/routers/subs/metrics.py b/api/routers/subs/metrics.py index 56b486977..2d3c116d3 100644 --- a/api/routers/subs/metrics.py +++ b/api/routers/subs/metrics.py @@ -1,32 +1,50 @@ from fastapi import Body, Depends import schemas -from chalicelib.core import dashboards2, templates, custom_metrics +from chalicelib.core import dashboards2, custom_metrics from or_dependencies import OR_context from routers.base import get_routers public_app, app, app_apikey = get_routers() -@app.post('/{projectId}/dashboards', tags=["dashboard", "metrics"]) -@app.put('/{projectId}/dashboards', tags=["dashboard", "metrics"]) +@app.post('/{projectId}/dashboards', tags=["dashboard"]) +@app.put('/{projectId}/dashboards', tags=["dashboard"]) def create_dashboards(projectId: int, data: schemas.CreateDashboardSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): return {"data": dashboards2.create_dashboard(project_id=projectId, user_id=context.user_id, data=data)} -@app.get('/{projectId}/dashboards', tags=["dashboard", "metrics"]) +@app.get('/{projectId}/dashboards', tags=["dashboard"]) def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): return {"data": dashboards2.get_dashboards(project_id=projectId, user_id=context.user_id)} -@app.get('/{projectId}/dashboards/{dashboardId}', tags=["dashboard", "metrics"]) +@app.get('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) def get_dashboard(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): return {"data": dashboards2.get_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId)} -@app.post('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard", "metrics"]) -@app.put('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard", "metrics"]) +@app.post('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) +@app.put('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) +def update_dashboard(projectId: int, dashboardId: int, data: schemas.CreateDashboardSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.update_dashboard(project_id=projectId, user_id=context.user_id, + dashboard_id=dashboardId, data=data)} + + +@app.delete('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) +def delete_dashboard(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): + return dashboards2.delete_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId) + + +@app.get('/{projectId}/dashboards/{dashboardId}/pin', tags=["dashboard"]) +def pin_dashboard(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.pin_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId)} + + +@app.post('/{projectId}/dashboards/{dashboardId}/widgets', tags=["dashboard"]) +@app.put('/{projectId}/dashboards/{dashboardId}/widgets', tags=["dashboard"]) def add_widget_to_dashboard(projectId: int, dashboardId: int, data: schemas.AddWidgetToDashboardPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): @@ -34,22 +52,25 @@ def add_widget_to_dashboard(projectId: int, dashboardId: int, data=data)} -@app.delete('/{projectId}/dashboards/{dashboardId}/metrics/{metricId}', tags=["dashboard", "metrics"]) -def remove_widget_from_dashboard(projectId: int, dashboardId: int, metricId: int, - data: schemas.AddWidgetToDashboardPayloadSchema = Body(...), +@app.post('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) +@app.put('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) +def update_widget_in_dashboard(projectId: int, dashboardId: int, widgetId: int, + data: schemas.AddWidgetToDashboardPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return dashboards2.update_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, + widget_id=widgetId, data=data) + + +@app.delete('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) +def remove_widget_from_dashboard(projectId: int, dashboardId: int, widgetId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": dashboards2.add_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, - data=data)} + return dashboards2.remove_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, + widget_id=widgetId) -# @app.get('/{projectId}/widgets', tags=["dashboard", "metrics"]) -# def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): -# return {"data": dashboards2.get_widgets(project_id=projectId)} - - -@app.get('/{projectId}/metrics/templates', tags=["dashboard", "metrics"]) +@app.get('/{projectId}/metrics/templates', tags=["dashboard"]) def get_templates(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": templates.get_templates(project_id=projectId, user_id=context.user_id)} + return {"data": dashboards2.get_templates(project_id=projectId, user_id=context.user_id)} @app.post('/{projectId}/metrics/try', tags=["dashboard"]) diff --git a/api/schemas.py b/api/schemas.py index bb111d61b..ad9558a4a 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -878,7 +878,7 @@ class SavedSearchSchema(FunnelSchema): class CreateDashboardSchema(BaseModel): - name: str = Field(...) + name: str = Field(..., min_length=1) is_public: bool = Field(default=False) is_pinned: bool = Field(default=False) @@ -887,10 +887,9 @@ class CreateDashboardSchema(BaseModel): class AddWidgetToDashboardPayloadSchema(BaseModel): - template_id: Optional[int] = Field(default=None) metric_id: Optional[int] = Field(default=None) name: Optional[str] = Field(default=None) - configuration: dict = Field(default={}) + config: dict = Field(default={}) @root_validator def validator(cls, values): diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index d0d31ecce..13e9e9e14 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -53,6 +53,7 @@ ALTER TABLE IF EXISTS metrics DROP CONSTRAINT IF EXISTS null_project_id_for_template_only; ALTER TABLE IF EXISTS metrics + ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 52f367e8e..35926bc96 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -953,6 +953,7 @@ $$ metric_value text[] NOT NULL DEFAULT '{}'::text[], metric_format text, category text NULL DEFAULT 'custom', + is_pinned boolean NOT NULL DEFAULT FALSE, is_predefined boolean NOT NULL DEFAULT FALSE, is_template boolean NOT NULL DEFAULT FALSE, key text NULL DEFAULT NULL, From b5932308be1b7edb0c77d0a767c2399154a4253a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 29 Mar 2022 18:45:58 +0200 Subject: [PATCH 006/125] feat(api): build local script --- api/build_local.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 api/build_local.sh diff --git a/api/build_local.sh b/api/build_local.sh new file mode 100644 index 000000000..4253fd812 --- /dev/null +++ b/api/build_local.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build -f ./Dockerfile --build-arg envarg="default-foss" -t chalice:local . \ No newline at end of file From 9f2db6f42e926ef1f29b57aa8c10a8d95f339d0c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 29 Mar 2022 19:58:22 +0200 Subject: [PATCH 007/125] feat(api): dashboard split old metrics grouped data 1/3 --- api/chalicelib/core/dashboard.py | 273 ++++++++++++++++++++++++++++--- api/routers/subs/dashboard.py | 27 +++ api/schemas.py | 8 + 3 files changed, 284 insertions(+), 24 deletions(-) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index 9cd88eb6a..72995c27a 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -138,7 +138,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) with pg_client.PostgresClient() as cur: pg_query = f"""\ SELECT generated_timestamp AS timestamp, - COALESCE(COUNT(sessions), 0) AS count + COALESCE(COUNT(sessions), 0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT 1 FROM public.sessions @@ -151,7 +151,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) cur.execute(cur.mogrify(pg_query, params)) rows = cur.fetchall() results = { - "count": sum([r["count"] for r in rows]), + "value": sum([r["value"] for r in rows]), "chart": rows } @@ -170,7 +170,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) count = cur.fetchone()["count"] - results["countProgress"] = helper.__progress(old_val=count, new_val=results["count"]) + results["progress"] = helper.__progress(old_val=count, new_val=results["value"]) return results @@ -468,8 +468,9 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ORDER BY generated_timestamp) AS chart ) AS chart ON (TRUE);""" - cur.execute(cur.mogrify(pg_query, {"step_size": step_size,"project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + cur.execute( + cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) rows = cur.fetchall() for i in range(len(rows)): rows[i]["sessions"] = rows[i].pop("sessions_count") @@ -672,8 +673,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} AND positionUTF8(url_path, %(value)s) != 0 LIMIT 10);""" print(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text.lower()), - "platform_0": platform})) + "value": helper.string_to_sql_like(text.lower()), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text.lower()), "platform_0": platform})) @@ -691,9 +692,9 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" print(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "resource_type": resource_type, - "platform_0": platform})) + "value": helper.string_to_sql_like(text), + "resource_type": resource_type, + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "resource_type": resource_type, @@ -709,8 +710,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" print(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "platform_0": platform})) + "value": helper.string_to_sql_like(text), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "platform_0": platform})) @@ -723,8 +724,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" print(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "platform_0": platform})) + "value": helper.string_to_sql_like(text), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "platform_0": platform})) @@ -737,8 +738,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" print(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "platform_0": platform})) + "value": helper.string_to_sql_like(text), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "platform_0": platform})) @@ -758,8 +759,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" print(cur.mogrify(pg_query, - {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, - "platform_0": platform})) + {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, "platform_0": platform})) @@ -785,9 +786,9 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, LIMIT 10)""") pg_query = " UNION ALL ".join(pg_query) print(cur.mogrify(pg_query, - {"project_id": project_id, "value": helper.string_to_sql_like(text), - "key": key, - "platform_0": platform})) + {"project_id": project_id, "value": helper.string_to_sql_like(text), + "key": key, + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, @@ -866,8 +867,6 @@ def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1), pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") - - with pg_client.PostgresClient() as cur: pg_query = f"""WITH resources AS (SELECT resources.session_id, resources.url_hostpath, @@ -1952,7 +1951,7 @@ def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), e pg_sub_query_subset.append("resources.status > 200") pg_sub_query_subset_e = __get_constraints(project_id=project_id, data=args, duration=False, main_table="m_errors", - time_constraint=False) + time_constraint=False) pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, data=args, main_table="", time_column="timestamp", project=False, duration=False) @@ -2284,3 +2283,229 @@ def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1) rows = cur.fetchall() return rows + + +def __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("resources.duration > 0") + pg_sub_query.append("resources.type= %(type)s") + pg_query = f"""\ + SELECT COALESCE(AVG(resources.duration),0) AS value + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + row = cur.fetchone() + return row + + +def get_application_activity_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["chart"] = get_performance_avg_image_load_time(project_id, startTimestamp, endTimestamp, **args) + return results + + +def get_performance_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) + img_constraints = [] + + img_constraints_vals = {} + + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + with pg_client.PostgresClient() as cur: + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") + + pg_query = f"""WITH resources AS (SELECT resources.duration, resources.timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + AND resources.type = 'img' AND resources.duration>0 + {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} + ) + SELECT + generated_timestamp AS timestamp, + COALESCE(AVG(resources.duration),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT resources.duration + FROM resources + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS resources ON (TRUE) + GROUP BY timestamp + ORDER BY timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **img_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() + rows = helper.list_to_camel_case(rows) + + return rows + + +def __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("pages.timestamp >= %(startTimestamp)s") + pg_sub_query.append("pages.timestamp > %(endTimestamp)s") + pg_sub_query.append("pages.load_time > 0") + pg_sub_query.append("pages.load_time IS NOT NULL") + pg_query = f"""\ + SELECT COALESCE(AVG(pages.load_time) ,0) AS value + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return row + + +@dev.timed +def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["chart"] = get_performance_avg_page_load_time(project_id, startTimestamp, endTimestamp, **args) + return results + + +@dev.timed +def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) + location_constraints = [] + location_constraints_vals = {} + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + with pg_client.PostgresClient() as cur: + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="pages", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s") + pg_query = f"""WITH pages AS(SELECT pages.load_time, timestamp + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} AND pages.load_time>0 AND pages.load_time IS NOT NULL + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + ) + SELECT + generated_timestamp AS timestamp, + COALESCE(AVG(pages.load_time),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT pages.load_time + FROM pages + WHERE {" AND ".join(pg_sub_query_chart)} + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + ) AS pages ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **location_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() + return rows + + +def __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("resources.duration > 0") + pg_sub_query.append("resources.type= %(type)s") + pg_query = f"""\ + SELECT COALESCE(AVG(resources.duration),0) AS value + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + + row = cur.fetchone() + return row + + +@dev.timed +def get_application_activity_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["chart"] = get_performance_avg_request_load_time(project_id, startTimestamp, endTimestamp, **args) + return results + + +@dev.timed +def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) + location_constraints = [] + img_constraints = [] + request_constraints = [] + + img_constraints_vals = {} + location_constraints_vals = {} + request_constraints_vals = {} + + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + with pg_client.PostgresClient() as cur: + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") + + pg_query = f"""WITH resources AS(SELECT resources.duration, resources.timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + AND resources.type = 'fetch' AND resources.duration>0 + {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} + ) + SELECT + generated_timestamp AS timestamp, + COALESCE(AVG(resources.duration),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT resources.duration + FROM resources + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **request_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() + + return rows diff --git a/api/routers/subs/dashboard.py b/api/routers/subs/dashboard.py index 169893693..4faa82d62 100644 --- a/api/routers/subs/dashboard.py +++ b/api/routers/subs/dashboard.py @@ -344,3 +344,30 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body *helper.explode_widget(dashboard.get_avg_cpu(project_id=projectId, **data.dict())), *helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **data.dict())), ]} + +@app.post('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) +@app.get('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) +def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body(...)): + return {"data": [ + {"key": schemas.TemplateKeys.count_sessions, + "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_image_load_time, + "data": dashboard.get_application_activity_avg_image_load_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_page_load_time, + "data": dashboard.get_application_activity_avg_page_load_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_request_load_time, + "data": dashboard.get_application_activity_avg_request_load_time(project_id=projectId, **data.dict())}, + + # *helper.explode_widget(data=dashboard.get_page_metrics(project_id=projectId, **data.dict())), + # *helper.explode_widget(data=dashboard.get_user_activity(project_id=projectId, **data.dict())), + # *helper.explode_widget(data=dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict()), + # key="avg_pages_dom_buildtime"), + # *helper.explode_widget(data=dashboard.get_pages_response_time(project_id=projectId, **data.dict()), + # key="avg_pages_response_time"), + # *helper.explode_widget(dashboard.get_top_metrics(project_id=projectId, **data.dict())), + # *helper.explode_widget(data=dashboard.get_time_to_render(project_id=projectId, **data.dict()), + # key="avg_time_to_render"), + # *helper.explode_widget(dashboard.get_memory_consumption(project_id=projectId, **data.dict())), + # *helper.explode_widget(dashboard.get_avg_cpu(project_id=projectId, **data.dict())), + # *helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **data.dict())), + ]} diff --git a/api/schemas.py b/api/schemas.py index ad9558a4a..b2b96d048 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -900,3 +900,11 @@ class AddWidgetToDashboardPayloadSchema(BaseModel): class Config: alias_generator = attribute_to_camel_case + + +# these values should match the keys in metrics table +class TemplateKeys(str, Enum): + count_sessions = "count_sessions" + avg_request_load_time = "avg_request_load_time" + avg_page_load_time = "avg_page_load_time" + avg_image_load_time = "avg_image_load_time" From b3018f9f76617a793863cbb986e918a47eb0f78f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 30 Mar 2022 10:49:45 +0200 Subject: [PATCH 008/125] feat(api): dashboard split old metrics grouped data 2/3 --- api/chalicelib/core/dashboard.py | 138 ++++++++++++++++++++++++++++++- api/routers/subs/dashboard.py | 48 ++++++----- api/schemas.py | 6 ++ 3 files changed, 168 insertions(+), 24 deletions(-) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index 72995c27a..62bdc5466 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -992,13 +992,13 @@ def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=- FROM public.sessions INNER JOIN events.pages USING (session_id) WHERE {" AND ".join(pg_sub_query_subset)}) - SELECT COALESCE(avg, 0) AS avg, chart + SELECT COALESCE(avg, 0) AS value, chart FROM (SELECT AVG(dom_building_time) FROM pages) AS avg LEFT JOIN (SELECT jsonb_agg(chart) AS chart FROM ( SELECT generated_timestamp AS timestamp, - COALESCE(AVG(dom_building_time), 0) AS avg + COALESCE(AVG(dom_building_time), 0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT pages.dom_building_time FROM pages @@ -1154,7 +1154,7 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 pg_sub_query_chart.append(f"url = %(value)s") with pg_client.PostgresClient() as cur: pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.response_time),0) AS avg + COALESCE(AVG(pages.response_time),0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT response_time @@ -1175,7 +1175,7 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"avg": avg, "chart": rows} + return {"value": avg, "chart": rows} @dev.timed @@ -2509,3 +2509,133 @@ def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now rows = cur.fetchall() return rows + + +def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + rows = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("pages.timestamp>=%(startTimestamp)s") + pg_sub_query.append("pages.timestamp<%(endTimestamp)s") + pg_sub_query.append("pages.dom_content_loaded_time > 0") + pg_query = f"""SELECT COALESCE(AVG(NULLIF(pages.dom_content_loaded_time, 0)), 0) AS value + FROM (SELECT pages.dom_content_loaded_time + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + ) AS pages;""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + return rows + + +def get_page_metrics_avg_first_contentful_pixel(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + rows = __get_page_metrics_avg_first_contentful_pixel(cur, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_page_metrics_avg_first_contentful_pixel(cur, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_page_metrics_avg_first_contentful_pixel(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("pages.timestamp>=%(startTimestamp)s") + pg_sub_query.append("pages.timestamp<%(endTimestamp)s") + pg_sub_query.append("pages.first_contentful_paint_time > 0") + pg_query = f"""SELECT COALESCE(AVG(NULLIF(pages.first_contentful_paint_time, 0)), 0) AS value + FROM (SELECT pages.first_contentful_paint_time + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + ) AS pages;""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + return rows + + + + +def get_user_activity_avg_visited_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_user_activity_avg_visited_pages(cur, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_user_activity_avg_visited_pages(cur, project_id, startTimestamp, endTimestamp, **args) + + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_user_activity_avg_visited_pages(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + pg_query = f"""\ + SELECT COALESCE(CEIL(AVG(NULLIF(sessions.pages_count,0))),0) AS value + FROM public.sessions + WHERE {" AND ".join(pg_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return row + + +def get_user_activity_avg_session_duration(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args) + + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + pg_query = f"""\ + SELECT COALESCE(AVG(NULLIF(sessions.duration,0)),0) AS value + FROM public.sessions + WHERE {" AND ".join(pg_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return row diff --git a/api/routers/subs/dashboard.py b/api/routers/subs/dashboard.py index 4faa82d62..e0b680362 100644 --- a/api/routers/subs/dashboard.py +++ b/api/routers/subs/dashboard.py @@ -326,17 +326,18 @@ def get_dashboard_resources_count_by_type(projectId: int, data: schemas.MetricPa @app.get('/{projectId}/dashboard/overview', tags=["dashboard", "metrics"]) def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body(...)): return {"data": [ - *helper.explode_widget(key="count_sessions", - data=dashboard.get_processed_sessions(project_id=projectId, **data.dict())), + {"key": "count_sessions", + "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, *helper.explode_widget(data={**dashboard.get_application_activity(project_id=projectId, **data.dict()), "chart": dashboard.get_performance(project_id=projectId, **data.dict()) .get("chart", [])}), *helper.explode_widget(data=dashboard.get_page_metrics(project_id=projectId, **data.dict())), *helper.explode_widget(data=dashboard.get_user_activity(project_id=projectId, **data.dict())), - *helper.explode_widget(data=dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict()), - key="avg_pages_dom_buildtime"), - *helper.explode_widget(data=dashboard.get_pages_response_time(project_id=projectId, **data.dict()), - key="avg_pages_response_time"), + {"key": "avg_pages_dom_buildtime", + "data": dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict())}, + {"key": "avg_pages_response_time", + "data": dashboard.get_pages_response_time(project_id=projectId, **data.dict()) + }, *helper.explode_widget(dashboard.get_top_metrics(project_id=projectId, **data.dict())), *helper.explode_widget(data=dashboard.get_time_to_render(project_id=projectId, **data.dict()), key="avg_time_to_render"), @@ -345,25 +346,32 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body *helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **data.dict())), ]} + @app.post('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) @app.get('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body(...)): return {"data": [ - {"key": schemas.TemplateKeys.count_sessions, - "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_image_load_time, - "data": dashboard.get_application_activity_avg_image_load_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_page_load_time, - "data": dashboard.get_application_activity_avg_page_load_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_request_load_time, - "data": dashboard.get_application_activity_avg_request_load_time(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplateKeys.count_sessions, + # "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplateKeys.avg_image_load_time, + # "data": dashboard.get_application_activity_avg_image_load_time(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplateKeys.avg_page_load_time, + # "data": dashboard.get_application_activity_avg_page_load_time(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplateKeys.avg_request_load_time, + # "data": dashboard.get_application_activity_avg_request_load_time(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplateKeys.avg_dom_content_load_start, + # "data": dashboard.get_page_metrics_avg_dom_content_load_start(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplateKeys.avg_first_contentful_pixel, + # "data": dashboard.get_page_metrics_avg_first_contentful_pixel(project_id=projectId, **data.dict())} + # {"key": schemas.TemplateKeys.avg_visited_pages, + # "data": dashboard.get_user_activity_avg_visited_pages(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplateKeys.avg_session_duration, + # "data": dashboard.get_user_activity_avg_session_duration(project_id=projectId, **data.dict())} + # {"key": schemas.TemplateKeys.avg_pages_dom_buildtime, + # "data": dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict())}, - # *helper.explode_widget(data=dashboard.get_page_metrics(project_id=projectId, **data.dict())), - # *helper.explode_widget(data=dashboard.get_user_activity(project_id=projectId, **data.dict())), - # *helper.explode_widget(data=dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict()), - # key="avg_pages_dom_buildtime"), - # *helper.explode_widget(data=dashboard.get_pages_response_time(project_id=projectId, **data.dict()), - # key="avg_pages_response_time"), + *helper.explode_widget(data=dashboard.get_pages_response_time(project_id=projectId, **data.dict()), + key="avg_pages_response_time"), # *helper.explode_widget(dashboard.get_top_metrics(project_id=projectId, **data.dict())), # *helper.explode_widget(data=dashboard.get_time_to_render(project_id=projectId, **data.dict()), # key="avg_time_to_render"), diff --git a/api/schemas.py b/api/schemas.py index b2b96d048..4f672a844 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -908,3 +908,9 @@ class TemplateKeys(str, Enum): avg_request_load_time = "avg_request_load_time" avg_page_load_time = "avg_page_load_time" avg_image_load_time = "avg_image_load_time" + avg_dom_content_load_start = "avg_dom_content_load_start" + avg_first_contentful_pixel = "avg_first_contentful_pixel" + avg_visited_pages = "avg_visited_pages" + avg_session_duration = "avg_session_duration" + avg_pages_dom_buildtime="avg_pages_dom_buildtime" + avg_pages_response_time="avg_pages_response_time" From d6c66e6d464bc114a520d6f25866a245e3b5b731 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 30 Mar 2022 10:53:37 +0200 Subject: [PATCH 009/125] workflow_dispatch --- .github/workflows/api.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index eb193dbb7..9fe8c5611 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -1,5 +1,6 @@ # This action will push the chalice changes to aws on: + workflow_dispatch: push: branches: - api-v1.5.5 From 3b6085606307aee70966bf74ef29fcf5ab01293c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 30 Mar 2022 12:29:06 +0200 Subject: [PATCH 010/125] feat(api): dashboard split old metrics grouped data 3/3 --- api/chalicelib/core/dashboard.py | 197 ++++++++++++------ api/routers/subs/dashboard.py | 74 ++++--- api/schemas.py | 14 +- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 22 +- 4 files changed, 203 insertions(+), 104 deletions(-) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index 62bdc5466..bab19ba12 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -127,7 +127,6 @@ SESSIONS_META_FIELDS = {"revId": "rev_id", "browser": "user_browser"} -@dev.timed def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -175,7 +174,6 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) return results -@dev.timed def get_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -234,7 +232,6 @@ def __count_distinct_errors(cur, project_id, startTimestamp, endTimestamp, pg_su return cur.fetchone()["count"] -@dev.timed def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -298,7 +295,6 @@ def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return rows -@dev.timed def get_page_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: @@ -316,7 +312,6 @@ def get_page_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return results -@dev.timed def __get_page_metrics(cur, project_id, startTimestamp, endTimestamp, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) pg_sub_query.append("pages.timestamp>=%(startTimestamp)s") @@ -336,7 +331,6 @@ def __get_page_metrics(cur, project_id, startTimestamp, endTimestamp, **args): return rows -@dev.timed def get_application_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: @@ -390,7 +384,6 @@ def __get_application_activity(cur, project_id, startTimestamp, endTimestamp, ** return result -@dev.timed def get_user_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: @@ -423,7 +416,6 @@ def __get_user_activity(cur, project_id, startTimestamp, endTimestamp, **args): return row -@dev.timed def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -479,7 +471,6 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return sorted(rows, key=lambda k: k["sessions"], reverse=True) -@dev.timed def __get_performance_constraint(l): if len(l) == 0: return "" @@ -487,7 +478,6 @@ def __get_performance_constraint(l): return f"AND ({' OR '.join(l)})" -@dev.timed def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, resources=None, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) @@ -622,7 +612,6 @@ def __get_resource_db_type_from_type(resource_type): return {v: k for k, v in RESOURCS_TYPE_TO_DB_TYPE.items()}.get(resource_type, resource_type) -@dev.timed def search(text, resource_type, project_id, performance=False, pages_only=False, events_only=False, metadata=False, key=None, platform=None): if not resource_type: @@ -799,7 +788,6 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, return [helper.dict_to_camel_case(row) for row in rows] -@dev.timed def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -855,7 +843,6 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day return rows -@dev.timed def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -921,7 +908,6 @@ def dashboard_args(params): return args -@dev.timed def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, type=None, url=None, **args): @@ -970,7 +956,6 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days return {"avg": avg, "chart": rows} -@dev.timed def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1016,7 +1001,6 @@ def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=- return row -@dev.timed def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), type="all", density=19, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1090,7 +1074,6 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return rows -@dev.timed def get_sessions_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1109,7 +1092,6 @@ def get_sessions_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return {"count": sum(i["count"] for i in rows), "chart": helper.list_to_camel_case(rows)} -@dev.timed def get_speed_index_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1138,7 +1120,6 @@ def get_speed_index_location(project_id, startTimestamp=TimeUTC.now(delta_days=- return {"avg": avg, "chart": helper.list_to_camel_case(rows)} -@dev.timed def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1178,7 +1159,6 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 return {"value": avg, "chart": rows} -@dev.timed def get_pages_response_time_distribution(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=20, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1296,7 +1276,6 @@ def get_pages_response_time_distribution(project_id, startTimestamp=TimeUTC.now( return result -@dev.timed def get_busiest_time_of_day(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1316,7 +1295,6 @@ def get_busiest_time_of_day(project_id, startTimestamp=TimeUTC.now(delta_days=-1 return rows -@dev.timed def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), value=None, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1367,7 +1345,6 @@ def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return helper.dict_to_camel_case(row) -@dev.timed def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1383,11 +1360,11 @@ def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), pg_query = f"""WITH pages AS(SELECT pages.visually_complete,pages.timestamp FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query_subset)}) - SELECT COALESCE((SELECT AVG(pages.visually_complete) FROM pages),0) AS avg, + SELECT COALESCE((SELECT AVG(pages.visually_complete) FROM pages),0) AS value, jsonb_agg(chart) AS chart FROM (SELECT generated_timestamp AS timestamp, - COALESCE(AVG(visually_complete), 0) AS avg + COALESCE(AVG(visually_complete), 0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT pages.visually_complete FROM pages @@ -1404,7 +1381,6 @@ def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return row -@dev.timed def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), value=None, density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1443,7 +1419,6 @@ def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(d return rows -@dev.timed def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1453,7 +1428,7 @@ def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1) with pg_client.PostgresClient() as cur: pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(performance.avg_used_js_heap_size),0) AS avg_used_js_heap_size + COALESCE(AVG(performance.avg_used_js_heap_size),0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT avg_used_js_heap_size @@ -1473,10 +1448,9 @@ def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1) WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"avgUsedJsHeapSize": avg, "chart": helper.list_to_camel_case(rows)} + return {"value": avg, "chart": helper.list_to_camel_case(rows)} -@dev.timed def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1486,7 +1460,7 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with pg_client.PostgresClient() as cur: pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(performance.avg_cpu),0) AS avg_cpu + COALESCE(AVG(performance.avg_cpu),0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT avg_cpu @@ -1506,10 +1480,9 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"avgCpu": avg, "chart": helper.list_to_camel_case(rows)} + return {"value": avg, "chart": helper.list_to_camel_case(rows)} -@dev.timed def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1519,7 +1492,7 @@ def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with pg_client.PostgresClient() as cur: pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(NULLIF(performance.avg_fps,0)),0) AS avg_fps + COALESCE(AVG(NULLIF(performance.avg_fps,0)),0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT avg_fps @@ -1539,10 +1512,9 @@ def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"avgFps": avg, "chart": helper.list_to_camel_case(rows)} + return {"value": avg, "chart": helper.list_to_camel_case(rows)} -@dev.timed def get_crashes(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1627,7 +1599,6 @@ def __merge_rows_with_neutral(rows, neutral): return rows -@dev.timed def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1678,7 +1649,6 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return result -@dev.timed def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1719,7 +1689,6 @@ def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) return rows -@dev.timed def get_domains_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1768,7 +1737,6 @@ def __nested_array_to_dict_array(rows, key="url_host", value="count"): return rows -@dev.timed def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1800,7 +1768,6 @@ def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return {"avg": avg, "partition": rows} -@dev.timed def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1822,7 +1789,6 @@ def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1) return helper.list_to_camel_case(rows) -@dev.timed def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1865,7 +1831,6 @@ def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=- return {"count": sum(i["count"] for i in rows), "chart": rows} -@dev.timed def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1891,7 +1856,6 @@ def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endT return helper.list_to_camel_case(rows) -@dev.timed def get_calls_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1915,7 +1879,6 @@ def get_calls_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return helper.list_to_camel_case(rows) -@dev.timed def get_calls_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1939,7 +1902,6 @@ def get_calls_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return helper.list_to_camel_case(rows) -@dev.timed def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -2004,7 +1966,6 @@ def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), e return rows -@dev.timed def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -2059,7 +2020,6 @@ def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_d return helper.list_to_camel_case(__merge_charts(response_end, actions)) -@dev.timed def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -2141,7 +2101,6 @@ def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(de return {**row_sessions, **row_errors, "chart": chart} -@dev.timed def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -2192,7 +2151,6 @@ def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(de return helper.list_to_camel_case(rows) -@dev.timed def get_resources_count_by_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -2229,7 +2187,6 @@ def get_resources_count_by_type(project_id, startTimestamp=TimeUTC.now(delta_day return rows -@dev.timed def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -2376,7 +2333,6 @@ def __get_application_activity_avg_page_load_time(cur, project_id, startTimestam return row -@dev.timed def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: @@ -2392,7 +2348,6 @@ def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeU return results -@dev.timed def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, **args): @@ -2448,7 +2403,6 @@ def __get_application_activity_avg_request_load_time(cur, project_id, startTimes return row -@dev.timed def get_application_activity_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: @@ -2464,7 +2418,6 @@ def get_application_activity_avg_request_load_time(project_id, startTimestamp=Ti return results -@dev.timed def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, **args): @@ -2579,10 +2532,8 @@ def __get_page_metrics_avg_first_contentful_pixel(cur, project_id, startTimestam return rows - - def get_user_activity_avg_visited_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): + endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: row = __get_user_activity_avg_visited_pages(cur, project_id, startTimestamp, endTimestamp, **args) results = helper.dict_to_camel_case(row) @@ -2612,7 +2563,7 @@ def __get_user_activity_avg_visited_pages(cur, project_id, startTimestamp, endTi def get_user_activity_avg_session_duration(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), **args): + endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: row = __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args) results = helper.dict_to_camel_case(row) @@ -2639,3 +2590,131 @@ def __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, en cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() return row + + +def get_top_metrics_avg_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(AVG(pages.response_time), 0) AS value + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND pages.timestamp >= %(startTimestamp)s + AND pages.timestamp < %(endTimestamp)s + AND pages.response_time > 0;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(AVG(pages.first_paint_time), 0) AS value + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND pages.timestamp >= %(startTimestamp)s + AND pages.timestamp < %(endTimestamp)s + AND pages.first_paint_time > 0;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(AVG(pages.dom_content_loaded_time), 0) AS value + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND pages.timestamp >= %(startTimestamp)s + AND pages.timestamp < %(endTimestamp)s + AND pages.dom_content_loaded_time > 0;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def get_top_metrics_avg_till_first_bit(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(AVG(pages.ttfb), 0) AS value + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND pages.timestamp >= %(startTimestamp)s + AND pages.timestamp < %(endTimestamp)s + AND pages.ttfb > 0;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def get_top_metrics_avg_time_to_interactive(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(AVG(pages.time_to_interactive), 0) AS value + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND pages.timestamp >= %(startTimestamp)s + AND pages.timestamp < %(endTimestamp)s + AND pages.time_to_interactive > 0;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def get_top_metrics_count_requests(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COUNT(pages.session_id) AS value + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + return helper.dict_to_camel_case(row) diff --git a/api/routers/subs/dashboard.py b/api/routers/subs/dashboard.py index e0b680362..e81e18b5e 100644 --- a/api/routers/subs/dashboard.py +++ b/api/routers/subs/dashboard.py @@ -339,11 +339,10 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body "data": dashboard.get_pages_response_time(project_id=projectId, **data.dict()) }, *helper.explode_widget(dashboard.get_top_metrics(project_id=projectId, **data.dict())), - *helper.explode_widget(data=dashboard.get_time_to_render(project_id=projectId, **data.dict()), - key="avg_time_to_render"), - *helper.explode_widget(dashboard.get_memory_consumption(project_id=projectId, **data.dict())), - *helper.explode_widget(dashboard.get_avg_cpu(project_id=projectId, **data.dict())), - *helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **data.dict())), + {"key": "avg_time_to_render", "data": dashboard.get_time_to_render(project_id=projectId, **data.dict())}, + {"key": "avg_used_js_heap_size", "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, + {"key": "avg_cpu", "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_fps, "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} ]} @@ -351,31 +350,42 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body @app.get('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body(...)): return {"data": [ - # {"key": schemas.TemplateKeys.count_sessions, - # "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplateKeys.avg_image_load_time, - # "data": dashboard.get_application_activity_avg_image_load_time(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplateKeys.avg_page_load_time, - # "data": dashboard.get_application_activity_avg_page_load_time(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplateKeys.avg_request_load_time, - # "data": dashboard.get_application_activity_avg_request_load_time(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplateKeys.avg_dom_content_load_start, - # "data": dashboard.get_page_metrics_avg_dom_content_load_start(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplateKeys.avg_first_contentful_pixel, - # "data": dashboard.get_page_metrics_avg_first_contentful_pixel(project_id=projectId, **data.dict())} - # {"key": schemas.TemplateKeys.avg_visited_pages, - # "data": dashboard.get_user_activity_avg_visited_pages(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplateKeys.avg_session_duration, - # "data": dashboard.get_user_activity_avg_session_duration(project_id=projectId, **data.dict())} - # {"key": schemas.TemplateKeys.avg_pages_dom_buildtime, - # "data": dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict())}, - - *helper.explode_widget(data=dashboard.get_pages_response_time(project_id=projectId, **data.dict()), - key="avg_pages_response_time"), - # *helper.explode_widget(dashboard.get_top_metrics(project_id=projectId, **data.dict())), - # *helper.explode_widget(data=dashboard.get_time_to_render(project_id=projectId, **data.dict()), - # key="avg_time_to_render"), - # *helper.explode_widget(dashboard.get_memory_consumption(project_id=projectId, **data.dict())), - # *helper.explode_widget(dashboard.get_avg_cpu(project_id=projectId, **data.dict())), - # *helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **data.dict())), + {"key": schemas.TemplateKeys.count_sessions, + "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_image_load_time, + "data": dashboard.get_application_activity_avg_image_load_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_page_load_time, + "data": dashboard.get_application_activity_avg_page_load_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_request_load_time, + "data": dashboard.get_application_activity_avg_request_load_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_dom_content_load_start, + "data": dashboard.get_page_metrics_avg_dom_content_load_start(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_first_contentful_pixel, + "data": dashboard.get_page_metrics_avg_first_contentful_pixel(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_visited_pages, + "data": dashboard.get_user_activity_avg_visited_pages(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_session_duration, + "data": dashboard.get_user_activity_avg_session_duration(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_pages_dom_buildtime, + "data": dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_pages_response_time, + "data": dashboard.get_pages_response_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_response_time, + "data": dashboard.get_top_metrics_avg_response_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_first_paint, + "data": dashboard.get_top_metrics_avg_first_paint(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_dom_content_loaded, + "data": dashboard.get_top_metrics_avg_dom_content_loaded(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_till_first_bit, + "data": dashboard.get_top_metrics_avg_till_first_bit(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_time_to_interactive, + "data": dashboard.get_top_metrics_avg_time_to_interactive(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.count_requests, + "data": dashboard.get_top_metrics_count_requests(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_time_to_render, + "data": dashboard.get_time_to_render(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_used_js_heap_size, + "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_cpu, "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_fps, "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} ]} diff --git a/api/schemas.py b/api/schemas.py index 4f672a844..ef1972d98 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -912,5 +912,15 @@ class TemplateKeys(str, Enum): avg_first_contentful_pixel = "avg_first_contentful_pixel" avg_visited_pages = "avg_visited_pages" avg_session_duration = "avg_session_duration" - avg_pages_dom_buildtime="avg_pages_dom_buildtime" - avg_pages_response_time="avg_pages_response_time" + avg_pages_dom_buildtime = "avg_pages_dom_buildtime" + avg_pages_response_time = "avg_pages_response_time" + avg_response_time = "avg_response_time" + avg_first_paint = "avg_first_paint" + avg_dom_content_loaded = "avg_dom_content_loaded" + avg_till_first_bit = "avg_till_first_bit" + avg_time_to_interactive = "avg_time_to_interactive" + count_requests = "count_requests" + avg_time_to_render = "avg_time_to_render" + avg_used_js_heap_size = "avg_used_js_heap_size" + avg_cpu = "avg_cpu" + avg_fps = "avg_fps" diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 13e9e9e14..3080f8dbb 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -6,17 +6,17 @@ SELECT 'v1.5.5' $$ LANGUAGE sql IMMUTABLE; --- CREATE TABLE dashboards --- ( --- dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, --- project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, --- user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, --- name text NOT NULL, --- is_public boolean NOT NULL DEFAULT TRUE, --- is_pinned boolean NOT NULL DEFAULT FALSE, --- created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), --- deleted_at timestamp NULL DEFAULT NULL --- ); +CREATE TABLE IF NOT EXISTS dashboards +( + dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT TRUE, + is_pinned boolean NOT NULL DEFAULT FALSE, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + deleted_at timestamp NULL DEFAULT NULL +); -- CREATE TABLE templates -- ( From 8c975769702f7d355aecc391c211156f0ce8505d Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 30 Mar 2022 18:27:16 +0200 Subject: [PATCH 011/125] codefix(backend): autoCommit as init consumer parameter --- backend/pkg/queue/import.go | 5 +-- backend/pkg/queue/messages.go | 5 +-- backend/pkg/queue/types/types.go | 8 +--- backend/pkg/redisstream/consumer.go | 47 +++++++++------------ backend/services/assets/main.go | 18 ++++---- backend/services/db/main.go | 2 +- backend/services/ender/main.go | 21 +++++----- backend/services/sink/main.go | 64 ++++++++++++++--------------- backend/services/storage/main.go | 48 +++++++++------------- ee/backend/pkg/kafka/consumer.go | 28 ++++++++----- ee/backend/pkg/queue/import.go | 7 ++-- 11 files changed, 117 insertions(+), 136 deletions(-) diff --git a/backend/pkg/queue/import.go b/backend/pkg/queue/import.go index 2bca9c8fd..623d301ca 100644 --- a/backend/pkg/queue/import.go +++ b/backend/pkg/queue/import.go @@ -1,15 +1,14 @@ package queue import ( - "openreplay/backend/pkg/redisstream" "openreplay/backend/pkg/queue/types" + "openreplay/backend/pkg/redisstream" ) -func NewConsumer(group string, topics []string, handler types.MessageHandler) types.Consumer { +func NewConsumer(group string, topics []string, handler types.MessageHandler, _ bool) types.Consumer { return redisstream.NewConsumer(group, topics, handler) } func NewProducer() types.Producer { return redisstream.NewProducer() } - diff --git a/backend/pkg/queue/messages.go b/backend/pkg/queue/messages.go index eca4a4d49..0ab184ee6 100644 --- a/backend/pkg/queue/messages.go +++ b/backend/pkg/queue/messages.go @@ -7,13 +7,12 @@ import ( "openreplay/backend/pkg/queue/types" ) - -func NewMessageConsumer(group string, topics []string, handler types.DecodedMessageHandler) types.Consumer { +func NewMessageConsumer(group string, topics []string, handler types.DecodedMessageHandler, autoCommit bool) types.Consumer { return NewConsumer(group, topics, func(sessionID uint64, value []byte, meta *types.Meta) { if err := messages.ReadBatch(value, func(msg messages.Message) { handler(sessionID, msg, meta) }); err != nil { log.Printf("Decode error: %v\n", err) } - }) + }, autoCommit) } diff --git a/backend/pkg/queue/types/types.go b/backend/pkg/queue/types/types.go index b671323d0..600babe25 100644 --- a/backend/pkg/queue/types/types.go +++ b/backend/pkg/queue/types/types.go @@ -6,26 +6,22 @@ import ( type Consumer interface { ConsumeNext() error - DisableAutoCommit() Commit() error CommitBack(gap int64) error Close() } - type Producer interface { Produce(topic string, key uint64, value []byte) error Close(timeout int) Flush(timeout int) } - type Meta struct { - ID uint64 - Topic string + ID uint64 + Topic string Timestamp int64 } type MessageHandler func(uint64, []byte, *Meta) type DecodedMessageHandler func(uint64, messages.Message, *Meta) - diff --git a/backend/pkg/redisstream/consumer.go b/backend/pkg/redisstream/consumer.go index 164ee9236..d32972981 100644 --- a/backend/pkg/redisstream/consumer.go +++ b/backend/pkg/redisstream/consumer.go @@ -1,24 +1,22 @@ package redisstream import ( + "log" "net" + "sort" "strconv" "strings" - "log" - "sort" "time" - "github.com/pkg/errors" _redis "github.com/go-redis/redis" + "github.com/pkg/errors" "openreplay/backend/pkg/queue/types" ) - - -type idsInfo struct{ - id []string - ts []int64 +type idsInfo struct { + id []string + ts []int64 } type streamPendingIDsMap map[string]*idsInfo @@ -41,26 +39,25 @@ func NewConsumer(group string, streams []string, messageHandler types.MessageHan } } - idsPending := make(streamPendingIDsMap) streamsCount := len(streams) for i := 0; i < streamsCount; i++ { - // ">" is for never-delivered messages. - // Otherwise - never acknoledged only + // ">" is for never-delivered messages. + // Otherwise - never acknoledged only // TODO: understand why in case of "0" it eats 100% cpu - streams = append(streams, ">") - + streams = append(streams, ">") + idsPending[streams[i]] = new(idsInfo) } return &Consumer{ - redis: redis, + redis: redis, messageHandler: messageHandler, - streams: streams, - group: group, - autoCommit: true, - idsPending: idsPending, + streams: streams, + group: group, + autoCommit: true, + idsPending: idsPending, } } @@ -106,9 +103,9 @@ func (c *Consumer) ConsumeNext() error { return errors.New("Too many messages per ms in redis") } c.messageHandler(sessionID, []byte(valueString), &types.Meta{ - Topic: r.Stream, + Topic: r.Stream, Timestamp: int64(ts), - ID: ts << 13 | (idx & 0x1FFF), // Max: 4096 messages/ms for 69 years + ID: ts<<13 | (idx & 0x1FFF), // Max: 4096 messages/ms for 69 years }) if c.autoCommit { if err = c.redis.XAck(r.Stream, c.group, m.ID).Err(); err != nil { @@ -119,7 +116,7 @@ func (c *Consumer) ConsumeNext() error { c.idsPending[r.Stream].id = append(c.idsPending[r.Stream].id, m.ID) c.idsPending[r.Stream].ts = append(c.idsPending[r.Stream].ts, int64(ts)) } - + } } return nil @@ -158,13 +155,9 @@ func (c *Consumer) CommitBack(gap int64) error { c.idsPending[stream].id = idsInfo.id[maxI:] c.idsPending[stream].ts = idsInfo.ts[maxI:] } - return nil -} - -func (c *Consumer) DisableAutoCommit() { - //c.autoCommit = false + return nil } func (c *Consumer) Close() { // noop -} \ No newline at end of file +} diff --git a/backend/services/assets/main.go b/backend/services/assets/main.go index 450dfc83c..664dc5b09 100644 --- a/backend/services/assets/main.go +++ b/backend/services/assets/main.go @@ -18,7 +18,7 @@ import ( func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - GROUP_CACHE := env.String("GROUP_CACHE") + GROUP_CACHE := env.String("GROUP_CACHE") TOPIC_CACHE := env.String("TOPIC_CACHE") cacher := cacher.NewCacher( @@ -29,10 +29,10 @@ func main() { ) consumer := queue.NewMessageConsumer( - GROUP_CACHE, - []string{ TOPIC_CACHE }, + GROUP_CACHE, + []string{TOPIC_CACHE}, func(sessionID uint64, message messages.Message, e *types.Meta) { - switch msg := message.(type) { + switch msg := message.(type) { case *messages.AssetCache: cacher.CacheURL(sessionID, msg.URL) case *messages.ErrorEvent: @@ -47,17 +47,17 @@ func main() { for _, source := range sourceList { cacher.CacheJSFile(source) } - } + } }, + true, ) - tick := time.Tick(20 * time.Minute) sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - log.Printf("Cacher service started\n") + log.Printf("Cacher service started\n") for { select { case sig := <-sigchan: @@ -74,4 +74,4 @@ func main() { } } } -} \ No newline at end of file +} diff --git a/backend/services/db/main.go b/backend/services/db/main.go index d6190a4f0..2ad6e4aa8 100644 --- a/backend/services/db/main.go +++ b/backend/services/db/main.go @@ -74,8 +74,8 @@ func main() { } }) }, + false, ) - consumer.DisableAutoCommit() sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) diff --git a/backend/services/ender/main.go b/backend/services/ender/main.go index e8d739f0e..f0f139dce 100644 --- a/backend/services/ender/main.go +++ b/backend/services/ender/main.go @@ -8,12 +8,12 @@ import ( "os/signal" "syscall" - "openreplay/backend/pkg/intervals" "openreplay/backend/pkg/env" + "openreplay/backend/pkg/intervals" + logger "openreplay/backend/pkg/log" "openreplay/backend/pkg/messages" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/queue/types" - logger "openreplay/backend/pkg/log" "openreplay/backend/services/ender/builder" ) @@ -29,24 +29,24 @@ func main() { producer := queue.NewProducer() consumer := queue.NewMessageConsumer( - GROUP_EVENTS, - []string{ + GROUP_EVENTS, + []string{ env.String("TOPIC_RAW_WEB"), env.String("TOPIC_RAW_IOS"), - }, + }, func(sessionID uint64, msg messages.Message, meta *types.Meta) { statsLogger.HandleAndLog(sessionID, meta) builderMap.HandleMessage(sessionID, msg, msg.Meta().Index) }, + false, ) - consumer.DisableAutoCommit() - + tick := time.Tick(intervals.EVENTS_COMMIT_INTERVAL * time.Millisecond) sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - log.Printf("Ender service started\n") + log.Printf("Ender service started\n") for { select { case sig := <-sigchan: @@ -55,7 +55,7 @@ func main() { consumer.CommitBack(intervals.EVENTS_BACK_COMMIT_GAP) consumer.Close() os.Exit(0) - case <- tick: + case <-tick: builderMap.IterateReadyMessages(time.Now().UnixNano()/1e6, func(sessionID uint64, readyMsg messages.Message) { producer.Produce(TOPIC_TRIGGER, sessionID, messages.Encode(readyMsg)) }) @@ -69,4 +69,3 @@ func main() { } } } - diff --git a/backend/services/sink/main.go b/backend/services/sink/main.go index 5893e93e6..a649bb6ef 100644 --- a/backend/services/sink/main.go +++ b/backend/services/sink/main.go @@ -1,8 +1,8 @@ package main import ( - "log" "encoding/binary" + "log" "time" "os" @@ -10,67 +10,64 @@ import ( "syscall" "openreplay/backend/pkg/env" + . "openreplay/backend/pkg/messages" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/queue/types" - . "openreplay/backend/pkg/messages" ) - - func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - FS_DIR := env.String("FS_DIR"); + FS_DIR := env.String("FS_DIR") if _, err := os.Stat(FS_DIR); os.IsNotExist(err) { log.Fatalf("%v doesn't exist. %v", FS_DIR, err) } writer := NewWriter(env.Uint16("FS_ULIMIT"), FS_DIR) - count := 0 + count := 0 consumer := queue.NewMessageConsumer( env.String("GROUP_SINK"), - []string{ + []string{ env.String("TOPIC_RAW_WEB"), env.String("TOPIC_RAW_IOS"), - }, - func(sessionID uint64, message Message, _ *types.Meta) { - //typeID, err := GetMessageTypeID(value) - // if err != nil { - // log.Printf("Message type decoding error: %v", err) - // return - // } - typeID := message.Meta().TypeID - if !IsReplayerType(typeID) { - return - } + }, + func(sessionID uint64, message Message, _ *types.Meta) { + //typeID, err := GetMessageTypeID(value) + // if err != nil { + // log.Printf("Message type decoding error: %v", err) + // return + // } + typeID := message.Meta().TypeID + if !IsReplayerType(typeID) { + return + } - count++ + count++ - value := message.Encode() - var data []byte - if IsIOSType(typeID) { - data = value - } else { + value := message.Encode() + var data []byte + if IsIOSType(typeID) { + data = value + } else { data = make([]byte, len(value)+8) copy(data[8:], value[:]) binary.LittleEndian.PutUint64(data[0:], message.Meta().Index) - } - if err := writer.Write(sessionID, data); err != nil { + } + if err := writer.Write(sessionID, data); err != nil { log.Printf("Writer error: %v\n", err) } - }, + }, + false, ) - consumer.DisableAutoCommit() - sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - tick := time.Tick(30 * time.Second) + tick := time.Tick(30 * time.Second) - log.Printf("Sink service started\n") + log.Printf("Sink service started\n") for { select { case sig := <-sigchan: @@ -85,7 +82,7 @@ func main() { log.Printf("%v messages during 30 sec", count) count = 0 - + consumer.Commit() default: err := consumer.ConsumeNext() @@ -96,4 +93,3 @@ func main() { } } - diff --git a/backend/services/storage/main.go b/backend/services/storage/main.go index 5033fb845..cd585f10e 100644 --- a/backend/services/storage/main.go +++ b/backend/services/storage/main.go @@ -2,45 +2,41 @@ package main import ( "log" - "time" "os" "strconv" + "time" "os/signal" "syscall" "openreplay/backend/pkg/env" - "openreplay/backend/pkg/storage" "openreplay/backend/pkg/messages" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/queue/types" + "openreplay/backend/pkg/storage" ) - - func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - - storageWeb := storage.NewS3(env.String("AWS_REGION_WEB"), env.String("S3_BUCKET_WEB")) - //storageIos := storage.NewS3(env.String("AWS_REGION_IOS"), env.String("S3_BUCKET_IOS")) + storage := storage.NewS3(env.String("AWS_REGION_WEB"), env.String("S3_BUCKET_WEB")) FS_DIR := env.String("FS_DIR") FS_CLEAN_HRS := env.Int("FS_CLEAN_HRS") - var uploadKey func(string, int, *storage.S3) - uploadKey = func(key string, retryCount int, s *storage.S3) { + var uploadKey func(string, int) + uploadKey = func(key string, retryCount int) { if retryCount <= 0 { - return; + return } file, err := os.Open(FS_DIR + "/" + key) defer file.Close() if err != nil { log.Printf("File error: %v; Will retry %v more time(s)\n", err, retryCount) time.AfterFunc(2*time.Minute, func() { - uploadKey(key, retryCount - 1, s) + uploadKey(key, retryCount-1) }) } else { - if err := s.Upload(gzipFile(file), key, "application/octet-stream", true); err != nil { + if err := storage.Upload(gzipFile(file), key, "application/octet-stream", true); err != nil { log.Fatalf("Storage upload error: %v\n", err) } } @@ -48,27 +44,24 @@ func main() { consumer := queue.NewMessageConsumer( env.String("GROUP_STORAGE"), - []string{ + []string{ env.String("TOPIC_TRIGGER"), - }, - func(sessionID uint64, msg messages.Message, meta *types.Meta) { - switch msg.(type) { - case *messages.SessionEnd: - uploadKey(strconv.FormatUint(sessionID, 10), 5, storageWeb) - //case *messages.IOSSessionEnd: - // uploadKey(strconv.FormatUint(sessionID, 10), 5, storageIos) - } - }, + }, + func(sessionID uint64, msg messages.Message, meta *types.Meta) { + switch msg.(type) { + case *messages.SessionEnd: + uploadKey(strconv.FormatUint(sessionID, 10), 5) + } + }, + true, ) sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + cleanTick := time.Tick(time.Duration(FS_CLEAN_HRS) * time.Hour) - cleanTick := time.Tick(time.Duration(FS_CLEAN_HRS) * time.Hour) - - - log.Printf("Storage service started\n") + log.Printf("Storage service started\n") for { select { case sig := <-sigchan: @@ -85,4 +78,3 @@ func main() { } } } - diff --git a/ee/backend/pkg/kafka/consumer.go b/ee/backend/pkg/kafka/consumer.go index ca0544923..82aa56d50 100644 --- a/ee/backend/pkg/kafka/consumer.go +++ b/ee/backend/pkg/kafka/consumer.go @@ -25,7 +25,12 @@ type Consumer struct { lastKafkaEventTs int64 } -func NewConsumer(group string, topics []string, messageHandler types.MessageHandler) *Consumer { +func NewConsumer( + group string, + topics []string, + messageHandler types.MessageHandler, + autoCommit bool, +) *Consumer { protocol := "plaintext" if env.Bool("KAFKA_USE_SSL") { protocol = "ssl" @@ -53,18 +58,19 @@ func NewConsumer(group string, topics []string, messageHandler types.MessageHand log.Fatalln(err) } + var commitTicker *time.Ticker + if autoCommit { + commitTicker = time.NewTicker(2 * time.Minute) + } + return &Consumer{ c: c, messageHandler: messageHandler, - commitTicker: time.NewTicker(2 * time.Minute), + commitTicker: commitTicker, pollTimeout: 200, } } -func (consumer *Consumer) DisableAutoCommit() { - consumer.commitTicker.Stop() -} - func (consumer *Consumer) Commit() error { consumer.c.Commit() // TODO: return error if it is not "No offset stored" return nil @@ -128,10 +134,12 @@ func (consumer *Consumer) ConsumeNext() error { return nil } - select { - case <-consumer.commitTicker.C: - consumer.Commit() - default: + if consumer.commitTicker != nil { + select { + case <-consumer.commitTicker.C: + consumer.Commit() + default: + } } switch e := ev.(type) { diff --git a/ee/backend/pkg/queue/import.go b/ee/backend/pkg/queue/import.go index abff07e9a..e95eb11e5 100644 --- a/ee/backend/pkg/queue/import.go +++ b/ee/backend/pkg/queue/import.go @@ -2,17 +2,16 @@ package queue import ( "openreplay/backend/pkg/kafka" - "openreplay/backend/pkg/queue/types" "openreplay/backend/pkg/license" + "openreplay/backend/pkg/queue/types" ) -func NewConsumer(group string, topics []string, handler types.MessageHandler) types.Consumer { +func NewConsumer(group string, topics []string, handler types.MessageHandler, autoCommit bool) types.Consumer { license.CheckLicense() - return kafka.NewConsumer(group, topics, handler) + return kafka.NewConsumer(group, topics, handler, autoCommit) } func NewProducer() types.Producer { license.CheckLicense() return kafka.NewProducer() } - From daa5a7fad5ca94eb9041570b241a81666944dfd0 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 30 Mar 2022 18:30:33 +0200 Subject: [PATCH 012/125] codefix(backend): code-format & Close() body on /i --- backend/services/http/handlers_web.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/services/http/handlers_web.go b/backend/services/http/handlers_web.go index 09d2511d8..6020c3eb1 100644 --- a/backend/services/http/handlers_web.go +++ b/backend/services/http/handlers_web.go @@ -11,8 +11,8 @@ import ( "time" "openreplay/backend/pkg/db/postgres" - "openreplay/backend/pkg/token" . "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/token" ) func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { @@ -30,12 +30,12 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { UserID string `json:"userID"` } type response struct { - Timestamp int64 `json:"timestamp"` - Delay int64 `json:"delay"` - Token string `json:"token"` - UserUUID string `json:"userUUID"` - SessionID string `json:"sessionID"` - BeaconSizeLimit int64 `json:"beaconSizeLimit"` + Timestamp int64 `json:"timestamp"` + Delay int64 `json:"delay"` + Token string `json:"token"` + UserUUID string `json:"userUUID"` + SessionID string `json:"sessionID"` + BeaconSizeLimit int64 `json:"beaconSizeLimit"` } startTime := time.Now() @@ -102,7 +102,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { UserCountry: country, UserDeviceMemorySize: req.DeviceMemory, UserDeviceHeapSize: req.JsHeapSizeLimit, - UserID: req.UserID, + UserID: req.UserID, })) } @@ -110,9 +110,9 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { responseWithJSON(w, &response{ //Timestamp: startTime.UnixNano() / 1e6, //Delay: delayDuration.Nanoseconds() / 1e6, - Token: tokenizer.Compose(*tokenData), - UserUUID: userUUID, - SessionID: strconv.FormatUint(tokenData.ID, 10), + Token: tokenizer.Compose(*tokenData), + UserUUID: userUUID, + SessionID: strconv.FormatUint(tokenData.ID, 10), BeaconSizeLimit: BEACON_SIZE_LIMIT, }) } @@ -124,7 +124,7 @@ func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { return } body := http.MaxBytesReader(w, r.Body, BEACON_SIZE_LIMIT) - //defer body.Close() + defer body.Close() buf, err := ioutil.ReadAll(body) if err != nil { responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging @@ -248,4 +248,4 @@ func notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { log.Printf("Unable to insert Unstarted Session: %v\n", err) } w.WriteHeader(http.StatusOK) -} \ No newline at end of file +} From 17436657691391eddd6d9bd8f0b04674ce13d99a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 30 Mar 2022 18:38:48 +0200 Subject: [PATCH 013/125] feat(api): upgrade JIRA feat(api): changed code to support new JIRA SDK feat(api): JIRA helper return error to UI feat(api): JIRA catch 401 --- api/chalicelib/utils/jira_client.py | 35 ++++++++++++++++------------- api/requirements.txt | 2 +- ee/api/requirements.txt | 2 +- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/api/chalicelib/utils/jira_client.py b/api/chalicelib/utils/jira_client.py index d3b637373..78ed06bbc 100644 --- a/api/chalicelib/utils/jira_client.py +++ b/api/chalicelib/utils/jira_client.py @@ -5,19 +5,20 @@ import requests from jira import JIRA from jira.exceptions import JIRAError from requests.auth import HTTPBasicAuth +from starlette import status +from starlette.exceptions import HTTPException fields = "id, summary, description, creator, reporter, created, assignee, status, updated, comment, issuetype, labels" class JiraManager: - # retries = 5 retries = 0 def __init__(self, url, username, password, project_id=None): self._config = {"JIRA_PROJECT_ID": project_id, "JIRA_URL": url, "JIRA_USERNAME": username, "JIRA_PASSWORD": password} try: - self._jira = JIRA({'server': url}, basic_auth=(username, password), logging=True, max_retries=1) + self._jira = JIRA(url, basic_auth=(username, password), logging=True, max_retries=1) except Exception as e: print("!!! JIRA AUTH ERROR") print(e) @@ -34,7 +35,7 @@ class JiraManager: time.sleep(1) return self.get_projects() print(f"=>Error {e.text}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") projects_dict_list = [] for project in projects: projects_dict_list.append(self.__parser_project_info(project)) @@ -50,7 +51,7 @@ class JiraManager: time.sleep(1) return self.get_project() print(f"=>Error {e.text}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") return self.__parser_project_info(project) def get_issues(self, sql: str, offset: int = 0): @@ -66,7 +67,7 @@ class JiraManager: time.sleep(1) return self.get_issues(sql, offset) print(f"=>Error {e.text}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") issue_dict_list = [] for issue in issues: @@ -86,7 +87,7 @@ class JiraManager: time.sleep(1) return self.get_issue(issue_id) print(f"=>Error {e.text}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") return self.__parser_issue_info(issue) def get_issue_v3(self, issue_id: str): @@ -106,7 +107,7 @@ class JiraManager: time.sleep(1) return self.get_issue_v3(issue_id) print(f"=>Error {e}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") return self.__parser_issue_info(issue.json()) def create_issue(self, issue_dict): @@ -120,7 +121,7 @@ class JiraManager: time.sleep(1) return self.create_issue(issue_dict) print(f"=>Error {e.text}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def close_issue(self, issue): try: @@ -132,7 +133,7 @@ class JiraManager: time.sleep(1) return self.close_issue(issue) print(f"=>Error {e.text}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def assign_issue(self, issue_id, account_id) -> bool: try: @@ -143,7 +144,7 @@ class JiraManager: time.sleep(1) return self.assign_issue(issue_id, account_id) print(f"=>Error {e.text}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def add_comment(self, issue_id: str, comment: str): try: @@ -154,7 +155,7 @@ class JiraManager: time.sleep(1) return self.add_comment(issue_id, comment) print(f"=>Error {e.text}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") return self.__parser_comment_info(comment) def add_comment_v3(self, issue_id: str, comment: str): @@ -191,7 +192,7 @@ class JiraManager: time.sleep(1) return self.add_comment_v3(issue_id, comment) print(f"=>Error {e}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") return self.__parser_comment_info(comment_response.json()) def get_comments(self, issueKey): @@ -207,7 +208,7 @@ class JiraManager: time.sleep(1) return self.get_comments(issueKey) print(f"=>Error {e.text}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def get_meta(self): meta = {} @@ -217,14 +218,16 @@ class JiraManager: def get_assignable_users(self): try: - users = self._jira.search_assignable_users_for_issues('', project=self._config['JIRA_PROJECT_ID']) + users = self._jira.search_assignable_users_for_issues(project=self._config['JIRA_PROJECT_ID'], query="*") except JIRAError as e: self.retries -= 1 if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_assignable_users() print(f"=>Error {e.text}") - raise e + if e.status_code == 401: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="JIRA: 401 Unauthorized") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") users_dict = [] for user in users: users_dict.append({ @@ -245,7 +248,7 @@ class JiraManager: time.sleep(1) return self.get_issue_types() print(f"=>Error {e.text}") - raise e + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") types_dict = [] for type in types: if not type.subtask and not type.name.lower() == "epic": diff --git a/api/requirements.txt b/api/requirements.txt index e5672d80a..198b535dd 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -4,7 +4,7 @@ boto3==1.16.1 pyjwt==1.7.1 psycopg2-binary==2.8.6 elasticsearch==7.9.1 -jira==2.0.0 +jira==3.1.1 diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 84a372567..ef143038b 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -4,7 +4,7 @@ boto3==1.16.1 pyjwt==1.7.1 psycopg2-binary==2.8.6 elasticsearch==7.9.1 -jira==2.0.0 +jira==3.1.1 clickhouse-driver==0.2.2 python3-saml==1.12.0 From 7d18c093ebf6479ce7aed626160ea93d6c6cd715 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 30 Mar 2022 20:00:58 +0200 Subject: [PATCH 014/125] feat(api): fixed JIRA error handling feat(api): fixed dashboard split old metrics --- api/chalicelib/core/dashboard.py | 22 ++++++++++----------- api/chalicelib/utils/jira_client.py | 30 ++++++++++++++--------------- api/routers/subs/dashboard.py | 12 ++++++++---- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index bab19ba12..6de949047 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -2261,14 +2261,15 @@ def get_application_activity_avg_image_load_time(project_id, startTimestamp=Time endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: row = __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - results = helper.dict_to_camel_case(row) + results = row + results["chart"] = get_performance_avg_image_load_time(project_id, startTimestamp, endTimestamp, **args) diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff row = __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) previous = helper.dict_to_camel_case(row) results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - results["chart"] = get_performance_avg_image_load_time(project_id, startTimestamp, endTimestamp, **args) + return results @@ -2297,8 +2298,7 @@ def get_performance_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(d AND resources.type = 'img' AND resources.duration>0 {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} ) - SELECT - generated_timestamp AS timestamp, + SELECT generated_timestamp AS timestamp, COALESCE(AVG(resources.duration),0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( @@ -2337,14 +2337,14 @@ def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeU endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: row = __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - results = helper.dict_to_camel_case(row) + results = row + results["chart"] = get_performance_avg_page_load_time(project_id, startTimestamp, endTimestamp, **args) diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff row = __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) previous = helper.dict_to_camel_case(row) results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - results["chart"] = get_performance_avg_page_load_time(project_id, startTimestamp, endTimestamp, **args) return results @@ -2369,8 +2369,7 @@ def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(de WHERE {" AND ".join(pg_sub_query_subset)} AND pages.load_time>0 AND pages.load_time IS NOT NULL {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} ) - SELECT - generated_timestamp AS timestamp, + SELECT generated_timestamp AS timestamp, COALESCE(AVG(pages.load_time),0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT pages.load_time @@ -2407,14 +2406,14 @@ def get_application_activity_avg_request_load_time(project_id, startTimestamp=Ti endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: row = __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) - results = helper.dict_to_camel_case(row) + results = row + results["chart"] = get_performance_avg_request_load_time(project_id, startTimestamp, endTimestamp, **args) diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff row = __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) previous = helper.dict_to_camel_case(row) results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - results["chart"] = get_performance_avg_request_load_time(project_id, startTimestamp, endTimestamp, **args) return results @@ -2447,8 +2446,7 @@ def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now AND resources.type = 'fetch' AND resources.duration>0 {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} ) - SELECT - generated_timestamp AS timestamp, + SELECT generated_timestamp AS timestamp, COALESCE(AVG(resources.duration),0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( diff --git a/api/chalicelib/utils/jira_client.py b/api/chalicelib/utils/jira_client.py index 78ed06bbc..8370feb5e 100644 --- a/api/chalicelib/utils/jira_client.py +++ b/api/chalicelib/utils/jira_client.py @@ -34,7 +34,7 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_projects() - print(f"=>Error {e.text}") + print(f"=>Exception {e.text}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") projects_dict_list = [] for project in projects: @@ -50,7 +50,7 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_project() - print(f"=>Error {e.text}") + print(f"=>Exception {e.text}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") return self.__parser_project_info(project) @@ -66,7 +66,7 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_issues(sql, offset) - print(f"=>Error {e.text}") + print(f"=>Exception {e.text}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") issue_dict_list = [] @@ -86,7 +86,7 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_issue(issue_id) - print(f"=>Error {e.text}") + print(f"=>Exception {e.text}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") return self.__parser_issue_info(issue) @@ -106,8 +106,8 @@ class JiraManager: if self.retries > 0: time.sleep(1) return self.get_issue_v3(issue_id) - print(f"=>Error {e}") - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") + print(f"=>Exception {e}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: get issue error") return self.__parser_issue_info(issue.json()) def create_issue(self, issue_dict): @@ -120,7 +120,7 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.create_issue(issue_dict) - print(f"=>Error {e.text}") + print(f"=>Exception {e.text}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def close_issue(self, issue): @@ -132,7 +132,7 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.close_issue(issue) - print(f"=>Error {e.text}") + print(f"=>Exception {e.text}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def assign_issue(self, issue_id, account_id) -> bool: @@ -143,7 +143,7 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.assign_issue(issue_id, account_id) - print(f"=>Error {e.text}") + print(f"=>Exception {e.text}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def add_comment(self, issue_id: str, comment: str): @@ -154,7 +154,7 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.add_comment(issue_id, comment) - print(f"=>Error {e.text}") + print(f"=>Exception {e.text}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") return self.__parser_comment_info(comment) @@ -191,8 +191,8 @@ class JiraManager: if self.retries > 0: time.sleep(1) return self.add_comment_v3(issue_id, comment) - print(f"=>Error {e}") - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") + print(f"=>Exception {e}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: comment error") return self.__parser_comment_info(comment_response.json()) def get_comments(self, issueKey): @@ -207,7 +207,7 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_comments(issueKey) - print(f"=>Error {e.text}") + print(f"=>Exception {e.text}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def get_meta(self): @@ -224,7 +224,7 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_assignable_users() - print(f"=>Error {e.text}") + print(f"=>Exception {e.text}") if e.status_code == 401: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="JIRA: 401 Unauthorized") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") @@ -247,7 +247,7 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_issue_types() - print(f"=>Error {e.text}") + print(f"=>Exception {e.text}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") types_dict = [] for type in types: diff --git a/api/routers/subs/dashboard.py b/api/routers/subs/dashboard.py index e81e18b5e..fdb5d641a 100644 --- a/api/routers/subs/dashboard.py +++ b/api/routers/subs/dashboard.py @@ -325,7 +325,7 @@ def get_dashboard_resources_count_by_type(projectId: int, data: schemas.MetricPa @app.post('/{projectId}/dashboard/overview', tags=["dashboard", "metrics"]) @app.get('/{projectId}/dashboard/overview', tags=["dashboard", "metrics"]) def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body(...)): - return {"data": [ + results = [ {"key": "count_sessions", "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, *helper.explode_widget(data={**dashboard.get_application_activity(project_id=projectId, **data.dict()), @@ -343,13 +343,15 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body {"key": "avg_used_js_heap_size", "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, {"key": "avg_cpu", "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, {"key": schemas.TemplateKeys.avg_fps, "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} - ]} + ] + results = sorted(results, key=lambda r: r["key"]) + return {"data": results} @app.post('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) @app.get('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body(...)): - return {"data": [ + results = [ {"key": schemas.TemplateKeys.count_sessions, "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, {"key": schemas.TemplateKeys.avg_image_load_time, @@ -388,4 +390,6 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, {"key": schemas.TemplateKeys.avg_cpu, "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, {"key": schemas.TemplateKeys.avg_fps, "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} - ]} + ] + results = sorted(results, key=lambda r: r["key"]) + return {"data": results} From af1c88a573424926bd37d9d769ad866dd43215ce Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 30 Mar 2022 20:29:22 +0200 Subject: [PATCH 015/125] feat(api): create dashboard with widgets at the same time --- api/chalicelib/core/dashboards2.py | 17 +++++++++++++---- api/routers/subs/metrics.py | 4 ++-- api/schemas.py | 18 ++++++++++-------- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 215a4f9ea..adb7884fe 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -27,11 +27,20 @@ def create_dashboard(project_id, user_id, data: schemas.CreateDashboardSchema): with pg_client.PostgresClient() as cur: pg_query = f"""INSERT INTO dashboards(project_id, user_id, name, is_public, is_pinned) VALUES(%(projectId)s, %(userId)s, %(name)s, %(is_public)s, %(is_pinned)s) - RETURNING *;""" + RETURNING *""" params = {"userId": user_id, "projectId": project_id, **data.dict()} + if data.metrics is not None and len(data.metrics) > 0: + pg_query = f"""WITH dash AS ({pg_query}) + INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id) + VALUES {",".join([f"((SELECT dashboard_id FROM dash),%(metric_id_{i})s, %(userId)s)" for i in range(len(data.metrics))])} + RETURNING (SELECT dashboard_id FROM dash)""" + for i, m in enumerate(data.metrics): + params[f"metric_id_{i}"] = m cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() - return helper.dict_to_camel_case(row) + if row is None: + return {"errors": ["something went wrong while creating the dashboard"]} + return {"data": get_dashboard(project_id=project_id, user_id=user_id, dashboard_id=row["dashboard_id"])} def get_dashboards(project_id, user_id): @@ -87,10 +96,10 @@ def delete_dashboard(project_id, user_id, dashboard_id): return {"data": {"success": True}} -def update_dashboard(project_id, user_id, dashboard_id, data: schemas.CreateDashboardSchema): +def update_dashboard(project_id, user_id, dashboard_id, data: schemas.EditDashboardSchema): with pg_client.PostgresClient() as cur: pg_query = """UPDATE dashboards - SET name = %(name)s, is_pinned = %(is_pinned)s, is_public = %(is_public)s + SET name = %(name)s, is_public = %(is_public)s WHERE dashboards.project_id = %(projectId)s AND dashboard_id = %(dashboard_id)s AND (dashboards.user_id = %(userId)s OR is_public) diff --git a/api/routers/subs/metrics.py b/api/routers/subs/metrics.py index 2d3c116d3..88cfd3937 100644 --- a/api/routers/subs/metrics.py +++ b/api/routers/subs/metrics.py @@ -12,7 +12,7 @@ public_app, app, app_apikey = get_routers() @app.put('/{projectId}/dashboards', tags=["dashboard"]) def create_dashboards(projectId: int, data: schemas.CreateDashboardSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return {"data": dashboards2.create_dashboard(project_id=projectId, user_id=context.user_id, data=data)} + return dashboards2.create_dashboard(project_id=projectId, user_id=context.user_id, data=data) @app.get('/{projectId}/dashboards', tags=["dashboard"]) @@ -27,7 +27,7 @@ def get_dashboard(projectId: int, dashboardId: int, context: schemas.CurrentCont @app.post('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) @app.put('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) -def update_dashboard(projectId: int, dashboardId: int, data: schemas.CreateDashboardSchema = Body(...), +def update_dashboard(projectId: int, dashboardId: int, data: schemas.EditDashboardSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): return {"data": dashboards2.update_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, data=data)} diff --git a/api/schemas.py b/api/schemas.py index ef1972d98..0135aafc0 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -881,23 +881,25 @@ class CreateDashboardSchema(BaseModel): name: str = Field(..., min_length=1) is_public: bool = Field(default=False) is_pinned: bool = Field(default=False) + metrics: Optional[List[int]] = Field(default=[]) + + class Config: + alias_generator = attribute_to_camel_case + + +class EditDashboardSchema(BaseModel): + name: str = Field(..., min_length=1) + is_public: bool = Field(default=False) class Config: alias_generator = attribute_to_camel_case class AddWidgetToDashboardPayloadSchema(BaseModel): - metric_id: Optional[int] = Field(default=None) + metric_id: int = Field(default=None) name: Optional[str] = Field(default=None) config: dict = Field(default={}) - @root_validator - def validator(cls, values): - assert bool(values.get("template_id") is not None) != bool(values.get("metric_id") is not None), \ - f"templateId or metricId should be provided, but not both at the same time" - - return values - class Config: alias_generator = attribute_to_camel_case From e909321040c0787c601df1adec724a5f80aead0e Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 30 Mar 2022 20:38:30 +0200 Subject: [PATCH 016/125] feat(api): create new metric and add it to dashboard --- api/chalicelib/core/custom_metrics.py | 4 +++- api/chalicelib/core/dashboards2.py | 7 +++++++ api/routers/subs/metrics.py | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index e0b0ed432..88f8fca10 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -102,7 +102,7 @@ def get_sessions(project_id, user_id, metric_id, data: schemas.CustomMetricSessi return results -def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): +def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema, dashboard=False): with pg_client.PostgresClient() as cur: _data = {} for i, s in enumerate(data.series): @@ -129,6 +129,8 @@ def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): query ) r = cur.fetchone() + if dashboard: + return r["metric_id"] return {"data": get(metric_id=r["metric_id"], project_id=project_id, user_id=user_id)} diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index adb7884fe..c292ed1a0 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -1,6 +1,7 @@ import json import schemas +from chalicelib.core import custom_metrics from chalicelib.utils import helper from chalicelib.utils import pg_client @@ -162,3 +163,9 @@ def pin_dashboard(project_id, user_id, dashboard_id): cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() return helper.dict_to_camel_case(row) + + +def create_metric_add_widget(project_id, user_id, dashboard_id, data: schemas.CreateCustomMetricsSchema): + metric_id = custom_metrics.create(project_id=project_id, user_id=user_id, data=data, dashboard=True) + return add_widget(project_id=project_id, user_id=user_id, dashboard_id=dashboard_id, + data=schemas.AddWidgetToDashboardPayloadSchema(metric_id=metric_id)) diff --git a/api/routers/subs/metrics.py b/api/routers/subs/metrics.py index 88cfd3937..bcec3d167 100644 --- a/api/routers/subs/metrics.py +++ b/api/routers/subs/metrics.py @@ -51,6 +51,12 @@ def add_widget_to_dashboard(projectId: int, dashboardId: int, return {"data": dashboards2.add_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, data=data)} +@app.post('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard"]) +@app.put('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard"]) +def create_metric_and_add_to_dashboard(projectId: int, dashboardId: int, data: schemas.CreateCustomMetricsSchema = Body(...),context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.create_metric_add_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, + data=data)} + @app.post('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) @app.put('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) From b799704efd051be7eb6e6edbc8a5607a077b70ea Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 30 Mar 2022 20:39:06 +0200 Subject: [PATCH 017/125] feat(api): dashboard format --- api/routers/subs/metrics.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/api/routers/subs/metrics.py b/api/routers/subs/metrics.py index bcec3d167..4504ba8f1 100644 --- a/api/routers/subs/metrics.py +++ b/api/routers/subs/metrics.py @@ -51,11 +51,15 @@ def add_widget_to_dashboard(projectId: int, dashboardId: int, return {"data": dashboards2.add_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, data=data)} + @app.post('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard"]) @app.put('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard"]) -def create_metric_and_add_to_dashboard(projectId: int, dashboardId: int, data: schemas.CreateCustomMetricsSchema = Body(...),context: schemas.CurrentContext = Depends(OR_context)): - return {"data": dashboards2.create_metric_add_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, - data=data)} +def create_metric_and_add_to_dashboard(projectId: int, dashboardId: int, + data: schemas.CreateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.create_metric_add_widget(project_id=projectId, user_id=context.user_id, + dashboard_id=dashboardId, + data=data)} @app.post('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) From 9e294d4edd6f52a67eab190e3f3772af7b0d460a Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 31 Mar 2022 18:32:27 +0200 Subject: [PATCH 018/125] fix(nginx): socketio endpoint directly from kube service nginx upstream block has some issue with forwarding the socketio connection Signed-off-by: rjshrjndrn --- .../openreplay/charts/nginx-ingress/templates/configMap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml index 6780c9ee6..6322318e3 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml @@ -82,7 +82,7 @@ data: proxy_set_header Host $host; proxy_set_header X-Forwarded-For $origin_forwarded_ip; proxy_set_header X-Real-IP $origin_forwarded_ip; - proxy_pass http://utilities-pool; + proxy_pass http://utilities-openreplay.app.svc.cluster.local:9001; } location /assets/ { rewrite ^/assets/(.*) /sessions-assets/$1 break; From 77fc8221ac6b49631ff92b8f7cc75558f917cd81 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 31 Mar 2022 19:50:07 +0200 Subject: [PATCH 019/125] chore(helm): moving utils default vars Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/charts/utilities/values.yaml | 3 +++ scripts/helmcharts/vars.yaml | 7 ------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/utilities/values.yaml b/scripts/helmcharts/openreplay/charts/utilities/values.yaml index 87b3d771d..8265b5aa5 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/values.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/values.yaml @@ -83,6 +83,9 @@ autoscaling: env: REDIS_URL: "redis://redis-master.db.svc.cluster.local:6379" + debug: 0 + uws: false + redis: false nodeSelector: {} diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index e4ed88711..24ea9344d 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -95,13 +95,6 @@ chalice: # idp_name: '' # idp_tenantKey: '' -utilities: - replicaCount: 1 - env: - debug: 0 - uws: false - redis: false - # If you want to override something # chartname: # filedFrom chart/Values.yaml: From cfcff0293aeca227cded754ae3a18ffd2528dd79 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 1 Apr 2022 15:26:24 +0200 Subject: [PATCH 020/125] feat(api): return attached dashboards with the custom metrics list feat(api): allow custom metrics visibility --- api/chalicelib/core/custom_metrics.py | 42 +++++++++++++++++++-------- api/schemas.py | 2 +- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 88f8fca10..6492080f8 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -205,29 +205,40 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche return get(metric_id=metric_id, project_id=project_id, user_id=user_id) -def get_all(project_id, user_id): +def get_all(project_id, user_id, include_series=False): with pg_client.PostgresClient() as cur: - cur.execute( - cur.mogrify( - """SELECT * - FROM metrics - LEFT JOIN LATERAL (SELECT jsonb_agg(metric_series.* ORDER BY index) AS series + sub_join = "" + if include_series: + sub_join = """LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(metric_series.* ORDER BY index),'[]'::jsonb) AS series FROM metric_series WHERE metric_series.metric_id = metrics.metric_id AND metric_series.deleted_at ISNULL - ) AS metric_series ON (TRUE) + ) AS metric_series ON (TRUE)""" + cur.execute( + cur.mogrify( + f"""SELECT * + FROM metrics + {sub_join} + LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards + FROM (SELECT dashboard_id, name, is_public + FROM dashboards + WHERE deleted_at ISNULL + AND project_id = %(project_id)s + AND ((user_id = %(user_id)s OR is_public))) AS connected_dashboards + ) AS connected_dashboards ON (TRUE) WHERE metrics.project_id = %(project_id)s AND metrics.deleted_at ISNULL - AND (user_id = %(user_id)s OR is_public) + AND (user_id = %(user_id)s OR metrics.is_public) ORDER BY created_at;""", {"project_id": project_id, "user_id": user_id} ) ) rows = cur.fetchall() - for r in rows: - r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) - for s in r["series"]: - s["filter"] = helper.old_search_payload_to_flat(s["filter"]) + if include_series: + for r in rows: + # r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + for s in r["series"]: + s["filter"] = helper.old_search_payload_to_flat(s["filter"]) rows = helper.list_to_camel_case(rows) return rows @@ -258,6 +269,13 @@ def get(metric_id, project_id, user_id, flatten=True): WHERE metric_series.metric_id = metrics.metric_id AND metric_series.deleted_at ISNULL ) AS metric_series ON (TRUE) + LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards + FROM (SELECT dashboard_id, name, is_public + FROM dashboards + WHERE deleted_at ISNULL + AND project_id = %(project_id)s + AND ((user_id = %(user_id)s OR is_public))) AS connected_dashboards + ) AS connected_dashboards ON (TRUE) WHERE metrics.project_id = %(project_id)s AND metrics.deleted_at ISNULL AND (metrics.user_id = %(user_id)s OR metrics.is_public) diff --git a/api/schemas.py b/api/schemas.py index 0135aafc0..821326728 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -820,7 +820,7 @@ class CustomMetricChartPayloadSchema(CustomMetricSessionsPayloadSchema): class CreateCustomMetricsSchema(CustomMetricChartPayloadSchema): name: str = Field(...) series: List[CustomMetricCreateSeriesSchema] = Field(..., min_items=1) - is_public: bool = Field(default=True, const=True) + is_public: bool = Field(default=True) view_type: Union[MetricTimeseriesViewType, MetricTableViewType] = Field(MetricTimeseriesViewType.line_chart) metric_type: MetricType = Field(MetricType.timeseries) metric_of: Union[TableMetricOfType, TimeseriesMetricOfType] = Field(TableMetricOfType.user_id) From a0a82e0d77a75d6154ea6fffc5e58128880fa117 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Fri, 1 Apr 2022 17:19:46 +0200 Subject: [PATCH 021/125] codeclean(backend): remove alert.go --- backend/pkg/db/postgres/alert.go | 228 ------------------------------- 1 file changed, 228 deletions(-) delete mode 100644 backend/pkg/db/postgres/alert.go diff --git a/backend/pkg/db/postgres/alert.go b/backend/pkg/db/postgres/alert.go deleted file mode 100644 index 964977bd3..000000000 --- a/backend/pkg/db/postgres/alert.go +++ /dev/null @@ -1,228 +0,0 @@ -package postgres - -import ( - "database/sql" - "errors" - "fmt" - sq "github.com/Masterminds/squirrel" - "log" - "strconv" - "time" -) - -type TimeString sql.NullString -type query struct { - Left string `db:"query.left" json:"left"` - Operator string `db:"query.operator" json:"operator"` - Right float64 `db:"query.right" json:"right"` -} -type options struct { - RenotifyInterval int64 `db:"options.renotifyInterval" json:"renotifyInterval"` - LastNotification int64 `db:"options.lastNotification" json:"lastNotification;omitempty"` - CurrentPeriod int64 `db:"options.currentPeriod" json:"currentPeriod"` - PreviousPeriod int64 `db:"options.previousPeriod" json:"previousPeriod;omitempty"` - Message []map[string]string `db:"options.message" json:"message;omitempty"` - Change string `db:"options.change" json:"change;omitempty"` -} -type Alert struct { - AlertID uint32 `db:"alert_id" json:"alert_id"` - ProjectID uint32 `db:"project_id" json:"project_id"` - Name string `db:"name" json:"name"` - Description sql.NullString `db:"description" json:"description"` - Active bool `db:"active" json:"active"` - DetectionMethod string `db:"detection_method" json:"detection_method"` - Query query `db:"query" json:"query"` - DeletedAt *int64 `db:"deleted_at" json:"deleted_at"` - CreatedAt *int64 `db:"created_at" json:"created_at"` - Options options `db:"options" json:"options"` - TenantId uint32 `db:"tenant_id" json:"tenant_id"` -} - -func (pg *Conn) IterateAlerts(iter func(alert *Alert, err error)) error { - rows, err := pg.query(` - SELECT - alerts.alert_id, - alerts.project_id, - alerts.name, - alerts.description, - alerts.active, - alerts.detection_method, - alerts.query, - CAST(EXTRACT(epoch FROM alerts.deleted_at) * 1000 AS BIGINT) AS deleted_at, - CAST(EXTRACT(epoch FROM alerts.created_at) * 1000 AS BIGINT) AS created_at, - alerts.options, - 0 AS tenant_id - FROM public.alerts - WHERE alerts.active AND alerts.deleted_at ISNULL; - `) - if err != nil { - return err - } - defer rows.Close() - for rows.Next() { - a := new(Alert) - if err = rows.Scan( - &a.AlertID, - &a.ProjectID, - &a.Name, - &a.Description, - &a.Active, - &a.DetectionMethod, - &a.Query, - &a.DeletedAt, - &a.CreatedAt, - &a.Options, - &a.TenantId, - ); err != nil { - iter(nil, err) - continue - } - iter(a, nil) - } - - if err = rows.Err(); err != nil { - return err - } - return nil -} - -func (pg *Conn) SaveLastNotification(allIds []uint32) error { - var paramrefs string - for _, v := range allIds { - paramrefs += strconv.Itoa(int(v)) + `,` - } - paramrefs = paramrefs[:len(paramrefs)-1] // remove last "," - q := "UPDATE public.Alerts SET options = options||'{\"lastNotification\":" + strconv.Itoa(int(time.Now().Unix()*1000)) + "}'::jsonb WHERE alert_id IN (" + paramrefs + ");" - //log.Println(q) - log.Println("Updating PG") - return pg.exec(q) -} - -type columnDefinition struct { - table string - formula string - condition string - group string -} - -var LeftToDb = map[string]columnDefinition{ - "performance.dom_content_loaded.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "COALESCE(AVG(NULLIF(dom_content_loaded_time ,0)),0)"}, - "performance.first_meaningful_paint.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "COALESCE(AVG(NULLIF(first_contentful_paint_time,0)),0)"}, - "performance.page_load_time.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(load_time ,0))"}, - "performance.dom_build_time.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(dom_building_time,0))"}, - "performance.speed_index.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(speed_index,0))"}, - "performance.page_response_time.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(response_time,0))"}, - "performance.ttfb.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(first_paint_time,0))"}, - "performance.time_to_render.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(visually_complete,0))"}, - "performance.image_load_time.average": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(resources.duration,0))", condition: "type='img'"}, - "performance.request_load_time.average": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(resources.duration,0))", condition: "type='fetch'"}, - "resources.load_time.average": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(resources.duration,0))"}, - "resources.missing.count": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "COUNT(DISTINCT url_hostpath)", condition: "success= FALSE"}, - "errors.4xx_5xx.count": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "COUNT(session_id)", condition: "status/100!=2"}, - "errors.4xx.count": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "COUNT(session_id)", condition: "status/100=4"}, - "errors.5xx.count": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "COUNT(session_id)", condition: "status/100=5"}, - "errors.javascript.impacted_sessions.count": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "COUNT(DISTINCT session_id)", condition: "success= FALSE AND type='script'"}, - "performance.crashes.count": {table: "(SELECT *, start_ts AS timestamp FROM public.sessions WHERE errors_count > 0) AS sessions", formula: "COUNT(DISTINCT session_id)", condition: "errors_count > 0"}, - "errors.javascript.count": {table: "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", formula: "COUNT(DISTINCT session_id)", condition: "source='js_exception'"}, - "errors.backend.count": {table: "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", formula: "COUNT(DISTINCT session_id)", condition: "source!='js_exception'"}, -} - -//This is the frequency of execution for each threshold -var TimeInterval = map[int64]int64{ - 15: 3, - 30: 5, - 60: 10, - 120: 20, - 240: 30, - 1440: 60, -} - -func (a *Alert) CanCheck() bool { - now := time.Now().Unix() * 1000 - var repetitionBase int64 - - if repetitionBase = a.Options.CurrentPeriod; a.DetectionMethod == "change" && a.Options.CurrentPeriod > a.Options.PreviousPeriod { - repetitionBase = a.Options.PreviousPeriod - } - - if _, ok := TimeInterval[repetitionBase]; !ok { - log.Printf("repetitionBase: %d NOT FOUND", repetitionBase) - return false - } - return a.DeletedAt == nil && a.Active && - (a.Options.RenotifyInterval <= 0 || - a.Options.LastNotification <= 0 || - ((now - a.Options.LastNotification) > a.Options.RenotifyInterval*60*1000)) && - ((now-*a.CreatedAt)%(TimeInterval[repetitionBase]*60*1000)) < 60*1000 -} - -func (a *Alert) Build() (sq.SelectBuilder, error) { - colDef, ok := LeftToDb[a.Query.Left] - if !ok { - return sq.Select(), errors.New(fmt.Sprintf("!! unsupported metric '%s' from alert: %d:%s\n", a.Query.Left, a.AlertID, a.Name)) - } - - subQ := sq. - Select(colDef.formula + " AS value"). - From(colDef.table). - Where(sq.And{sq.Expr("project_id = $1 ", a.ProjectID), - sq.Expr(colDef.condition)}) - q := sq.Select(fmt.Sprint("value, coalesce(value,0)", a.Query.Operator, a.Query.Right, " AS valid")) - if len(colDef.group) > 0 { - subQ = subQ.Column(colDef.group + " AS group_value") - subQ = subQ.GroupBy(colDef.group) - q = q.Column("group_value") - } - - if a.DetectionMethod == "threshold" { - q = q.FromSelect(subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod*60)), "stat") - } else if a.DetectionMethod == "change" { - if a.Options.Change == "change" { - if len(colDef.group) == 0 { - sub1, args1, _ := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod*60)).ToSql() - sub2, args2, _ := subQ.Where( - sq.And{ - sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod*60), - sq.Expr("timestamp>=$4 ", time.Now().Unix()-2*a.Options.CurrentPeriod*60), - }).ToSql() - sub1, _, _ = sq.Expr("SELECT ((" + sub1 + ")-(" + sub2 + ")) AS value").ToSql() - q = q.JoinClause("FROM ("+sub1+") AS stat", append(args1, args2...)...) - } else { - subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod*60)) - sub2, args2, _ := subQ.Where( - sq.And{ - sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod*60), - sq.Expr("timestamp>=$4 ", time.Now().Unix()-2*a.Options.CurrentPeriod*60), - }).ToSql() - sub1 := sq.Select("group_value", "(stat1.value-stat2.value) AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) - q = q.FromSelect(sub1, "stat") - } - } else if a.Options.Change == "percent" { - if len(colDef.group) == 0 { - sub1, args1, _ := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod*60)).ToSql() - sub2, args2, _ := subQ.Where( - sq.And{ - sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod*60), - sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod*60-a.Options.CurrentPeriod*60), - }).ToSql() - sub1, _, _ = sq.Expr("SELECT ((" + sub1 + ")/(" + sub2 + ")-1)*100 AS value").ToSql() - q = q.JoinClause("FROM ("+sub1+") AS stat", append(args1, args2...)...) - } else { - subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod*60)) - sub2, args2, _ := subQ.Where( - sq.And{ - sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod*60), - sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod*60-a.Options.CurrentPeriod*60), - }).ToSql() - sub1 := sq.Select("group_value", "(stat1.value/stat2.value-1)*100 AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) - q = q.FromSelect(sub1, "stat") - } - } else { - return q, errors.New("unsupported change method") - } - - } else { - return q, errors.New("unsupported detection method") - } - return q, nil -} \ No newline at end of file From 197c39f96ac83200f67d0ff40db86a93373b19eb Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Fri, 1 Apr 2022 19:14:08 +0200 Subject: [PATCH 022/125] feat(backend): go v.1.18 upgrade --- backend/Dockerfile | 2 +- backend/go.mod | 43 +++++- backend/go.sum | 21 +-- ...{messages_common.go => messages-common.go} | 24 --- .../{messages_ios.go => messages-ios.go} | 0 .../{messages_web.go => messages-web.go} | 0 .../pkg/db/cache/{pg_cache.go => pg-cache.go} | 0 backend/pkg/db/postgres/listener.go | 25 ---- ...{messages_common.go => messages-common.go} | 0 .../{messages_ios.go => messages-ios.go} | 0 ...ges_web_stats.go => messages-web-stats.go} | 0 .../{messages_web.go => messages-web.go} | 0 ...tarted_session.go => unstarted-session.go} | 0 .../pkg/env/{worker_id.go => worker-id.go} | 0 .../{get_timestamp.go => get-timestamp.go} | 0 ...ansform.go => legacy-message-transform.go} | 0 .../{read_message.go => read-message.go} | 0 ...s_depricated.go => handlers-depricated.go} | 0 .../http/{handlers_ios.go => handlers-ios.go} | 98 ++++++------- .../http/{handlers_web.go => handlers-web.go} | 0 backend/services/http/ios-device.go | 138 ++++++++++++++++++ backend/services/http/ios_device.go | 79 ---------- backend/services/http/main.go | 18 +-- backend/services/http/project_id.go | 12 -- 24 files changed, 235 insertions(+), 225 deletions(-) rename backend/pkg/db/cache/{messages_common.go => messages-common.go} (66%) rename backend/pkg/db/cache/{messages_ios.go => messages-ios.go} (100%) rename backend/pkg/db/cache/{messages_web.go => messages-web.go} (100%) rename backend/pkg/db/cache/{pg_cache.go => pg-cache.go} (100%) rename backend/pkg/db/postgres/{messages_common.go => messages-common.go} (100%) rename backend/pkg/db/postgres/{messages_ios.go => messages-ios.go} (100%) rename backend/pkg/db/postgres/{messages_web_stats.go => messages-web-stats.go} (100%) rename backend/pkg/db/postgres/{messages_web.go => messages-web.go} (100%) rename backend/pkg/db/postgres/{unstarted_session.go => unstarted-session.go} (100%) rename backend/pkg/env/{worker_id.go => worker-id.go} (100%) rename backend/pkg/messages/{get_timestamp.go => get-timestamp.go} (100%) rename backend/pkg/messages/{legacy_message_transform.go => legacy-message-transform.go} (100%) rename backend/pkg/messages/{read_message.go => read-message.go} (100%) rename backend/services/http/{handlers_depricated.go => handlers-depricated.go} (100%) rename backend/services/http/{handlers_ios.go => handlers-ios.go} (71%) rename backend/services/http/{handlers_web.go => handlers-web.go} (100%) create mode 100644 backend/services/http/ios-device.go delete mode 100644 backend/services/http/ios_device.go delete mode 100644 backend/services/http/project_id.go diff --git a/backend/Dockerfile b/backend/Dockerfile index 5cefd4cb4..4c31518ef 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13-alpine3.10 AS prepare +FROM golang:1.18-alpine3.15 AS prepare RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash diff --git a/backend/go.mod b/backend/go.mod index ab98ca444..6588529a8 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,14 +1,12 @@ module openreplay/backend -go 1.13 +go 1.18 require ( cloud.google.com/go/logging v1.4.2 github.com/ClickHouse/clickhouse-go v1.4.3 - github.com/Masterminds/squirrel v1.5.0 github.com/aws/aws-sdk-go v1.35.23 github.com/btcsuite/btcutil v1.0.2 - github.com/confluentinc/confluent-kafka-go v1.7.0 // indirect github.com/elastic/go-elasticsearch/v7 v7.13.1 github.com/go-redis/redis v6.15.9+incompatible github.com/google/uuid v1.1.2 @@ -16,14 +14,47 @@ require ( github.com/jackc/pgconn v1.6.0 github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 github.com/jackc/pgx/v4 v4.6.0 - github.com/klauspost/compress v1.11.9 // indirect github.com/klauspost/pgzip v1.2.5 - github.com/lib/pq v1.2.0 github.com/oschwald/maxminddb-golang v1.7.0 github.com/pkg/errors v0.9.1 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce github.com/ua-parser/uap-go v0.0.0-20200325213135-e1c09f13e2fe + golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 google.golang.org/api v0.50.0 gopkg.in/confluentinc/confluent-kafka-go.v1 v1.7.0 - +) + +require ( + cloud.google.com/go v0.84.0 // indirect + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect + github.com/confluentinc/confluent-kafka-go v1.7.0 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.0.2 // indirect + github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 // indirect + github.com/jackc/pgtype v1.3.0 // indirect + github.com/jackc/puddle v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/klauspost/compress v1.11.9 // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/mod v0.4.2 // indirect + golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/tools v0.1.4 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84 // indirect + google.golang.org/grpc v1.38.0 // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index 8d538a0b4..607936204 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -46,8 +46,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/Masterminds/squirrel v1.5.0 h1:JukIZisrUXadA9pl3rMkjhiamxiB0cXiu+HGp/Y8cY8= -github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/aws/aws-sdk-go v1.35.23 h1:SCP0d0XvyJTDmfnHEQPvBaYi3kea1VNUo7uQmkVgFts= github.com/aws/aws-sdk-go v1.35.23/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= @@ -75,8 +73,8 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/confluentinc/confluent-kafka-go v1.5.2 h1:l+qt+a0Okmq0Bdr1P55IX4fiwFJyg0lZQmfHkAFkv7E= -github.com/confluentinc/confluent-kafka-go v1.5.2/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg= +github.com/confluentinc/confluent-kafka-go v1.7.0 h1:tXh3LWb2Ne0WiU3ng4h5qiGA9XV61rz46w60O+cq8bM= +github.com/confluentinc/confluent-kafka-go v1.7.0/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -93,7 +91,6 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -135,7 +132,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -152,11 +148,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -184,7 +178,6 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -203,7 +196,6 @@ github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye47 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= @@ -219,7 +211,6 @@ github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCM github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.3.0 h1:l8JvKrby3RI7Kg3bYEeU9TA4vqC38QDpFCfcrC7KuN0= github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik= -github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= @@ -254,10 +245,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= @@ -682,8 +669,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.5.2 h1:g0WBLy6fobNUU8W/e9zx6I0Yl79Ya+BDW1NwzAlTiiQ= -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.5.2/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY= +gopkg.in/confluentinc/confluent-kafka-go.v1 v1.7.0 h1:+RlmciBLDd/XwM1iudiG3HtCg45purnsOxEoY/+JZdQ= +gopkg.in/confluentinc/confluent-kafka-go.v1 v1.7.0/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/backend/pkg/db/cache/messages_common.go b/backend/pkg/db/cache/messages-common.go similarity index 66% rename from backend/pkg/db/cache/messages_common.go rename to backend/pkg/db/cache/messages-common.go index 3983982fe..8ca7b2f85 100644 --- a/backend/pkg/db/cache/messages_common.go +++ b/backend/pkg/db/cache/messages-common.go @@ -28,30 +28,6 @@ func (c *PGCache) InsertIssueEvent(sessionID uint64, crash *IssueEvent) error { return c.Conn.InsertIssueEvent(sessionID, session.ProjectID, crash) } -func (c *PGCache) InsertUserID(sessionID uint64, userID *IOSUserID) error { - if err := c.Conn.InsertIOSUserID(sessionID, userID); err != nil { - return err - } - session, err := c.GetSession(sessionID) - if err != nil { - return err - } - session.UserID = &userID.Value - return nil -} - -func (c *PGCache) InsertUserAnonymousID(sessionID uint64, userAnonymousID *IOSUserAnonymousID) error { - if err := c.Conn.InsertIOSUserAnonymousID(sessionID, userAnonymousID); err != nil { - return err - } - session, err := c.GetSession(sessionID) - if err != nil { - return err - } - session.UserAnonymousID = &userAnonymousID.Value - return nil -} - func (c *PGCache) InsertMetadata(sessionID uint64, metadata *Metadata) error { session, err := c.GetSession(sessionID) if err != nil { diff --git a/backend/pkg/db/cache/messages_ios.go b/backend/pkg/db/cache/messages-ios.go similarity index 100% rename from backend/pkg/db/cache/messages_ios.go rename to backend/pkg/db/cache/messages-ios.go diff --git a/backend/pkg/db/cache/messages_web.go b/backend/pkg/db/cache/messages-web.go similarity index 100% rename from backend/pkg/db/cache/messages_web.go rename to backend/pkg/db/cache/messages-web.go diff --git a/backend/pkg/db/cache/pg_cache.go b/backend/pkg/db/cache/pg-cache.go similarity index 100% rename from backend/pkg/db/cache/pg_cache.go rename to backend/pkg/db/cache/pg-cache.go diff --git a/backend/pkg/db/postgres/listener.go b/backend/pkg/db/postgres/listener.go index f90d83485..ef99c2c59 100644 --- a/backend/pkg/db/postgres/listener.go +++ b/backend/pkg/db/postgres/listener.go @@ -11,7 +11,6 @@ import ( type Listener struct { conn *pgx.Conn Integrations chan *Integration - Alerts chan *Alert Errors chan error } @@ -32,23 +31,6 @@ func NewIntegrationsListener(url string) (*Listener, error) { return listener, nil } -func NewAlertsListener(url string) (*Listener, error) { - conn, err := pgx.Connect(context.Background(), url) - if err != nil { - return nil, err - } - listener := &Listener{ - conn: conn, - Errors: make(chan error), - } - listener.Alerts = make(chan *Alert, 50) - if _, err := conn.Exec(context.Background(), "LISTEN alert"); err != nil { - return nil, err - } - go listener.listen() - return listener, nil -} - func (listener *Listener) listen() { for { notification, err := listener.conn.WaitForNotification(context.Background()) @@ -64,13 +46,6 @@ func (listener *Listener) listen() { } else { listener.Integrations <- integrationP } - case "alert": - alertP := new(Alert) - if err := json.Unmarshal([]byte(notification.Payload), alertP); err != nil { - listener.Errors <- fmt.Errorf("%v | Payload: %v", err, notification.Payload) - } else { - listener.Alerts <- alertP - } } } } diff --git a/backend/pkg/db/postgres/messages_common.go b/backend/pkg/db/postgres/messages-common.go similarity index 100% rename from backend/pkg/db/postgres/messages_common.go rename to backend/pkg/db/postgres/messages-common.go diff --git a/backend/pkg/db/postgres/messages_ios.go b/backend/pkg/db/postgres/messages-ios.go similarity index 100% rename from backend/pkg/db/postgres/messages_ios.go rename to backend/pkg/db/postgres/messages-ios.go diff --git a/backend/pkg/db/postgres/messages_web_stats.go b/backend/pkg/db/postgres/messages-web-stats.go similarity index 100% rename from backend/pkg/db/postgres/messages_web_stats.go rename to backend/pkg/db/postgres/messages-web-stats.go diff --git a/backend/pkg/db/postgres/messages_web.go b/backend/pkg/db/postgres/messages-web.go similarity index 100% rename from backend/pkg/db/postgres/messages_web.go rename to backend/pkg/db/postgres/messages-web.go diff --git a/backend/pkg/db/postgres/unstarted_session.go b/backend/pkg/db/postgres/unstarted-session.go similarity index 100% rename from backend/pkg/db/postgres/unstarted_session.go rename to backend/pkg/db/postgres/unstarted-session.go diff --git a/backend/pkg/env/worker_id.go b/backend/pkg/env/worker-id.go similarity index 100% rename from backend/pkg/env/worker_id.go rename to backend/pkg/env/worker-id.go diff --git a/backend/pkg/messages/get_timestamp.go b/backend/pkg/messages/get-timestamp.go similarity index 100% rename from backend/pkg/messages/get_timestamp.go rename to backend/pkg/messages/get-timestamp.go diff --git a/backend/pkg/messages/legacy_message_transform.go b/backend/pkg/messages/legacy-message-transform.go similarity index 100% rename from backend/pkg/messages/legacy_message_transform.go rename to backend/pkg/messages/legacy-message-transform.go diff --git a/backend/pkg/messages/read_message.go b/backend/pkg/messages/read-message.go similarity index 100% rename from backend/pkg/messages/read_message.go rename to backend/pkg/messages/read-message.go diff --git a/backend/services/http/handlers_depricated.go b/backend/services/http/handlers-depricated.go similarity index 100% rename from backend/services/http/handlers_depricated.go rename to backend/services/http/handlers-depricated.go diff --git a/backend/services/http/handlers_ios.go b/backend/services/http/handlers-ios.go similarity index 71% rename from backend/services/http/handlers_ios.go rename to backend/services/http/handlers-ios.go index 6c3f945bd..affcab59d 100644 --- a/backend/services/http/handlers_ios.go +++ b/backend/services/http/handlers-ios.go @@ -2,50 +2,50 @@ package main import ( "encoding/json" - "net/http" "errors" - "time" - "math/rand" - "strconv" "log" + "math/rand" + "net/http" + "strconv" + "time" "openreplay/backend/pkg/db/postgres" - "openreplay/backend/pkg/token" . "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/token" ) -const FILES_SIZE_LIMIT int64 = 1e7 // 10Mb +const FILES_SIZE_LIMIT int64 = 1e7 // 10Mb func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { type request struct { - Token string `json:"token"` - ProjectKey *string `json:"projectKey"` - TrackerVersion string `json:"trackerVersion"` - RevID string `json:"revID"` - UserUUID *string `json:"userUUID"` + Token string `json:"token"` + ProjectKey *string `json:"projectKey"` + TrackerVersion string `json:"trackerVersion"` + RevID string `json:"revID"` + UserUUID *string `json:"userUUID"` //UserOS string `json"userOS"` //hardcoded 'MacOS' - UserOSVersion string `json:"userOSVersion"` - UserDevice string `json:"userDevice"` - Timestamp uint64 `json:"timestamp"` + UserOSVersion string `json:"userOSVersion"` + UserDevice string `json:"userDevice"` + Timestamp uint64 `json:"timestamp"` // UserDeviceType uint 0:phone 1:pad 2:tv 3:carPlay 5:mac // “performances”:{ - // “activeProcessorCount”:8, - // “isLowPowerModeEnabled”:0, - // “orientation”:0, - // “systemUptime”:585430, - // “batteryState”:0, - // “thermalState”:0, - // “batteryLevel”:0, - // “processorCount”:8, - // “physicalMemory”:17179869184 - // }, + // “activeProcessorCount”:8, + // “isLowPowerModeEnabled”:0, + // “orientation”:0, + // “systemUptime”:585430, + // “batteryState”:0, + // “thermalState”:0, + // “batteryLevel”:0, + // “processorCount”:8, + // “physicalMemory”:17179869184 + // }, } type response struct { - Token string `json:"token"` - ImagesHashList []string `json:"imagesHashList"` - UserUUID string `json:"userUUID"` - BeaconSizeLimit int64 `json:"beaconSizeLimit"` - SessionID string `json:"sessionID"` + Token string `json:"token"` + ImagesHashList []string `json:"imagesHashList"` + UserUUID string `json:"userUUID"` + BeaconSizeLimit int64 `json:"beaconSizeLimit"` + SessionID string `json:"sessionID"` } startTime := time.Now() req := &request{} @@ -98,16 +98,16 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { // The difference with web is mostly here: producer.Produce(TOPIC_RAW_IOS, tokenData.ID, Encode(&IOSSessionStart{ - Timestamp: req.Timestamp, - ProjectID: uint64(p.ProjectID), - TrackerVersion: req.TrackerVersion, - RevID: req.RevID, - UserUUID: userUUID, - UserOS: "IOS", - UserOSVersion: req.UserOSVersion, - UserDevice: MapIOSDevice(req.UserDevice), - UserDeviceType: GetIOSDeviceType(req.UserDevice), - UserCountry: country, + Timestamp: req.Timestamp, + ProjectID: uint64(p.ProjectID), + TrackerVersion: req.TrackerVersion, + RevID: req.RevID, + UserUUID: userUUID, + UserOS: "IOS", + UserOSVersion: req.UserOSVersion, + UserDevice: MapIOSDevice(req.UserDevice), + UserDeviceType: GetIOSDeviceType(req.UserDevice), + UserCountry: country, })) } @@ -119,14 +119,13 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { responseWithJSON(w, &response{ // ImagesHashList: imagesHashList, - Token: tokenizer.Compose(*tokenData), - UserUUID: userUUID, - SessionID: strconv.FormatUint(tokenData.ID, 10), + Token: tokenizer.Compose(*tokenData), + UserUUID: userUUID, + SessionID: strconv.FormatUint(tokenData.ID, 10), BeaconSizeLimit: BEACON_SIZE_LIMIT, }) } - func pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { sessionData, err := tokenizer.ParseFromHTTPRequest(r) if err != nil { @@ -136,8 +135,6 @@ func pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { pushMessages(w, r, sessionData.ID, TOPIC_RAW_IOS) } - - func pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { sessionData, err := tokenizer.ParseFromHTTPRequest(r) if err != nil && err != token.EXPIRED { @@ -145,10 +142,9 @@ func pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { return } // Check timestamps here? - pushMessages(w, r, sessionData.ID,TOPIC_RAW_IOS) + pushMessages(w, r, sessionData.ID, TOPIC_RAW_IOS) } - func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { log.Printf("recieved imagerequest") @@ -163,12 +159,12 @@ func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { err = r.ParseMultipartForm(1e6) // ~1Mb if err == http.ErrNotMultipart || err == http.ErrMissingBoundary { responseWithError(w, http.StatusUnsupportedMediaType, err) - // } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB + // } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB } else if err != nil { responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging } - if (r.MultipartForm == nil) { + if r.MultipartForm == nil { responseWithError(w, http.StatusInternalServerError, errors.New("Multipart not parsed")) } @@ -177,7 +173,7 @@ func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { return } - prefix := r.MultipartForm.Value["projectKey"][0] + "/" + strconv.FormatUint(sessionData.ID, 10) + "/" + prefix := r.MultipartForm.Value["projectKey"][0] + "/" + strconv.FormatUint(sessionData.ID, 10) + "/" for _, fileHeaderList := range r.MultipartForm.File { for _, fileHeader := range fileHeaderList { @@ -187,7 +183,7 @@ func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { } key := prefix + fileHeader.Filename log.Printf("Uploading image... %v", key) - go func() { //TODO: mime type from header + go func() { //TODO: mime type from header if err := s3.Upload(file, key, "image/jpeg", false); err != nil { log.Printf("Upload ios screen error. %v", err) } diff --git a/backend/services/http/handlers_web.go b/backend/services/http/handlers-web.go similarity index 100% rename from backend/services/http/handlers_web.go rename to backend/services/http/handlers-web.go diff --git a/backend/services/http/ios-device.go b/backend/services/http/ios-device.go new file mode 100644 index 000000000..bec1f3b36 --- /dev/null +++ b/backend/services/http/ios-device.go @@ -0,0 +1,138 @@ +package main + +import ( + "strings" +) + +func MapIOSDevice(identifier string) string { + switch identifier { + case "iPod5,1": + return "iPod touch (5th generation)" + case "iPod7,1": + return "iPod touch (6th generation)" + case "iPod9,1": + return "iPod touch (7th generation)" + case "iPhone3,1", "iPhone3,2", "iPhone3,3": + return "iPhone 4" + case "iPhone4,1": + return "iPhone 4s" + case "iPhone5,1", "iPhone5,2": + return "iPhone 5" + case "iPhone5,3", "iPhone5,4": + return "iPhone 5c" + case "iPhone6,1", "iPhone6,2": + return "iPhone 5s" + case "iPhone7,2": + return "iPhone 6" + case "iPhone7,1": + return "iPhone 6 Plus" + case "iPhone8,1": + return "iPhone 6s" + case "iPhone8,2": + return "iPhone 6s Plus" + case "iPhone8,4": + return "iPhone SE" + case "iPhone9,1", "iPhone9,3": + return "iPhone 7" + case "iPhone9,2", "iPhone9,4": + return "iPhone 7 Plus" + case "iPhone10,1", "iPhone10,4": + return "iPhone 8" + case "iPhone10,2", "iPhone10,5": + return "iPhone 8 Plus" + case "iPhone10,3", "iPhone10,6": + return "iPhone X" + case "iPhone11,2": + return "iPhone XS" + case "iPhone11,4", "iPhone11,6": + return "iPhone XS Max" + case "iPhone11,8": + return "iPhone XR" + case "iPhone12,1": + return "iPhone 11" + case "iPhone12,3": + return "iPhone 11 Pro" + case "iPhone12,5": + return "iPhone 11 Pro Max" + case "iPhone12,8": + return "iPhone SE (2nd generation)" + case "iPhone13,1": + return "iPhone 12 mini" + case "iPhone13,2": + return "iPhone 12" + case "iPhone13,3": + return "iPhone 12 Pro" + case "iPhone13,4": + return "iPhone 12 Pro Max" + case "iPad2,1", "iPad2,2", "iPad2,3", "iPad2,4": + return "iPad 2" + case "iPad3,1", "iPad3,2", "iPad3,3": + return "iPad (3rd generation)" + case "iPad3,4", "iPad3,5", "iPad3,6": + return "iPad (4th generation)" + case "iPad6,11", "iPad6,12": + return "iPad (5th generation)" + case "iPad7,5", "iPad7,6": + return "iPad (6th generation)" + case "iPad7,11", "iPad7,12": + return "iPad (7th generation)" + case "iPad11,6", "iPad11,7": + return "iPad (8th generation)" + case "iPad4,1", "iPad4,2", "iPad4,3": + return "iPad Air" + case "iPad5,3", "iPad5,4": + return "iPad Air 2" + case "iPad11,3", "iPad11,4": + return "iPad Air (3rd generation)" + case "iPad13,1", "iPad13,2": + return "iPad Air (4th generation)" + case "iPad2,5", "iPad2,6", "iPad2,7": + return "iPad mini" + case "iPad4,4", "iPad4,5", "iPad4,6": + return "iPad mini 2" + case "iPad4,7", "iPad4,8", "iPad4,9": + return "iPad mini 3" + case "iPad5,1", "iPad5,2": + return "iPad mini 4" + case "iPad11,1", "iPad11,2": + return "iPad mini (5th generation)" + case "iPad6,3", "iPad6,4": + return "iPad Pro (9.7-inch)" + case "iPad7,3", "iPad7,4": + return "iPad Pro (10.5-inch)" + case "iPad8,1", "iPad8,2", "iPad8,3", "iPad8,4": + return "iPad Pro (11-inch) (1st generation)" + case "iPad8,9", "iPad8,10": + return "iPad Pro (11-inch) (2nd generation)" + case "iPad6,7", "iPad6,8": + return "iPad Pro (12.9-inch) (1st generation)" + case "iPad7,1", "iPad7,2": + return "iPad Pro (12.9-inch) (2nd generation)" + case "iPad8,5", "iPad8,6", "iPad8,7", "iPad8,8": + return "iPad Pro (12.9-inch) (3rd generation)" + case "iPad8,11", "iPad8,12": + return "iPad Pro (12.9-inch) (4th generation)" + case "AppleTV5,3": + return "Apple TV" + case "AppleTV6,2": + return "Apple TV 4K" + case "AudioAccessory1,1": + return "HomePod" + case "AudioAccessory5,1": + return "HomePod mini" + case "i386", "x86_64": + return "Simulator" + default: + return identifier + } +} + +func GetIOSDeviceType(identifier string) string { + if strings.Contains(identifier, "iPhone") { + return "mobile" //"phone" + } + if strings.Contains(identifier, "iPad") { + return "tablet" + } + return "other" +} diff --git a/backend/services/http/ios_device.go b/backend/services/http/ios_device.go deleted file mode 100644 index 2c3474157..000000000 --- a/backend/services/http/ios_device.go +++ /dev/null @@ -1,79 +0,0 @@ -package main - -import ( - "strings" -) - -func MapIOSDevice(identifier string) string { - switch identifier { - case "iPod5,1": return "iPod touch (5th generation)" - case "iPod7,1": return "iPod touch (6th generation)" - case "iPod9,1": return "iPod touch (7th generation)" - case "iPhone3,1", "iPhone3,2", "iPhone3,3": return "iPhone 4" - case "iPhone4,1": return "iPhone 4s" - case "iPhone5,1", "iPhone5,2": return "iPhone 5" - case "iPhone5,3", "iPhone5,4": return "iPhone 5c" - case "iPhone6,1", "iPhone6,2": return "iPhone 5s" - case "iPhone7,2": return "iPhone 6" - case "iPhone7,1": return "iPhone 6 Plus" - case "iPhone8,1": return "iPhone 6s" - case "iPhone8,2": return "iPhone 6s Plus" - case "iPhone8,4": return "iPhone SE" - case "iPhone9,1", "iPhone9,3": return "iPhone 7" - case "iPhone9,2", "iPhone9,4": return "iPhone 7 Plus" - case "iPhone10,1", "iPhone10,4": return "iPhone 8" - case "iPhone10,2", "iPhone10,5": return "iPhone 8 Plus" - case "iPhone10,3", "iPhone10,6": return "iPhone X" - case "iPhone11,2": return "iPhone XS" - case "iPhone11,4", "iPhone11,6": return "iPhone XS Max" - case "iPhone11,8": return "iPhone XR" - case "iPhone12,1": return "iPhone 11" - case "iPhone12,3": return "iPhone 11 Pro" - case "iPhone12,5": return "iPhone 11 Pro Max" - case "iPhone12,8": return "iPhone SE (2nd generation)" - case "iPhone13,1": return "iPhone 12 mini" - case "iPhone13,2": return "iPhone 12" - case "iPhone13,3": return "iPhone 12 Pro" - case "iPhone13,4": return "iPhone 12 Pro Max" - case "iPad2,1", "iPad2,2", "iPad2,3", "iPad2,4":return "iPad 2" - case "iPad3,1", "iPad3,2", "iPad3,3": return "iPad (3rd generation)" - case "iPad3,4", "iPad3,5", "iPad3,6": return "iPad (4th generation)" - case "iPad6,11", "iPad6,12": return "iPad (5th generation)" - case "iPad7,5", "iPad7,6": return "iPad (6th generation)" - case "iPad7,11", "iPad7,12": return "iPad (7th generation)" - case "iPad11,6", "iPad11,7": return "iPad (8th generation)" - case "iPad4,1", "iPad4,2", "iPad4,3": return "iPad Air" - case "iPad5,3", "iPad5,4": return "iPad Air 2" - case "iPad11,3", "iPad11,4": return "iPad Air (3rd generation)" - case "iPad13,1", "iPad13,2": return "iPad Air (4th generation)" - case "iPad2,5", "iPad2,6", "iPad2,7": return "iPad mini" - case "iPad4,4", "iPad4,5", "iPad4,6": return "iPad mini 2" - case "iPad4,7", "iPad4,8", "iPad4,9": return "iPad mini 3" - case "iPad5,1", "iPad5,2": return "iPad mini 4" - case "iPad11,1", "iPad11,2": return "iPad mini (5th generation)" - case "iPad6,3", "iPad6,4": return "iPad Pro (9.7-inch)" - case "iPad7,3", "iPad7,4": return "iPad Pro (10.5-inch)" - case "iPad8,1", "iPad8,2", "iPad8,3", "iPad8,4":return "iPad Pro (11-inch) (1st generation)" - case "iPad8,9", "iPad8,10": return "iPad Pro (11-inch) (2nd generation)" - case "iPad6,7", "iPad6,8": return "iPad Pro (12.9-inch) (1st generation)" - case "iPad7,1", "iPad7,2": return "iPad Pro (12.9-inch) (2nd generation)" - case "iPad8,5", "iPad8,6", "iPad8,7", "iPad8,8":return "iPad Pro (12.9-inch) (3rd generation)" - case "iPad8,11", "iPad8,12": return "iPad Pro (12.9-inch) (4th generation)" - case "AppleTV5,3": return "Apple TV" - case "AppleTV6,2": return "Apple TV 4K" - case "AudioAccessory1,1": return "HomePod" - case "AudioAccessory5,1": return "HomePod mini" - case "i386", "x86_64": return "Simulator" - default: return identifier - } -} - -func GetIOSDeviceType(identifier string) string { - if strings.Contains(identifier, "iPhone") { - return "mobile" //"phone" - } - if strings.Contains(identifier, "iPad") { - return "tablet" - } - return "other" -} \ No newline at end of file diff --git a/backend/services/http/main.go b/backend/services/http/main.go index 8ed8b6d95..f0ae22d32 100644 --- a/backend/services/http/main.go +++ b/backend/services/http/main.go @@ -10,19 +10,17 @@ import ( "golang.org/x/net/http2" - + "openreplay/backend/pkg/db/cache" + "openreplay/backend/pkg/db/postgres" "openreplay/backend/pkg/env" "openreplay/backend/pkg/flakeid" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/queue/types" "openreplay/backend/pkg/storage" - "openreplay/backend/pkg/db/postgres" - "openreplay/backend/pkg/db/cache" - "openreplay/backend/pkg/url/assets" "openreplay/backend/pkg/token" + "openreplay/backend/pkg/url/assets" "openreplay/backend/services/http/geoip" "openreplay/backend/services/http/uaparser" - ) var rewriter *assets.Rewriter @@ -38,6 +36,7 @@ var TOPIC_RAW_WEB string var TOPIC_RAW_IOS string var TOPIC_CACHE string var TOPIC_TRIGGER string + //var TOPIC_ANALYTICS string var CACHE_ASSESTS bool var BEACON_SIZE_LIMIT int64 @@ -53,7 +52,7 @@ func main() { TOPIC_TRIGGER = env.String("TOPIC_TRIGGER") //TOPIC_ANALYTICS = env.String("TOPIC_ANALYTICS") rewriter = assets.NewRewriter(env.String("ASSETS_ORIGIN")) - pgconn = cache.NewPGCache(postgres.NewConn(env.String("POSTGRES_STRING")), 1000 * 60 * 20) + pgconn = cache.NewPGCache(postgres.NewConn(env.String("POSTGRES_STRING")), 1000*60*20) defer pgconn.Close() s3 = storage.NewS3(env.String("AWS_REGION"), env.String("S3_BUCKET_IOS_IMAGES")) tokenizer = token.NewTokenizer(env.String("TOKEN_SECRET")) @@ -70,7 +69,7 @@ func main() { Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // TODO: agree with specification - w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Methods", "POST") w.Header().Set("Access-Control-Allow-Headers", "Content-Type,Authorization") if r.Method == http.MethodOptions { @@ -79,13 +78,12 @@ func main() { return } - log.Printf("Request: %v - %v ", r.Method, r.URL.Path) - + log.Printf("Request: %v - %v ", r.Method, r.URL.Path) switch r.URL.Path { case "/": w.WriteHeader(http.StatusOK) - case "/v1/web/not-started": + case "/v1/web/not-started": switch r.Method { case http.MethodPost: notStartedHandlerWeb(w, r) diff --git a/backend/services/http/project_id.go b/backend/services/http/project_id.go deleted file mode 100644 index 059576fe8..000000000 --- a/backend/services/http/project_id.go +++ /dev/null @@ -1,12 +0,0 @@ -package main - -func decodeProjectID(projectID uint64) uint64 { - if projectID < 0x10000000000000 || projectID >= 0x20000000000000 { - return 0 - } - projectID = (projectID - 0x10000000000000) * 4212451012670231 & 0xfffffffffffff - if projectID > 0xffffffff { - return 0 - } - return projectID -} From 6196e79d000f69a50b0b5e2e80d7298922b8e2a0 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Fri, 1 Apr 2022 19:16:24 +0200 Subject: [PATCH 023/125] feat(backend): everyday update of an assets' version --- backend/pkg/url/assets/url.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/pkg/url/assets/url.go b/backend/pkg/url/assets/url.go index 1fe717531..b55921149 100644 --- a/backend/pkg/url/assets/url.go +++ b/backend/pkg/url/assets/url.go @@ -5,11 +5,18 @@ import ( "path/filepath" "strconv" "strings" + "time" + + "openreplay/backend/pkg/flakeid" ) func getSessionKey(sessionID uint64) string { - // Based on timestamp, changes once per week. Check pkg/flakeid for understanding sessionID - return strconv.FormatUint(sessionID>>50, 10) + return strconv.FormatUint( + uint64(time.UnixMilli( + int64(flakeid.ExtractTimestamp(sessionID)), + ).Weekday()), + 10, + ) } func ResolveURL(baseurl string, rawurl string) string { From c8872064ec3c35f455c4d435a3a9e3826e19d34d Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Fri, 1 Apr 2022 19:43:46 +0200 Subject: [PATCH 024/125] code(backend): use 1.18 time features --- backend/pkg/token/tokenizer.go | 12 +- backend/pkg/utime/utime.go | 11 -- backend/services/ender/builder/builder.go | 4 +- backend/services/ender/main.go | 2 +- backend/services/http/handlers-ios.go | 4 +- backend/services/http/handlers-web.go | 8 +- .../integrations/integration/bugsnag.go | 32 +++--- .../integrations/integration/client.go | 37 +++--- .../integrations/integration/datadog.go | 43 ++++--- .../integrations/integration/elasticsearch.go | 3 +- .../integrations/integration/newrelic.go | 32 +++--- .../integrations/integration/sentry.go | 49 ++++---- .../integrations/integration/stackdriver.go | 105 +++++++++--------- .../integrations/integration/sumologic.go | 53 +++++---- ee/backend/pkg/kafka/consumer.go | 2 +- 15 files changed, 184 insertions(+), 213 deletions(-) delete mode 100644 backend/pkg/utime/utime.go diff --git a/backend/pkg/token/tokenizer.go b/backend/pkg/token/tokenizer.go index 3f1069a63..f61e1f145 100644 --- a/backend/pkg/token/tokenizer.go +++ b/backend/pkg/token/tokenizer.go @@ -22,8 +22,8 @@ func NewTokenizer(secret string) *Tokenizer { } type TokenData struct { - ID uint64 - ExpTime int64 + ID uint64 + ExpTime int64 } func (tokenizer *Tokenizer) sign(body string) []byte { @@ -33,7 +33,7 @@ func (tokenizer *Tokenizer) sign(body string) []byte { } func (tokenizer *Tokenizer) Compose(d TokenData) string { - body := strconv.FormatUint(d.ID, 36) + + body := strconv.FormatUint(d.ID, 36) + "." + strconv.FormatInt(d.ExpTime, 36) sign := base58.Encode(tokenizer.sign(body)) return body + "." + sign @@ -58,8 +58,8 @@ func (tokenizer *Tokenizer) Parse(token string) (*TokenData, error) { if err != nil { return nil, err } - if expTime <= time.Now().UnixNano()/1e6 { - return &TokenData{id,expTime}, EXPIRED + if expTime <= time.Now().UnixMilli() { + return &TokenData{id, expTime}, EXPIRED } - return &TokenData{id,expTime}, nil + return &TokenData{id, expTime}, nil } diff --git a/backend/pkg/utime/utime.go b/backend/pkg/utime/utime.go deleted file mode 100644 index e3b5a2751..000000000 --- a/backend/pkg/utime/utime.go +++ /dev/null @@ -1,11 +0,0 @@ -package utime - -import "time" - -func CurrentTimestamp() int64 { - return time.Now().UnixNano() / 1e6 -} - -func ToMilliseconds(t time.Time) int64 { - return t.UnixNano() / 1e6 -} diff --git a/backend/services/ender/builder/builder.go b/backend/services/ender/builder/builder.go index e36bdcbe3..9c2067985 100644 --- a/backend/services/ender/builder/builder.go +++ b/backend/services/ender/builder/builder.go @@ -110,11 +110,11 @@ func (b *builder) buildInputEvent() { func (b *builder) handleMessage(message Message, messageID uint64) { timestamp := GetTimestamp(message) - if b.timestamp <= timestamp { // unnecessary? TODO: test and remove + if b.timestamp < timestamp { // unnecessary? TODO: test and remove b.timestamp = timestamp } - b.lastProcessedTimestamp = time.Now().UnixNano() / 1e6 + b.lastProcessedTimestamp = time.Now().UnixMilli() // Might happen before the first timestamp. switch msg := message.(type) { diff --git a/backend/services/ender/main.go b/backend/services/ender/main.go index f0f139dce..f2430f3a0 100644 --- a/backend/services/ender/main.go +++ b/backend/services/ender/main.go @@ -56,7 +56,7 @@ func main() { consumer.Close() os.Exit(0) case <-tick: - builderMap.IterateReadyMessages(time.Now().UnixNano()/1e6, func(sessionID uint64, readyMsg messages.Message) { + builderMap.IterateReadyMessages(time.Now().UnixMilli(), func(sessionID uint64, readyMsg messages.Message) { producer.Produce(TOPIC_TRIGGER, sessionID, messages.Encode(readyMsg)) }) // TODO: why exactly do we need Flush here and not in any other place? diff --git a/backend/services/http/handlers-ios.go b/backend/services/http/handlers-ios.go index affcab59d..f15a6af60 100644 --- a/backend/services/http/handlers-ios.go +++ b/backend/services/http/handlers-ios.go @@ -85,14 +85,14 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { responseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } - sessionID, err := flaker.Compose(uint64(startTime.UnixNano() / 1e6)) + sessionID, err := flaker.Compose(uint64(startTime.UnixMilli())) if err != nil { responseWithError(w, http.StatusInternalServerError, err) return } // TODO: if EXPIRED => send message for two sessions association expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) - tokenData = &token.TokenData{sessionID, expTime.UnixNano() / 1e6} + tokenData = &token.TokenData{sessionID, expTime.UnixMilli()} country := geoIP.ExtractISOCodeFromHTTPRequest(r) diff --git a/backend/services/http/handlers-web.go b/backend/services/http/handlers-web.go index 6020c3eb1..29dcf161d 100644 --- a/backend/services/http/handlers-web.go +++ b/backend/services/http/handlers-web.go @@ -76,14 +76,14 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { responseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } - sessionID, err := flaker.Compose(uint64(startTime.UnixNano() / 1e6)) + sessionID, err := flaker.Compose(uint64(startTime.UnixMilli())) if err != nil { responseWithError(w, http.StatusInternalServerError, err) return } // TODO: if EXPIRED => send message for two sessions association expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) - tokenData = &token.TokenData{sessionID, expTime.UnixNano() / 1e6} + tokenData = &token.TokenData{sessionID, expTime.UnixMilli()} country := geoIP.ExtractISOCodeFromHTTPRequest(r) producer.Produce(TOPIC_RAW_WEB, tokenData.ID, Encode(&SessionStart{ @@ -108,8 +108,8 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { //delayDuration := time.Now().Sub(startTime) responseWithJSON(w, &response{ - //Timestamp: startTime.UnixNano() / 1e6, - //Delay: delayDuration.Nanoseconds() / 1e6, + //Timestamp: startTime.UnixMilli(), + //Delay: delayDuration.Milliseconds(), Token: tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), diff --git a/backend/services/integrations/integration/bugsnag.go b/backend/services/integrations/integration/bugsnag.go index 7c31db3cb..118cdb84d 100644 --- a/backend/services/integrations/integration/bugsnag.go +++ b/backend/services/integrations/integration/bugsnag.go @@ -1,15 +1,14 @@ package integration import ( + "encoding/json" "fmt" + "io" + "io/ioutil" "net/http" - "encoding/json" "net/url" "time" - "io" - "io/ioutil" - "openreplay/backend/pkg/utime" "openreplay/backend/pkg/messages" ) @@ -18,15 +17,14 @@ import ( */ type bugsnag struct { - BugsnagProjectId string // `json:"bugsnag_project_id"` + BugsnagProjectId string // `json:"bugsnag_project_id"` AuthorizationToken string // `json:"auth_token"` } - type bugsnagEvent struct { MetaData struct { SpecialInfo struct { - AsayerSessionId uint64 `json:"asayerSessionId,string"` + AsayerSessionId uint64 `json:"asayerSessionId,string"` OpenReplaySessionToken string `json:"openReplaySessionToken"` } `json:"special_info"` } `json:"metaData"` @@ -38,7 +36,7 @@ type bugsnagEvent struct { func (b *bugsnag) Request(c *client) error { sinceTs := c.getLastMessageTimestamp() + 1000 // From next second - sinceFormatted := time.Unix(0, int64(sinceTs*1e6)).Format(time.RFC3339) + sinceFormatted := time.UnixMilli(int64(sinceTs)).Format(time.RFC3339) requestURL := fmt.Sprintf("https://api.bugsnag.com/projects/%v/events", b.BugsnagProjectId) req, err := http.NewRequest("GET", requestURL, nil) if err != nil { @@ -47,10 +45,10 @@ func (b *bugsnag) Request(c *client) error { q := req.URL.Query() // q.Add("per_page", "100") // Up to a maximum of 30. Default: 30 // q.Add("sort", "timestamp") // Default: timestamp (timestamp == ReceivedAt ??) - q.Add("direction", "asc") // Default: desc + q.Add("direction", "asc") // Default: desc q.Add("full_reports", "true") // Default: false - q.Add("filters[event.since][][type]", "eq") - q.Add("filters[event.since][][value]", sinceFormatted) // seems like inclusively + q.Add("filters[event.since][][type]", "eq") + q.Add("filters[event.since][][value]", sinceFormatted) // seems like inclusively req.URL.RawQuery = q.Encode() authToken := "token " + b.AuthorizationToken @@ -85,7 +83,7 @@ func (b *bugsnag) Request(c *client) error { } sessionID := e.MetaData.SpecialInfo.AsayerSessionId token := e.MetaData.SpecialInfo.OpenReplaySessionToken - if sessionID == 0 && token == "" { + if sessionID == 0 && token == "" { // c.errChan <- "No AsayerSessionId found. | Message: %v", e continue } @@ -94,16 +92,16 @@ func (b *bugsnag) Request(c *client) error { c.errChan <- err continue } - timestamp := uint64(utime.ToMilliseconds(parsedTime)) + timestamp := uint64(parsedTime.UnixMilli()) c.setLastMessageTimestamp(timestamp) c.evChan <- &SessionErrorEvent{ SessionID: sessionID, - Token: token, + Token: token, RawErrorEvent: &messages.RawErrorEvent{ - Source: "bugsnag", + Source: "bugsnag", Timestamp: timestamp, - Name: e.Exceptions[0].Message, - Payload: string(jsonEvent), + Name: e.Exceptions[0].Message, + Payload: string(jsonEvent), }, } } diff --git a/backend/services/integrations/integration/client.go b/backend/services/integrations/integration/client.go index 2abf9913d..315bfe4e9 100644 --- a/backend/services/integrations/integration/client.go +++ b/backend/services/integrations/integration/client.go @@ -5,10 +5,10 @@ import ( "fmt" "log" "sync" + "time" "openreplay/backend/pkg/db/postgres" "openreplay/backend/pkg/messages" - "openreplay/backend/pkg/utime" ) const MAX_ATTEMPTS_IN_A_ROW = 4 @@ -20,10 +20,10 @@ type requester interface { } type requestData struct { - LastMessageTimestamp uint64 // `json:"lastMessageTimestamp, string"` - LastMessageId string + LastMessageTimestamp uint64 // `json:"lastMessageTimestamp, string"` + LastMessageId string UnsuccessfullAttemptsCount int - LastAttemptTimestamp int64 + LastAttemptTimestamp int64 } type client struct { @@ -31,19 +31,19 @@ type client struct { requester integration *postgres.Integration // TODO: timeout ? - mux sync.Mutex + mux sync.Mutex updateChan chan<- postgres.Integration - evChan chan<- *SessionErrorEvent - errChan chan<- error + evChan chan<- *SessionErrorEvent + errChan chan<- error } type SessionErrorEvent struct { SessionID uint64 - Token string + Token string *messages.RawErrorEvent } -type ClientMap map[ string ]*client +type ClientMap map[string]*client func NewClient(i *postgres.Integration, updateChan chan<- postgres.Integration, evChan chan<- *SessionErrorEvent, errChan chan<- error) (*client, error) { c := new(client) @@ -60,15 +60,14 @@ func NewClient(i *postgres.Integration, updateChan chan<- postgres.Integration, // TODO: RequestData manager if c.requestData.LastMessageTimestamp == 0 { // ? - c.requestData.LastMessageTimestamp = uint64(utime.CurrentTimestamp() - 24*60*60*1000) + c.requestData.LastMessageTimestamp = uint64(time.Now().Add(-time.Hour * 24).UnixMilli()) } return c, nil } - // from outside -func (c* client) Update(i *postgres.Integration) error { +func (c *client) Update(i *postgres.Integration) error { c.mux.Lock() defer c.mux.Unlock() var r requester @@ -111,8 +110,8 @@ func (c *client) getLastMessageTimestamp() uint64 { } func (c *client) setLastMessageId(timestamp uint64, id string) { //if timestamp >= c.requestData.LastMessageTimestamp { - c.requestData.LastMessageId = id - c.requestData.LastMessageTimestamp = timestamp + c.requestData.LastMessageId = id + c.requestData.LastMessageTimestamp = timestamp //} } func (c *client) getLastMessageId() string { @@ -128,18 +127,18 @@ func (c *client) Request() { c.mux.Lock() defer c.mux.Unlock() if c.requestData.UnsuccessfullAttemptsCount >= MAX_ATTEMPTS || - (c.requestData.UnsuccessfullAttemptsCount >= MAX_ATTEMPTS_IN_A_ROW && - utime.CurrentTimestamp() - c.requestData.LastAttemptTimestamp < ATTEMPTS_INTERVAL) { + (c.requestData.UnsuccessfullAttemptsCount >= MAX_ATTEMPTS_IN_A_ROW && + time.Now().UnixMilli()-c.requestData.LastAttemptTimestamp < ATTEMPTS_INTERVAL) { return } - c.requestData.LastAttemptTimestamp = utime.CurrentTimestamp() + c.requestData.LastAttemptTimestamp = time.Now().UnixMilli() err := c.requester.Request(c) if err != nil { log.Println("ERRROR L139") log.Println(err) c.handleError(err) - c.requestData.UnsuccessfullAttemptsCount++; + c.requestData.UnsuccessfullAttemptsCount++ } else { c.requestData.UnsuccessfullAttemptsCount = 0 } @@ -152,5 +151,3 @@ func (c *client) Request() { c.integration.RequestData = rd c.updateChan <- *c.integration } - - diff --git a/backend/services/integrations/integration/datadog.go b/backend/services/integrations/integration/datadog.go index eb7b5daee..096c3b822 100644 --- a/backend/services/integrations/integration/datadog.go +++ b/backend/services/integrations/integration/datadog.go @@ -1,38 +1,37 @@ package integration import ( - "fmt" - "net/http" - "encoding/json" "bytes" - "time" + "encoding/json" + "fmt" "io" - "io/ioutil" + "io/ioutil" + "net/http" + "time" - "openreplay/backend/pkg/utime" "openreplay/backend/pkg/messages" ) -/* +/* We collect Logs. Datadog also has Events */ type datadog struct { - ApplicationKey string //`json:"application_key"` - ApiKey string //`json:"api_key"` + ApplicationKey string //`json:"application_key"` + ApiKey string //`json:"api_key"` } type datadogResponce struct { - Logs []json.RawMessage + Logs []json.RawMessage NextLogId *string - Status string + Status string } type datadogLog struct { Content struct { - Timestamp string - Message string + Timestamp string + Message string Attributes struct { Error struct { // Not sure about this Message string @@ -48,10 +47,10 @@ func (d *datadog) makeRequest(nextLogId *string, fromTs uint64, toTs uint64) (*h d.ApplicationKey, ) startAt := "null" - if nextLogId != nil && *nextLogId != "" { + if nextLogId != nil && *nextLogId != "" { startAt = *nextLogId } - // Query: status:error/info/warning? + // Query: status:error/info/warning? // openReplaySessionToken instead of asayer_session_id jsonBody := fmt.Sprintf(`{ "limit": 1000, @@ -72,8 +71,8 @@ func (d *datadog) makeRequest(nextLogId *string, fromTs uint64, toTs uint64) (*h } func (d *datadog) Request(c *client) error { - fromTs := c.getLastMessageTimestamp() + 1 // From next millisecond - toTs := uint64(utime.CurrentTimestamp()) + fromTs := c.getLastMessageTimestamp() + 1 // From next millisecond + toTs := uint64(time.Now().UnixMilli()) var nextLogId *string for { req, err := d.makeRequest(nextLogId, fromTs, toTs) @@ -111,16 +110,16 @@ func (d *datadog) Request(c *client) error { c.errChan <- err continue } - timestamp := uint64(utime.ToMilliseconds(parsedTime)) + timestamp := uint64(parsedTime.UnixMilli()) c.setLastMessageTimestamp(timestamp) c.evChan <- &SessionErrorEvent{ //SessionID: sessionID, Token: token, RawErrorEvent: &messages.RawErrorEvent{ - Source: "datadog", + Source: "datadog", Timestamp: timestamp, - Name: ddLog.Content.Attributes.Error.Message, - Payload: string(jsonLog), + Name: ddLog.Content.Attributes.Error.Message, + Payload: string(jsonLog), }, } } @@ -129,4 +128,4 @@ func (d *datadog) Request(c *client) error { return nil } } -} \ No newline at end of file +} diff --git a/backend/services/integrations/integration/elasticsearch.go b/backend/services/integrations/integration/elasticsearch.go index 14480e0b8..dd6f5d5f9 100644 --- a/backend/services/integrations/integration/elasticsearch.go +++ b/backend/services/integrations/integration/elasticsearch.go @@ -12,7 +12,6 @@ import ( "time" "openreplay/backend/pkg/messages" - "openreplay/backend/pkg/utime" ) type elasticsearch struct { @@ -164,7 +163,7 @@ func (es *elasticsearch) Request(c *client) error { c.errChan <- err continue } - timestamp := uint64(utime.ToMilliseconds(esLog.Time)) + timestamp := uint64(esLog.Time.UnixMilli()) c.setLastMessageTimestamp(timestamp) var sessionID uint64 diff --git a/backend/services/integrations/integration/newrelic.go b/backend/services/integrations/integration/newrelic.go index 937ab166d..2dce79aa5 100644 --- a/backend/services/integrations/integration/newrelic.go +++ b/backend/services/integrations/integration/newrelic.go @@ -2,25 +2,24 @@ package integration import ( "encoding/json" - "time" + "errors" "fmt" - "net/http" "io" - "io/ioutil" - "errors" + "io/ioutil" + "net/http" + "time" "openreplay/backend/pkg/messages" ) /* - We use insights-api for query. They also have Logs and Events + We use insights-api for query. They also have Logs and Events */ - // TODO: Eu/us type newrelic struct { - ApplicationId string //`json:"application_id"` - XQueryKey string //`json:"x_query_key"` + ApplicationId string //`json:"application_id"` + XQueryKey string //`json:"x_query_key"` } // TODO: Recheck @@ -34,14 +33,14 @@ type newrelicResponce struct { type newrelicEvent struct { //AsayerSessionID uint64 `json:"asayer_session_id,string"` // string/int decoder? OpenReplaySessionToken string `json:"openReplaySessionToken"` - ErrorClass string `json:"error.class"` - Timestamp uint64 `json:"timestamp"` + ErrorClass string `json:"error.class"` + Timestamp uint64 `json:"timestamp"` } func (nr *newrelic) Request(c *client) error { sinceTs := c.getLastMessageTimestamp() + 1000 // From next second // In docs - format "yyyy-mm-dd HH:MM:ss", but time.RFC3339 works fine too - sinceFormatted := time.Unix(0, int64(sinceTs*1e6)).Format(time.RFC3339) + sinceFormatted := time.UnixMilli(int64(sinceTs)).Format(time.RFC3339) // US/EU endpoint ?? requestURL := fmt.Sprintf("https://insights-api.eu.newrelic.com/v1/accounts/%v/query", nr.ApplicationId) req, err := http.NewRequest("GET", requestURL, nil) @@ -64,11 +63,10 @@ func (nr *newrelic) Request(c *client) error { } defer resp.Body.Close() - // 401 (unauthorised) if wrong XQueryKey/deploymentServer is wrong or 403 (Forbidden) if ApplicationId is wrong // 400 if Query has problems if resp.StatusCode >= 400 { - io.Copy(ioutil.Discard, resp.Body) // Read the body to free socket + io.Copy(ioutil.Discard, resp.Body) // Read the body to free socket return fmt.Errorf("Newrelic: server respond with the code %v| Request: ", resp.StatusCode, *req) } // Pagination depending on returning metadata ? @@ -92,13 +90,13 @@ func (nr *newrelic) Request(c *client) error { c.evChan <- &SessionErrorEvent{ Token: e.OpenReplaySessionToken, RawErrorEvent: &messages.RawErrorEvent{ - Source: "newrelic", + Source: "newrelic", Timestamp: e.Timestamp, - Name: e.ErrorClass, - Payload: string(jsonEvent), + Name: e.ErrorClass, + Payload: string(jsonEvent), }, } } } return nil -} \ No newline at end of file +} diff --git a/backend/services/integrations/integration/sentry.go b/backend/services/integrations/integration/sentry.go index 0330430c3..1c5bfdaad 100644 --- a/backend/services/integrations/integration/sentry.go +++ b/backend/services/integrations/integration/sentry.go @@ -1,44 +1,41 @@ package integration import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" "net/http" "net/url" - "encoding/json" - "strings" - "fmt" - "time" "strconv" - "io" - "io/ioutil" + "strings" + "time" - "openreplay/backend/pkg/utime" "openreplay/backend/pkg/messages" ) - -/* +/* They also have different stuff - Documentation says: + Documentation says: "Note: This endpoint is experimental and may be removed without notice." */ type sentry struct { OrganizationSlug string // `json:"organization_slug"` - ProjectSlug string // `json:"project_slug"` - Token string // `json:"token"` + ProjectSlug string // `json:"project_slug"` + Token string // `json:"token"` } type sentryEvent struct { Tags []struct { - Key string - Value string `json:"value"` + Key string + Value string `json:"value"` } - DateCreated string `json:"dateCreated"` // or dateReceived ? - Title string - EventID string `json:"eventID"` + DateCreated string `json:"dateCreated"` // or dateReceived ? + Title string + EventID string `json:"eventID"` } - func (sn *sentry) Request(c *client) error { requestURL := fmt.Sprintf("https://sentry.io/api/0/projects/%v/%v/events/", sn.OrganizationSlug, sn.ProjectSlug) req, err := http.NewRequest("GET", requestURL, nil) @@ -88,9 +85,9 @@ PageLoop: c.errChan <- fmt.Errorf("%v | Event: %v", err, e) continue } - timestamp := uint64(utime.ToMilliseconds(parsedTime)) + timestamp := uint64(parsedTime.UnixMilli()) // TODO: not to receive all the messages (use default integration timestamp) - if firstEvent { // TODO: reverse range? + if firstEvent { // TODO: reverse range? c.setLastMessageId(timestamp, e.EventID) firstEvent = false } @@ -117,12 +114,12 @@ PageLoop: c.evChan <- &SessionErrorEvent{ SessionID: sessionID, - Token: token, + Token: token, RawErrorEvent: &messages.RawErrorEvent{ - Source: "sentry", + Source: "sentry", Timestamp: timestamp, - Name: e.Title, - Payload: string(jsonEvent), + Name: e.Title, + Payload: string(jsonEvent), }, } } @@ -137,7 +134,7 @@ PageLoop: return fmt.Errorf("Link header format error. Got: '%v'", linkHeader) } - nextLinkInfo := pagInfo[ 1 ] + nextLinkInfo := pagInfo[1] if strings.Contains(nextLinkInfo, `results="false"`) { break } @@ -151,4 +148,4 @@ PageLoop: } } return nil -} \ No newline at end of file +} diff --git a/backend/services/integrations/integration/stackdriver.go b/backend/services/integrations/integration/stackdriver.go index bb8e3cef9..e852d5d36 100644 --- a/backend/services/integrations/integration/stackdriver.go +++ b/backend/services/integrations/integration/stackdriver.go @@ -1,22 +1,19 @@ package integration - import ( - "google.golang.org/api/option" "cloud.google.com/go/logging/logadmin" "google.golang.org/api/iterator" - - //"strconv" - "encoding/json" - "time" - "fmt" - "context" + "google.golang.org/api/option" + + //"strconv" + "context" + "encoding/json" + "fmt" + "time" - "openreplay/backend/pkg/utime" "openreplay/backend/pkg/messages" ) - // Old: asayerSessionId const SD_FILTER_QUERY = ` @@ -28,7 +25,7 @@ const SD_FILTER_QUERY = ` type stackdriver struct { ServiceAccountCredentials string // `json:"service_account_credentials"` - LogName string // `json:"log_name"` + LogName string // `json:"log_name"` } type saCreds struct { @@ -37,10 +34,10 @@ type saCreds struct { func (sd *stackdriver) Request(c *client) error { fromTs := c.getLastMessageTimestamp() + 1 // Timestamp is RFC3339Nano, so we take the next millisecond - fromFormatted := time.Unix(0, int64(fromTs *1e6)).Format(time.RFC3339Nano) + fromFormatted := time.UnixMilli(int64(fromTs)).Format(time.RFC3339Nano) ctx := context.Background() - var parsedCreds saCreds + var parsedCreds saCreds err := json.Unmarshal([]byte(sd.ServiceAccountCredentials), &parsedCreds) if err != nil { return err @@ -49,56 +46,56 @@ func (sd *stackdriver) Request(c *client) error { opt := option.WithCredentialsJSON([]byte(sd.ServiceAccountCredentials)) client, err := logadmin.NewClient(ctx, parsedCreds.ProjectId, opt) if err != nil { - return err + return err } defer client.Close() - - filter := fmt.Sprintf(SD_FILTER_QUERY, parsedCreds.ProjectId, sd.LogName, fromFormatted) - // By default, Entries are listed from oldest to newest. - /* ResourceNames(rns []string) - "projects/[PROJECT_ID]" - "organizations/[ORGANIZATION_ID]" - "billingAccounts/[BILLING_ACCOUNT_ID]" - "folders/[FOLDER_ID]" - */ - it := client.Entries(ctx, logadmin.Filter(filter)) - // TODO: Pagination: - //pager := iterator.NewPager(it, 1000, "") - //nextToken, err := pager.NextPage(&entries) - //if nextToken == "" { break } - for { - e, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - return err - } + filter := fmt.Sprintf(SD_FILTER_QUERY, parsedCreds.ProjectId, sd.LogName, fromFormatted) + // By default, Entries are listed from oldest to newest. + /* ResourceNames(rns []string) + "projects/[PROJECT_ID]" + "organizations/[ORGANIZATION_ID]" + "billingAccounts/[BILLING_ACCOUNT_ID]" + "folders/[FOLDER_ID]" + */ + it := client.Entries(ctx, logadmin.Filter(filter)) - token := e.Labels["openReplaySessionToken"] - // sessionID, err := strconv.ParseUint(strSessionID, 10, 64) - // if err != nil { - // c.errChan <- err - // continue - // } - jsonEvent, err := json.Marshal(e) - if err != nil { - c.errChan <- err - continue - } - timestamp := uint64(utime.ToMilliseconds(e.Timestamp)) - c.setLastMessageTimestamp(timestamp) - c.evChan <- &SessionErrorEvent{ + // TODO: Pagination: + //pager := iterator.NewPager(it, 1000, "") + //nextToken, err := pager.NextPage(&entries) + //if nextToken == "" { break } + for { + e, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return err + } + + token := e.Labels["openReplaySessionToken"] + // sessionID, err := strconv.ParseUint(strSessionID, 10, 64) + // if err != nil { + // c.errChan <- err + // continue + // } + jsonEvent, err := json.Marshal(e) + if err != nil { + c.errChan <- err + continue + } + timestamp := uint64(e.Timestamp.UnixMilli()) + c.setLastMessageTimestamp(timestamp) + c.evChan <- &SessionErrorEvent{ //SessionID: sessionID, Token: token, RawErrorEvent: &messages.RawErrorEvent{ - Source: "stackdriver", + Source: "stackdriver", Timestamp: timestamp, - Name: e.InsertID, // not sure about that - Payload: string(jsonEvent), + Name: e.InsertID, // not sure about that + Payload: string(jsonEvent), }, } } return nil -} \ No newline at end of file +} diff --git a/backend/services/integrations/integration/sumologic.go b/backend/services/integrations/integration/sumologic.go index 2660dd6ac..8ff39ec9e 100644 --- a/backend/services/integrations/integration/sumologic.go +++ b/backend/services/integrations/integration/sumologic.go @@ -1,20 +1,19 @@ package integration import ( - "net/http" - "time" "encoding/json" "fmt" - "strings" "io" - "io/ioutil" + "io/ioutil" + "net/http" + "strings" + "time" - "openreplay/backend/pkg/utime" "openreplay/backend/pkg/messages" ) -/* - The maximum value for limit is 10,000 messages or 100 MB in total message size, +/* + The maximum value for limit is 10,000 messages or 100 MB in total message size, which means the query may return less than 10,000 messages if you exceed the size limit. API Documentation: https://help.sumologic.com/APIs/Search-Job-API/About-the-Search-Job-API @@ -22,31 +21,30 @@ import ( const SL_LIMIT = 10000 type sumologic struct { - AccessId string // `json:"access_id"` - AccessKey string // `json:"access_key"` - cookies []*http.Cookie + AccessId string // `json:"access_id"` + AccessKey string // `json:"access_key"` + cookies []*http.Cookie } - type sumplogicJobResponce struct { Id string } type sumologicJobStatusResponce struct { - State string + State string MessageCount int //PendingErrors []string } type sumologicResponce struct { - Messages [] struct { + Messages []struct { Map json.RawMessage } } type sumologicEvent struct { Timestamp uint64 `json:"_messagetime,string"` - Raw string `json:"_raw"` + Raw string `json:"_raw"` } func (sl *sumologic) deleteJob(jobId string, errChan chan<- error) { @@ -68,10 +66,9 @@ func (sl *sumologic) deleteJob(jobId string, errChan chan<- error) { resp.Body.Close() } - func (sl *sumologic) Request(c *client) error { fromTs := c.getLastMessageTimestamp() + 1 // From next millisecond - toTs := utime.CurrentTimestamp() + toTs := time.Now().UnixMilli() requestURL := fmt.Sprintf("https://api.%vsumologic.com/api/v1/search/jobs", "eu.") // deployment server?? jsonBody := fmt.Sprintf(`{ "query": "\"openReplaySessionToken=\" AND (*error* OR *fail* OR *exception*)", @@ -132,7 +129,7 @@ func (sl *sumologic) Request(c *client) error { tick := time.Tick(5 * time.Second) for { - <- tick + <-tick resp, err = http.DefaultClient.Do(req) if err != nil { return err // TODO: retry, counter/timeout @@ -147,12 +144,12 @@ func (sl *sumologic) Request(c *client) error { } if jobStatus.State == "DONE GATHERING RESULTS" { offset := 0 - for ;offset < jobStatus.MessageCount; { + for offset < jobStatus.MessageCount { requestURL = fmt.Sprintf( - "https://api.%vsumologic.com/api/v1/search/jobs/%v/messages?offset=%v&limit=%v", - "eu.", - jobResponce.Id, - offset, + "https://api.%vsumologic.com/api/v1/search/jobs/%v/messages?offset=%v&limit=%v", + "eu.", + jobResponce.Id, + offset, SL_LIMIT, ) req, err = http.NewRequest("GET", requestURL, nil) @@ -190,17 +187,17 @@ func (sl *sumologic) Request(c *client) error { } name := e.Raw if len(name) > 20 { - name = name[:20] // not sure about that + name = name[:20] // not sure about that } c.setLastMessageTimestamp(e.Timestamp) c.evChan <- &SessionErrorEvent{ //SessionID: sessionID, Token: token, RawErrorEvent: &messages.RawErrorEvent{ - Source: "sumologic", + Source: "sumologic", Timestamp: e.Timestamp, - Name: name, - Payload: string(m.Map), //e.Raw ? + Name: name, + Payload: string(m.Map), //e.Raw ? }, } @@ -209,11 +206,11 @@ func (sl *sumologic) Request(c *client) error { } break } - if jobStatus.State != "NOT STARTED" && + if jobStatus.State != "NOT STARTED" && jobStatus.State != "GATHERING RESULTS" { // error break } } return nil -} \ No newline at end of file +} diff --git a/ee/backend/pkg/kafka/consumer.go b/ee/backend/pkg/kafka/consumer.go index 82aa56d50..cb3714316 100644 --- a/ee/backend/pkg/kafka/consumer.go +++ b/ee/backend/pkg/kafka/consumer.go @@ -147,7 +147,7 @@ func (consumer *Consumer) ConsumeNext() error { if e.TopicPartition.Error != nil { return errors.Wrap(e.TopicPartition.Error, "Consumer Partition Error") } - ts := e.Timestamp.UnixNano() / 1e6 + ts := e.Timestamp.UnixMilli() consumer.messageHandler(decodeKey(e.Key), e.Value, &types.Meta{ Topic: *(e.TopicPartition.Topic), ID: uint64(e.TopicPartition.Offset), From 1807174b8be9b1755588e954f72119798ada9125 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 1 Apr 2022 20:29:13 +0200 Subject: [PATCH 025/125] feat(api): dashboard get predefined metrics charts feat(api): support root_path for Uvicorn server --- api/app.py | 2 +- api/chalicelib/core/custom_metrics.py | 51 +++++++++++++---- api/chalicelib/core/dashboards2.py | 79 ++++++++++++++++++++++++++- api/routers/subs/metrics.py | 20 ++++++- api/schemas.py | 10 ++++ 5 files changed, 144 insertions(+), 18 deletions(-) diff --git a/api/app.py b/api/app.py index ae7f051bf..959f1ef8f 100644 --- a/api/app.py +++ b/api/app.py @@ -13,7 +13,7 @@ from routers.crons import core_crons from routers.crons import core_dynamic_crons from routers.subs import dashboard, insights, metrics, v1_api -app = FastAPI() +app = FastAPI(root_path="/api") @app.middleware('http') diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 6492080f8..839ba4d72 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -54,13 +54,9 @@ def merged_live(project_id, data: schemas.CreateCustomMetricsSchema): return results -def __get_merged_metric(project_id, user_id, metric_id, - data: Union[schemas.CustomMetricChartPayloadSchema, - schemas.CustomMetricSessionsPayloadSchema]) \ +def __merge_metric_with_data(metric, data: Union[schemas.CustomMetricChartPayloadSchema, + schemas.CustomMetricSessionsPayloadSchema]) \ -> Union[schemas.CreateCustomMetricsSchema, None]: - metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) - if metric is None: - return None metric: schemas.CreateCustomMetricsSchema = schemas.CreateCustomMetricsSchema.parse_obj({**data.dict(), **metric}) if len(data.filters) > 0 or len(data.events) > 0: for s in metric.series: @@ -71,11 +67,12 @@ def __get_merged_metric(project_id, user_id, metric_id, return metric -def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema): - metric: schemas.CreateCustomMetricsSchema = __get_merged_metric(project_id=project_id, user_id=user_id, - metric_id=metric_id, data=data) +def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema, metric=None): + if metric is None: + metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) if metric is None: return None + metric: schemas.CreateCustomMetricsSchema = __merge_metric_with_data(metric=metric, data=data) series_charts = __try_live(project_id=project_id, data=metric) if metric.view_type == schemas.MetricTimeseriesViewType.progress or metric.metric_type == schemas.MetricType.table: return series_charts @@ -88,8 +85,10 @@ def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPa def get_sessions(project_id, user_id, metric_id, data: schemas.CustomMetricSessionsPayloadSchema): - metric: schemas.CreateCustomMetricsSchema = __get_merged_metric(project_id=project_id, user_id=user_id, - metric_id=metric_id, data=data) + metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) + if metric is None: + return None + metric: schemas.CreateCustomMetricsSchema = __merge_metric_with_data(metric=metric, data=data) if metric is None: return None results = [] @@ -294,6 +293,36 @@ def get(metric_id, project_id, user_id, flatten=True): return helper.dict_to_camel_case(row) +def get_with_template(metric_id, project_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT * + FROM metrics + LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(metric_series.* ORDER BY index),'[]'::jsonb) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards + FROM (SELECT dashboard_id, name, is_public + FROM dashboards + WHERE deleted_at ISNULL + AND project_id = %(project_id)s + AND ((user_id = %(user_id)s OR is_public))) AS connected_dashboards + ) AS connected_dashboards ON (TRUE) + WHERE (metrics.project_id = %(project_id)s OR metrics.project_id ISNULL) + AND metrics.deleted_at ISNULL + AND (metrics.user_id = %(user_id)s OR metrics.is_public) + AND metrics.metric_id = %(metric_id)s + ORDER BY created_at;""", + {"metric_id": metric_id, "project_id": project_id, "user_id": user_id} + ) + ) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + def get_series_for_alert(project_id, user_id): with pg_client.PostgresClient() as cur: cur.execute( diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index c292ed1a0..7e06f52ac 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -1,7 +1,7 @@ import json import schemas -from chalicelib.core import custom_metrics +from chalicelib.core import custom_metrics, dashboard from chalicelib.utils import helper from chalicelib.utils import pg_client @@ -79,7 +79,6 @@ def get_dashboard(project_id, user_id, dashboard_id): AND dashboard_id = %(dashboard_id)s AND (dashboards.user_id = %(userId)s OR is_public);""" params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id} - print(cur.mogrify(pg_query, params)) cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() return helper.dict_to_camel_case(row) @@ -111,6 +110,30 @@ def update_dashboard(project_id, user_id, dashboard_id, data: schemas.EditDashbo return helper.dict_to_camel_case(row) +def get_widget(project_id, user_id, dashboard_id, widget_id): + with pg_client.PostgresClient() as cur: + pg_query = """SELECT metrics.*, metric_series.series + FROM dashboard_widgets + INNER JOIN dashboards USING (dashboard_id) + INNER JOIN metrics USING (metric_id) + LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(metric_series.* ORDER BY index), '[]'::jsonb) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + WHERE dashboard_id = %(dashboard_id)s + AND widget_id = %(widget_id)s + AND (dashboards.is_public OR dashboards.user_id = %(userId)s) + AND dashboards.deleted_at IS NULL + AND metrics.deleted_at ISNULL + AND (metrics.project_id = %(projectId)s OR metrics.project_id ISNULL) + AND (metrics.is_public OR metrics.user_id = %(userId)s);""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, "widget_id": widget_id} + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + def add_widget(project_id, user_id, dashboard_id, data: schemas.AddWidgetToDashboardPayloadSchema): with pg_client.PostgresClient() as cur: pg_query = """INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config, name) @@ -154,7 +177,7 @@ def pin_dashboard(project_id, user_id, dashboard_id): with pg_client.PostgresClient() as cur: pg_query = """UPDATE dashboards SET is_pinned = FALSE - WHERE dashboard_id=%(dashboard_id)s AND project_id=%(project_id)s; + WHERE project_id=%(project_id)s; UPDATE dashboards SET is_pinned = True WHERE dashboard_id=%(dashboard_id)s AND project_id=%(project_id)s AND deleted_at ISNULL @@ -169,3 +192,53 @@ def create_metric_add_widget(project_id, user_id, dashboard_id, data: schemas.Cr metric_id = custom_metrics.create(project_id=project_id, user_id=user_id, data=data, dashboard=True) return add_widget(project_id=project_id, user_id=user_id, dashboard_id=dashboard_id, data=schemas.AddWidgetToDashboardPayloadSchema(metric_id=metric_id)) + + +PREDEFINED = {schemas.TemplateKeys.count_sessions: dashboard.get_processed_sessions, + schemas.TemplateKeys.avg_image_load_time: dashboard.get_application_activity_avg_image_load_time, + schemas.TemplateKeys.avg_page_load_time: dashboard.get_application_activity_avg_page_load_time, + schemas.TemplateKeys.avg_request_load_time: dashboard.get_application_activity_avg_request_load_time, + schemas.TemplateKeys.avg_dom_content_load_start: dashboard.get_page_metrics_avg_dom_content_load_start, + schemas.TemplateKeys.avg_first_contentful_pixel: dashboard.get_page_metrics_avg_first_contentful_pixel, + schemas.TemplateKeys.avg_visited_pages: dashboard.get_user_activity_avg_visited_pages, + schemas.TemplateKeys.avg_session_duration: dashboard.get_user_activity_avg_session_duration, + schemas.TemplateKeys.avg_pages_dom_buildtime: dashboard.get_pages_dom_build_time, + schemas.TemplateKeys.avg_pages_response_time: dashboard.get_pages_response_time, + schemas.TemplateKeys.avg_response_time: dashboard.get_top_metrics_avg_response_time, + schemas.TemplateKeys.avg_first_paint: dashboard.get_top_metrics_avg_first_paint, + schemas.TemplateKeys.avg_dom_content_loaded: dashboard.get_top_metrics_avg_dom_content_loaded, + schemas.TemplateKeys.avg_till_first_bit: dashboard.get_top_metrics_avg_till_first_bit, + schemas.TemplateKeys.avg_time_to_interactive: dashboard.get_top_metrics_avg_time_to_interactive, + schemas.TemplateKeys.count_requests: dashboard.get_top_metrics_count_requests, + schemas.TemplateKeys.avg_time_to_render: dashboard.get_time_to_render, + schemas.TemplateKeys.avg_used_js_heap_size: dashboard.get_memory_consumption, + schemas.TemplateKeys.avg_cpu: dashboard.get_avg_cpu, + schemas.TemplateKeys.avg_fps: dashboard.get_avg_fps} + + +def get_predefined_metric(key: schemas.TemplateKeys, project_id: int, data: dict): + return PREDEFINED.get(key, lambda *args: None)(project_id=project_id, **data) + + +def make_chart_metrics(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema): + raw_metric = custom_metrics.get_with_template(metric_id=metric_id, project_id=project_id, user_id=user_id) + if raw_metric is None: + return None + metric = schemas.CustomMetricAndTemplate = schemas.CustomMetricAndTemplate.parse_obj(raw_metric) + if metric.is_template: + return get_predefined_metric(key=metric.key, project_id=project_id, data=data.dict()) + else: + return custom_metrics.make_chart(project_id=project_id, user_id=user_id, metric_id=metric_id, data=data, + metric=raw_metric) + + +def make_chart_widget(dashboard_id, project_id, user_id, widget_id, data: schemas.CustomMetricChartPayloadSchema): + raw_metric = get_widget(widget_id=widget_id, project_id=project_id, user_id=user_id, dashboard_id=dashboard_id) + if raw_metric is None: + return None + metric = schemas.CustomMetricAndTemplate = schemas.CustomMetricAndTemplate.parse_obj(raw_metric) + if metric.is_template: + return get_predefined_metric(key=metric.key, project_id=project_id, data=data.dict()) + else: + return custom_metrics.make_chart(project_id=project_id, user_id=user_id, metric_id=raw_metric["metricId"], + data=data, metric=raw_metric) diff --git a/api/routers/subs/metrics.py b/api/routers/subs/metrics.py index 4504ba8f1..fc875e0fd 100644 --- a/api/routers/subs/metrics.py +++ b/api/routers/subs/metrics.py @@ -22,7 +22,10 @@ def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_ @app.get('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) def get_dashboard(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": dashboards2.get_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId)} + data = dashboards2.get_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId) + if data is None: + return {"errors": ["dashboard not found"]} + return {"data": data} @app.post('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) @@ -78,6 +81,17 @@ def remove_widget_from_dashboard(projectId: int, dashboardId: int, widgetId: int widget_id=widgetId) +@app.post('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}/chart', tags=["dashboard"]) +def get_widget_chart(projectId: int, dashboardId: int, widgetId: int, + data: schemas.CustomMetricChartPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + data = dashboards2.make_chart_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, + widget_id=widgetId, data=data) + if data is None: + return {"errors": ["widget not found"]} + return {"data": data} + + @app.get('/{projectId}/metrics/templates', tags=["dashboard"]) def get_templates(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): return {"data": dashboards2.get_templates(project_id=projectId, user_id=context.user_id)} @@ -131,8 +145,8 @@ def get_custom_metric_sessions(projectId: int, metric_id: int, @app.post('/{projectId}/custom_metrics/{metric_id}/chart', tags=["customMetrics"]) def get_custom_metric_chart(projectId: int, metric_id: int, data: schemas.CustomMetricChartPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - data = custom_metrics.make_chart(project_id=projectId, user_id=context.user_id, metric_id=metric_id, - data=data) + data = dashboards2.make_chart_metrics(project_id=projectId, user_id=context.user_id, metric_id=metric_id, + data=data) if data is None: return {"errors": ["custom metric not found"]} return {"data": data} diff --git a/api/schemas.py b/api/schemas.py index 821326728..b707a53e1 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -926,3 +926,13 @@ class TemplateKeys(str, Enum): avg_used_js_heap_size = "avg_used_js_heap_size" avg_cpu = "avg_cpu" avg_fps = "avg_fps" + + +# class CustomMetricAndTemplate(CreateCustomMetricsSchema): +class CustomMetricAndTemplate(BaseModel): + is_template: bool = Field(...) + project_id: Optional[int] = Field(...) + key: Optional[TemplateKeys] = Field(...) + + class Config: + alias_generator = attribute_to_camel_case From d7710356e94505e2146528412f9517709168b2f2 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Fri, 1 Apr 2022 22:15:51 +0200 Subject: [PATCH 026/125] feat(tracker-fetch):3.5.3: common improved sanitiser --- tracker/tracker-fetch/README.md | 123 ++++++++++++++++++++++++----- tracker/tracker-fetch/package.json | 2 +- tracker/tracker-fetch/src/index.ts | 122 +++++++++++++++------------- 3 files changed, 170 insertions(+), 77 deletions(-) diff --git a/tracker/tracker-fetch/README.md b/tracker/tracker-fetch/README.md index b7fca2e4b..53fddcf8e 100644 --- a/tracker/tracker-fetch/README.md +++ b/tracker/tracker-fetch/README.md @@ -1,7 +1,6 @@ -# OpenReplay Tracker Fetch plugin +# Fetch plugin for OpenReplay -Tracker plugin to support tracking of the `fetch` requests payload. -Additionally it populates the requests with `sessionToken` header for backend logging. +This plugin allows you to capture `fetch` payloads and inspect them later on while replaying session recordings. This is very useful for understanding and fixing issues. ## Installation @@ -11,36 +10,120 @@ npm i @openreplay/tracker-fetch ## Usage -Initialize the `@openreplay/tracker` package as usual and load the plugin into it. -Then you can use the provided `fetch` method from the plugin instead of built-in. +Use the provided `fetch` method from the plugin instead of the one built-in. + +### If your website is a Single Page Application (SPA) ```js -import Tracker from '@openreplay/tracker'; +import tracker from '@openreplay/tracker'; import trackerFetch from '@openreplay/tracker-fetch'; -const tracker = new Tracker({ - projectKey: YOUR_PROJECT_KEY, +const tracker = new OpenReplay({ + projectKey: PROJECT_KEY }); +const fetch = tracker.use(trackerFetch(options)); // check list of available options below + tracker.start(); -export const fetch = tracker.use(trackerFetch({ /* options here*/ })); - -fetch('https://my.api.io/resource').then(response => response.json()).then(body => console.log(body)); +fetch('https://myapi.com/').then(response => console.log(response.json())); ``` -Options: -```ts -{ - failuresOnly: boolean, // default false - sessionTokenHeader: string | undefined, // default undefined - ignoreHeaders: Array | boolean, // default [ 'Cookie', 'Set-Cookie', 'Authorization' ] +### If your web app is Server-Side-Rendered (SSR) + +Follow the below example if your app is SSR. Ensure `tracker.start()` is called once the app is started (in `useEffect` or `componentDidMount`). + +```js +import OpenReplay from '@openreplay/tracker/cjs'; +import trackerFetch from '@openreplay/tracker-fetch/cjs'; + +const tracker = new OpenReplay({ + projectKey: PROJECT_KEY +}); +const fetch = tracker.use(trackerFetch(options)); // check list of available options below + +//... +function MyApp() { + useEffect(() => { // use componentDidMount in case of React Class Component + tracker.start(); + + fetch('https://myapi.com/').then(response => console.log(response.json())); + }, []) +//... } ``` -Set `failuresOnly` option to `true` if you want to record only requests with the status code >= 400. +## Options -In case you use [OpenReplay integrations (sentry, bugsnag or others)](https://docs.openreplay.com/integrations), you can use `sessionTokenHeader` option to specify the header name. This header will be appended automatically to the each fetch request and will contain OpenReplay session identificator value. +```js +trackerFetch({ + overrideGlobal: boolean; + failuresOnly: boolean; + sessionTokenHeader: string; + ignoreHeaders: Array | boolean; + sanitiser: (RequestResponseData) => RequestResponseData | null; +}) +``` -You can define list of headers that you don't want to capture with the `ignoreHeaders` options. Set its value to `false` if you want to catch them all (`true` if opposite). By default plugin ignores the list of headers that might be sensetive such as `[ 'Cookie', 'Set-Cookie', 'Authorization' ]`. +- `overrideGlobal`: Overrides the default `window.fetch`. Default: `false`. +- `failuresOnly`: Captures requests having 4xx-5xx HTTP status code. Default: `false`. +- `sessionTokenHeader`: In case you have enabled some of our backend [integrations](/integrations) (i.e. Sentry), you can use this option to specify the header name (i.e. 'X-OpenReplay-SessionToken'). This latter gets appended automatically to each fetch request to contain the OpenReplay sessionToken's value. Default: `undefined`. +- `ignoreHeaders`: Helps define a list of headers you don't wish to capture. Set its value to `false` to capture all of them (`true` if none). Default: `['Cookie', 'Set-Cookie', 'Authorization']` so sensitive headers won't be captured. +- `sanitiser`: Sanitise sensitive data from fetch request/response or ignore request comletely. You can redact fields on the request object by modifying then returning it from the function: +```typescript +interface RequestData { + body: BodyInit | null | undefined; // whatewer you've put in the init.body in fetch(url, init) + headers: Record; +} + +interface ResponseData { + body: string | Object | null; // Object if response is of JSON type + headers: Record; +} + +interface RequestResponseData { + readonly status: number; + readonly method: string; + url: string; + request: RequestData; + response: ResponseData; +} + +sanitiser: (data: RequestResponseData) => { // sanitise the body or headers + if (data.url === "/auth") { + data.request.body = null + } + + if (data.request.headers['x-auth-token']) { // can also use ignoreHeaders option instead + data.request.headers['x-auth-token'] = 'SANITISED'; + } + + // Sanitise response + if (data.status < 400 && data.response.body.token) { + data.response.body.token = "" + } + + return data +} + +// OR + +sanitiser: data => { // ignore requests that start with /secure + if (data.url.startsWith("/secure")) { + return null + } + return data +} + +// OR + +sanitiser: data => { // sanitise request url: replace all numbers + data.url = data.url.replace(/\d/g, "*") + return data +} +``` + +## Troubleshooting + +Having trouble setting up this plugin? please connect to our [Discord](https://discord.openreplay.com) and get help from our community. \ No newline at end of file diff --git a/tracker/tracker-fetch/package.json b/tracker/tracker-fetch/package.json index 0b1373edc..c13b1a28b 100644 --- a/tracker/tracker-fetch/package.json +++ b/tracker/tracker-fetch/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-fetch", "description": "Tracker plugin for fetch requests recording ", - "version": "3.5.2", + "version": "3.5.3", "keywords": [ "fetch", "logging", diff --git a/tracker/tracker-fetch/src/index.ts b/tracker/tracker-fetch/src/index.ts index 93e39cce5..922913923 100644 --- a/tracker/tracker-fetch/src/index.ts +++ b/tracker/tracker-fetch/src/index.ts @@ -1,38 +1,48 @@ import { App, Messages } from '@openreplay/tracker'; -interface Request { - url: string, - body: string | Object, - headers: Record, +interface RequestData { + body: BodyInit | null | undefined + headers: Record } -interface Response { - url: string, - status: number, - body: string, - headers: Record, +interface ResponseData { + body: string | Object | null + headers: Record } +interface RequestResponseData { + readonly status: number + readonly method: string + url: string + request: RequestData + response: ResponseData +} + + export interface Options { - sessionTokenHeader?: string; - replaceDefault: boolean; // overrideDefault ? - failuresOnly: boolean; - ignoreHeaders: Array | boolean; - requestSanitizer: ((Request) => Request | null) | null; - responseSanitizer: ((Response) => Response | null) | null; + sessionTokenHeader?: string + failuresOnly: boolean + overrideGlobal: boolean + ignoreHeaders: Array | boolean + sanitiser?: (RequestResponseData) => RequestResponseData | null + + requestSanitizer?: any + responseSanitizer?: any } export default function(opts: Partial = {}) { const options: Options = Object.assign( { - replaceDefault: false, + overrideGlobal: false, failuresOnly: false, ignoreHeaders: [ 'Cookie', 'Set-Cookie', 'Authorization' ], - requestSanitizer: null, - responseSanitizer: null, }, opts, ); + if (options.requestSanitizer && options.responseSanitizer) { + console.warn("OpenReplay fetch plugin: `requestSanitizer` and `responseSanitizer` options are depricated. Please, use `sanitiser` instead (check out documentation at https://docs.openreplay.com/plugins/fetch).") + } + const origFetch = window.fetch return (app: App | null) => { if (app === null) { @@ -90,56 +100,55 @@ export default function(opts: Partial = {}) { r.headers.forEach((v, n) => { if (!isHIgnoring(n)) resHs[n] = v }) } - // Request forming - let reqBody = '' - if (typeof init.body === 'string') { - reqBody = init.body - } else if (typeof init.body === 'object') { - try { - reqBody = JSON.stringify(init.body) - } catch {} - } - let req: Request | null = { - url: input, + const req: RequestData = { headers: reqHs, - body: reqBody, - } - if (options.requestSanitizer !== null) { - req = options.requestSanitizer(req) - if (!req) { - return - } + body: init.body, } // Response forming - let res: Response | null = { - url: input, - status: r.status, + const res: ResponseData = { headers: resHs, body: text, } - if (options.responseSanitizer !== null) { - res = options.responseSanitizer(res) - if (!res) { + + const method = typeof init.method === 'string' + ? init.method.toUpperCase() + : 'GET' + let reqResInfo: RequestResponseData | null = { + url: input, + method, + status: r.status, + request: req, + response: res, + } + if (options.sanitiser) { + try { + reqResInfo.response.body = JSON.parse(text) as Object // Why the returning type is "any"? + } catch {} + reqResInfo = options.sanitiser(reqResInfo) + if (!reqResInfo) { return } } - const reqStr = JSON.stringify({ - headers: req.headers, - body: req.body, - }) - const resStr = JSON.stringify({ - headers: res.headers, - body: res.body, - }) + const getStj = (r: RequestData | ResponseData): string => { + if (r && typeof r.body !== 'string') { + try { + r.body = JSON.stringify(r.body) + } catch { + r.body = "" + //app.log.warn("Openreplay fetch") // TODO: version check + } + } + return JSON.stringify(r) + } app.send( Messages.Fetch( - typeof init.method === 'string' ? init.method.toUpperCase() : 'GET', - input, - reqStr, - resStr, + method, + String(reqResInfo.url), + getStj(reqResInfo.request), + getStj(reqResInfo.response), r.status, startTime + performance.timing.navigationStart, duration, @@ -147,8 +156,9 @@ export default function(opts: Partial = {}) { ) }); return response; - }; - if (options.replaceDefault) { + } + + if (options.overrideGlobal) { window.fetch = fetch } return fetch; From 3b3015e025225ae0ed85ee65981daef8675f385c Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 3 Apr 2022 13:31:54 +0200 Subject: [PATCH 027/125] fix(backend-http): always Close() body --- backend/services/http/handlers-ios.go | 4 ++-- backend/services/http/handlers-web.go | 2 +- backend/services/http/handlers.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/services/http/handlers-ios.go b/backend/services/http/handlers-ios.go index f15a6af60..8116980e1 100644 --- a/backend/services/http/handlers-ios.go +++ b/backend/services/http/handlers-ios.go @@ -50,7 +50,7 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { startTime := time.Now() req := &request{} body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) - //defer body.Close() + defer body.Close() if err := json.NewDecoder(body).Decode(req); err != nil { responseWithError(w, http.StatusBadRequest, err) return @@ -155,7 +155,7 @@ func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { } r.Body = http.MaxBytesReader(w, r.Body, FILES_SIZE_LIMIT) - // defer r.Body.Close() + defer r.Body.Close() err = r.ParseMultipartForm(1e6) // ~1Mb if err == http.ErrNotMultipart || err == http.ErrMissingBoundary { responseWithError(w, http.StatusUnsupportedMediaType, err) diff --git a/backend/services/http/handlers-web.go b/backend/services/http/handlers-web.go index 29dcf161d..dcbd33720 100644 --- a/backend/services/http/handlers-web.go +++ b/backend/services/http/handlers-web.go @@ -41,7 +41,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { startTime := time.Now() req := &request{} body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) // what if Body == nil?? // use r.ContentLength to return specific error? - //defer body.Close() + defer body.Close() if err := json.NewDecoder(body).Decode(req); err != nil { responseWithError(w, http.StatusBadRequest, err) return diff --git a/backend/services/http/handlers.go b/backend/services/http/handlers.go index e45e84e64..dd73925af 100644 --- a/backend/services/http/handlers.go +++ b/backend/services/http/handlers.go @@ -9,11 +9,11 @@ import ( gzip "github.com/klauspost/pgzip" ) -const JSON_SIZE_LIMIT int64 = 1e3 // 1Kb +const JSON_SIZE_LIMIT int64 = 1e3 // 1Kb func pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64, topicName string) { body := http.MaxBytesReader(w, r.Body, BEACON_SIZE_LIMIT) - //defer body.Close() + defer body.Close() var reader io.ReadCloser var err error switch r.Header.Get("Content-Encoding") { From 642618046ad44e8c65ef067ed293c9f1e4faf464 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 4 Apr 2022 10:40:44 +0200 Subject: [PATCH 028/125] feat(api): get live session by session_id --- api/chalicelib/core/sessions.py | 9 +++++---- api/routers/core.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 1903cc08b..f2b34dc37 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -39,7 +39,8 @@ def __group_metadata(session, project_metadata): return meta -def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_viewed=False, group_metadata=False): +def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_viewed=False, group_metadata=False, + live=True): with pg_client.PostgresClient() as cur: extra_query = [] if include_fav_viewed: @@ -97,9 +98,9 @@ def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_ data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['issues'] = issues.get_by_session_id(session_id=session_id) - data['live'] = assist.is_live(project_id=project_id, - session_id=session_id, - project_key=data["projectKey"]) + data['live'] = live and assist.is_live(project_id=project_id, + session_id=session_id, + project_key=data["projectKey"]) data["inDB"] = True return data else: diff --git a/api/routers/core.py b/api/routers/core.py index 53fdba667..20864288d 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -21,6 +21,7 @@ from routers.base import get_routers public_app, app, app_apikey = get_routers() +@app.get('/{projectId}/sessions/{sessionId}', tags=["sessions"]) @app.get('/{projectId}/sessions2/{sessionId}', tags=["sessions"]) def get_session2(projectId: int, sessionId: Union[int, str], context: schemas.CurrentContext = Depends(OR_context)): if isinstance(sessionId, str): @@ -36,6 +37,7 @@ def get_session2(projectId: int, sessionId: Union[int, str], context: schemas.Cu } +@app.get('/{projectId}/sessions/{sessionId}/favorite', tags=["sessions"]) @app.get('/{projectId}/sessions2/{sessionId}/favorite', tags=["sessions"]) def add_remove_favorite_session2(projectId: int, sessionId: int, context: schemas.CurrentContext = Depends(OR_context)): @@ -44,6 +46,7 @@ def add_remove_favorite_session2(projectId: int, sessionId: int, session_id=sessionId)} +@app.get('/{projectId}/sessions/{sessionId}/assign', tags=["sessions"]) @app.get('/{projectId}/sessions2/{sessionId}/assign', tags=["sessions"]) def assign_session(projectId: int, sessionId, context: schemas.CurrentContext = Depends(OR_context)): data = sessions_assignments.get_by_session(project_id=projectId, session_id=sessionId, @@ -56,6 +59,7 @@ def assign_session(projectId: int, sessionId, context: schemas.CurrentContext = } +@app.get('/{projectId}/sessions/{sessionId}/errors/{errorId}/sourcemaps', tags=["sessions", "sourcemaps"]) @app.get('/{projectId}/sessions2/{sessionId}/errors/{errorId}/sourcemaps', tags=["sessions", "sourcemaps"]) def get_error_trace(projectId: int, sessionId: int, errorId: str, context: schemas.CurrentContext = Depends(OR_context)): @@ -67,6 +71,7 @@ def get_error_trace(projectId: int, sessionId: int, errorId: str, } +@app.get('/{projectId}/sessions/{sessionId}/assign/{issueId}', tags=["sessions", "issueTracking"]) @app.get('/{projectId}/sessions2/{sessionId}/assign/{issueId}', tags=["sessions", "issueTracking"]) def assign_session(projectId: int, sessionId: int, issueId: str, context: schemas.CurrentContext = Depends(OR_context)): @@ -79,6 +84,8 @@ def assign_session(projectId: int, sessionId: int, issueId: str, } +@app.post('/{projectId}/sessions/{sessionId}/assign/{issueId}/comment', tags=["sessions", "issueTracking"]) +@app.put('/{projectId}/sessions/{sessionId}/assign/{issueId}/comment', tags=["sessions", "issueTracking"]) @app.post('/{projectId}/sessions2/{sessionId}/assign/{issueId}/comment', tags=["sessions", "issueTracking"]) @app.put('/{projectId}/sessions2/{sessionId}/assign/{issueId}/comment', tags=["sessions", "issueTracking"]) def comment_assignment(projectId: int, sessionId: int, issueId: str, data: schemas.CommentAssignmentSchema = Body(...), @@ -825,6 +832,19 @@ def sessions_live(projectId: int, userId: str = None, context: schemas.CurrentCo return {'data': data} +@app.get('/{projectId}/assist/sessions/{sessionId}', tags=["assist"]) +def get_live_session(projectId: int, sessionId: str, context: schemas.CurrentContext = Depends(OR_context)): + data = assist.get_live_session_by_id(project_id=projectId, session_id=sessionId) + if data is None: + data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, + user_id=context.user_id, include_fav_viewed=True, group_metadata=True, live=False) + if data is None: + return {"errors": ["session not found"]} + if data.get("inDB"): + sessions_favorite_viewed.view_session(project_id=projectId, user_id=context.user_id, session_id=sessionId) + return {'data': data} + + @app.post('/{projectId}/heatmaps/url', tags=["heatmaps"]) def get_heatmaps_by_url(projectId: int, data: schemas.GetHeatmapPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): From ba9ca1d06044642648756f6a19a67d03eb6d1591 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 4 Apr 2022 12:03:30 +0200 Subject: [PATCH 029/125] feat(api): get save_request_payloads with the projects info --- api/chalicelib/core/projects.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/core/projects.py b/api/chalicelib/core/projects.py index c5ae912aa..ab0f33b4f 100644 --- a/api/chalicelib/core/projects.py +++ b/api/chalicelib/core/projects.py @@ -57,7 +57,7 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st cur.execute(f"""\ SELECT - s.project_id, s.name, s.project_key + s.project_id, s.name, s.project_key, s.save_request_payloads {',s.gdpr' if gdpr else ''} {',COALESCE((SELECT TRUE FROM public.sessions WHERE sessions.project_id = s.project_id LIMIT 1), FALSE) AS recorded' if recorded else ''} {',stack_integrations.count>0 AS stack_integrations' if stack_integrations else ''} @@ -109,7 +109,8 @@ def get_project(tenant_id, project_id, include_last_session=False, include_gdpr= SELECT s.project_id, s.project_key, - s.name + s.name, + s.save_request_payloads {",(SELECT max(ss.start_ts) FROM public.sessions AS ss WHERE ss.project_id = %(project_id)s) AS last_recorded_session_at" if include_last_session else ""} {',s.gdpr' if include_gdpr else ''} {tracker_query} From 72f2a96f31eda3129fd500706bc4cd145081bb49 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 4 Apr 2022 14:09:49 +0200 Subject: [PATCH 030/125] parrot global changes --- api/chalicelib/core/telemetry.py | 4 ++-- ee/api/chalicelib/core/telemetry.py | 4 ++-- ee/backend/pkg/license/check.go | 2 +- scripts/helm/install.sh | 2 +- scripts/helm/roles/openreplay/tasks/pre-check.yaml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/chalicelib/core/telemetry.py b/api/chalicelib/core/telemetry.py index 48f403f57..28eb97f73 100644 --- a/api/chalicelib/core/telemetry.py +++ b/api/chalicelib/core/telemetry.py @@ -30,7 +30,7 @@ def compute(): RETURNING *,(SELECT email FROM public.users WHERE role='owner' LIMIT 1);""" ) data = cur.fetchone() - requests.post('https://parrot.asayer.io/os/telemetry', json={"stats": [process_data(data)]}) + requests.post('https://api.openreplay.com/os/telemetry', json={"stats": [process_data(data)]}) def new_client(): @@ -40,4 +40,4 @@ def new_client(): (SELECT email FROM public.users WHERE role='owner' LIMIT 1) AS email FROM public.tenants;""") data = cur.fetchone() - requests.post('https://parrot.asayer.io/os/signup', json=process_data(data)) + requests.post('https://api.openreplay.com/os/signup', json=process_data(data)) diff --git a/ee/api/chalicelib/core/telemetry.py b/ee/api/chalicelib/core/telemetry.py index d9843e37d..e05df7fdc 100644 --- a/ee/api/chalicelib/core/telemetry.py +++ b/ee/api/chalicelib/core/telemetry.py @@ -53,7 +53,7 @@ def compute(): RETURNING *,(SELECT email FROM users_ee WHERE role = 'owner' AND users_ee.tenant_id = tenants.tenant_id LIMIT 1);""" ) data = cur.fetchall() - requests.post('https://parrot.asayer.io/os/telemetry', + requests.post('https://api.openreplay.com/os/telemetry', json={"stats": [process_data(d, edition='ee') for d in data]}) @@ -65,4 +65,4 @@ def new_client(tenant_id): FROM public.tenants WHERE tenant_id=%(tenant_id)s;""", {"tenant_id": tenant_id})) data = cur.fetchone() - requests.post('https://parrot.asayer.io/os/signup', json=process_data(data, edition='ee')) \ No newline at end of file + requests.post('https://api.openreplay.com/os/signup', json=process_data(data, edition='ee')) \ No newline at end of file diff --git a/ee/backend/pkg/license/check.go b/ee/backend/pkg/license/check.go index 6b33a625e..771558946 100644 --- a/ee/backend/pkg/license/check.go +++ b/ee/backend/pkg/license/check.go @@ -33,7 +33,7 @@ func CheckLicense() { log.Fatal("Can not form a license check request.") } - resp, err := http.Post("https://parrot.asayer.io/os/license", "application/json", bytes.NewReader(requestBody)) + resp, err := http.Post("https://api.openreplay.com/os/license", "application/json", bytes.NewReader(requestBody)) if err != nil { log.Fatalf("Error while checking license. %v", err) } diff --git a/scripts/helm/install.sh b/scripts/helm/install.sh index 529d08a8c..2bf39988c 100755 --- a/scripts/helm/install.sh +++ b/scripts/helm/install.sh @@ -26,7 +26,7 @@ which docker &> /dev/null || { } -# https://parrot.asayer.io/os/license +# https://api.openreplay.com/os/license # payload: {"mid": "UUID of the machine", "license": ""} # response {"data":{"valid": TRUE|FALSE, "expiration": expiration date in ms}} diff --git a/scripts/helm/roles/openreplay/tasks/pre-check.yaml b/scripts/helm/roles/openreplay/tasks/pre-check.yaml index 7f85d0ea1..90e6021c8 100644 --- a/scripts/helm/roles/openreplay/tasks/pre-check.yaml +++ b/scripts/helm/roles/openreplay/tasks/pre-check.yaml @@ -108,7 +108,7 @@ - all - name: Checking Enterprise Licence uri: - url: https://parrot.asayer.io/os/license + url: https://api.openreplay.com/os/license body: mid: "UUID of the machine" license: "{{ enterprise_edition_license }}" From 2b55aef0a782af5e4eba1f17af30bf8f668572ef Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 4 Apr 2022 14:29:49 +0200 Subject: [PATCH 031/125] feat(api): optimized boarding endpoints --- api/chalicelib/core/boarding.py | 86 +++++++++++++++--------------- api/chalicelib/core/projects.py | 10 ++++ ee/api/chalicelib/core/boarding.py | 86 +++++++++++++++--------------- ee/api/chalicelib/core/projects.py | 12 ++++- 4 files changed, 105 insertions(+), 89 deletions(-) diff --git a/api/chalicelib/core/boarding.py b/api/chalicelib/core/boarding.py index c303643c8..68843b2f8 100644 --- a/api/chalicelib/core/boarding.py +++ b/api/chalicelib/core/boarding.py @@ -5,39 +5,38 @@ from chalicelib.core import users def get_state(tenant_id): - my_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) - pids = [s["projectId"] for s in my_projects] + pids = projects.get_projects_ids(tenant_id=tenant_id) with pg_client.PostgresClient() as cur: recorded = False meta = False if len(pids) > 0: cur.execute( - cur.mogrify("""\ - SELECT - COUNT(*) - FROM public.sessions AS s - where s.project_id IN %(ids)s - LIMIT 1;""", + cur.mogrify("""SELECT EXISTS(( SELECT 1 + FROM public.sessions AS s + WHERE s.project_id IN %(ids)s)) AS exists;""", {"ids": tuple(pids)}) ) - recorded = cur.fetchone()["count"] > 0 + recorded = cur.fetchone()["exists"] meta = False if recorded: - cur.execute("""SELECT SUM((SELECT COUNT(t.meta) - FROM (VALUES (p.metadata_1), (p.metadata_2), (p.metadata_3), (p.metadata_4), (p.metadata_5), - (p.metadata_6), (p.metadata_7), (p.metadata_8), (p.metadata_9), (p.metadata_10), - (sessions.user_id)) AS t(meta) - WHERE t.meta NOTNULL)) - FROM public.projects AS p - LEFT JOIN LATERAL ( SELECT 'defined' - FROM public.sessions - WHERE sessions.project_id=p.project_id AND sessions.user_id IS NOT NULL - LIMIT 1) AS sessions(user_id) ON(TRUE) - WHERE p.deleted_at ISNULL;""" - ) + cur.execute("""SELECT EXISTS((SELECT 1 + FROM public.projects AS p + LEFT JOIN LATERAL ( SELECT 1 + FROM public.sessions + WHERE sessions.project_id = p.project_id + AND sessions.user_id IS NOT NULL + LIMIT 1) AS sessions(user_id) ON (TRUE) + WHERE p.deleted_at ISNULL + AND ( sessions.user_id IS NOT NULL OR p.metadata_1 IS NOT NULL + OR p.metadata_2 IS NOT NULL OR p.metadata_3 IS NOT NULL + OR p.metadata_4 IS NOT NULL OR p.metadata_5 IS NOT NULL + OR p.metadata_6 IS NOT NULL OR p.metadata_7 IS NOT NULL + OR p.metadata_8 IS NOT NULL OR p.metadata_9 IS NOT NULL + OR p.metadata_10 IS NOT NULL ) + )) AS exists;""") - meta = cur.fetchone()["sum"] > 0 + meta = cur.fetchone()["exists"] return [ {"task": "Install OpenReplay", @@ -58,22 +57,18 @@ def get_state(tenant_id): def get_state_installing(tenant_id): - my_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) - pids = [s["projectId"] for s in my_projects] + pids = projects.get_projects_ids(tenant_id=tenant_id) with pg_client.PostgresClient() as cur: recorded = False if len(pids) > 0: cur.execute( - cur.mogrify("""\ - SELECT - COUNT(*) - FROM public.sessions AS s - where s.project_id IN %(ids)s - LIMIT 1;""", + cur.mogrify("""SELECT EXISTS(( SELECT 1 + FROM public.sessions AS s + WHERE s.project_id IN %(ids)s)) AS exists;""", {"ids": tuple(pids)}) ) - recorded = cur.fetchone()["count"] > 0 + recorded = cur.fetchone()["exists"] return {"task": "Install OpenReplay", "done": recorded, @@ -82,20 +77,23 @@ def get_state_installing(tenant_id): def get_state_identify_users(tenant_id): with pg_client.PostgresClient() as cur: - cur.execute( - """SELECT SUM((SELECT COUNT(t.meta) - FROM (VALUES (p.metadata_1), (p.metadata_2), (p.metadata_3), (p.metadata_4), (p.metadata_5), - (p.metadata_6), (p.metadata_7), (p.metadata_8), (p.metadata_9), (p.metadata_10), - (sessions.user_id)) AS t(meta) - WHERE t.meta NOTNULL)) - FROM public.projects AS p - LEFT JOIN LATERAL ( SELECT 'defined' - FROM public.sessions - WHERE sessions.project_id=p.project_id AND sessions.user_id IS NOT NULL - LIMIT 1) AS sessions(user_id) ON(TRUE) - WHERE p.deleted_at ISNULL;""") + cur.execute("""SELECT EXISTS((SELECT 1 + FROM public.projects AS p + LEFT JOIN LATERAL ( SELECT 1 + FROM public.sessions + WHERE sessions.project_id = p.project_id + AND sessions.user_id IS NOT NULL + LIMIT 1) AS sessions(user_id) ON (TRUE) + WHERE p.deleted_at ISNULL + AND ( sessions.user_id IS NOT NULL OR p.metadata_1 IS NOT NULL + OR p.metadata_2 IS NOT NULL OR p.metadata_3 IS NOT NULL + OR p.metadata_4 IS NOT NULL OR p.metadata_5 IS NOT NULL + OR p.metadata_6 IS NOT NULL OR p.metadata_7 IS NOT NULL + OR p.metadata_8 IS NOT NULL OR p.metadata_9 IS NOT NULL + OR p.metadata_10 IS NOT NULL ) + )) AS exists;""") - meta = cur.fetchone()["sum"] > 0 + meta = cur.fetchone()["exists"] return {"task": "Identify Users", "done": meta, diff --git a/api/chalicelib/core/projects.py b/api/chalicelib/core/projects.py index c5ae912aa..c57360dc4 100644 --- a/api/chalicelib/core/projects.py +++ b/api/chalicelib/core/projects.py @@ -280,3 +280,13 @@ def update_capture_status(project_id, changes): ) return changes + + +def get_projects_ids(tenant_id): + with pg_client.PostgresClient() as cur: + cur.execute(f"""SELECT s.project_id + FROM public.projects AS s + WHERE s.deleted_at IS NULL + ORDER BY s.project_id;""") + rows = cur.fetchall() + return [r["project_id"] for r in rows] diff --git a/ee/api/chalicelib/core/boarding.py b/ee/api/chalicelib/core/boarding.py index 6690e59f2..8a2076b58 100644 --- a/ee/api/chalicelib/core/boarding.py +++ b/ee/api/chalicelib/core/boarding.py @@ -6,41 +6,40 @@ from chalicelib.core import projects def get_state(tenant_id): - my_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) - pids = [s["projectId"] for s in my_projects] + pids = projects.get_projects_ids(tenant_id=tenant_id) with pg_client.PostgresClient() as cur: recorded = False meta = False if len(pids) > 0: cur.execute( - cur.mogrify("""\ - SELECT - COUNT(*) - FROM public.sessions AS s - where s.project_id IN %(ids)s - LIMIT 1;""", + cur.mogrify("""SELECT EXISTS(( SELECT 1 + FROM public.sessions AS s + WHERE s.project_id IN %(ids)s)) AS exists;""", {"ids": tuple(pids)}) ) - recorded = cur.fetchone()["count"] > 0 + recorded = cur.fetchone()["exists"] meta = False if recorded: cur.execute( - cur.mogrify("""SELECT SUM((SELECT COUNT(t.meta) - FROM (VALUES (p.metadata_1), (p.metadata_2), (p.metadata_3), (p.metadata_4), (p.metadata_5), - (p.metadata_6), (p.metadata_7), (p.metadata_8), (p.metadata_9), (p.metadata_10), - (sessions.user_id)) AS t(meta) - WHERE t.meta NOTNULL)) - FROM public.projects AS p - LEFT JOIN LATERAL ( SELECT 'defined' - FROM public.sessions - WHERE sessions.project_id=p.project_id AND sessions.user_id IS NOT NULL - LIMIT 1) AS sessions(user_id) ON(TRUE) - WHERE p.tenant_id = %(tenant_id)s - AND p.deleted_at ISNULL;""" + cur.mogrify("""SELECT EXISTS((SELECT 1 + FROM public.projects AS p + LEFT JOIN LATERAL ( SELECT 1 + FROM public.sessions + WHERE sessions.project_id = p.project_id + AND sessions.user_id IS NOT NULL + LIMIT 1) AS sessions(user_id) ON (TRUE) + WHERE p.tenant_id = %(tenant_id)s AND p.deleted_at ISNULL + AND ( sessions.user_id IS NOT NULL OR p.metadata_1 IS NOT NULL + OR p.metadata_2 IS NOT NULL OR p.metadata_3 IS NOT NULL + OR p.metadata_4 IS NOT NULL OR p.metadata_5 IS NOT NULL + OR p.metadata_6 IS NOT NULL OR p.metadata_7 IS NOT NULL + OR p.metadata_8 IS NOT NULL OR p.metadata_9 IS NOT NULL + OR p.metadata_10 IS NOT NULL ) + )) AS exists;""" , {"tenant_id": tenant_id})) - meta = cur.fetchone()["sum"] > 0 + meta = cur.fetchone()["exists"] return [ {"task": "Install OpenReplay", @@ -61,22 +60,18 @@ def get_state(tenant_id): def get_state_installing(tenant_id): - my_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) - pids = [s["projectId"] for s in my_projects] + pids = projects.get_projects_ids(tenant_id=tenant_id) with pg_client.PostgresClient() as cur: recorded = False if len(pids) > 0: cur.execute( - cur.mogrify("""\ - SELECT - COUNT(*) - FROM public.sessions AS s - where s.project_id IN %(ids)s - LIMIT 1;""", + cur.mogrify("""SELECT EXISTS(( SELECT 1 + FROM public.sessions AS s + WHERE s.project_id IN %(ids)s)) AS exists;""", {"ids": tuple(pids)}) ) - recorded = cur.fetchone()["count"] > 0 + recorded = cur.fetchone()["exists"] return {"task": "Install OpenReplay", "done": recorded, @@ -86,21 +81,24 @@ def get_state_installing(tenant_id): def get_state_identify_users(tenant_id): with pg_client.PostgresClient() as cur: cur.execute( - cur.mogrify("""SELECT SUM((SELECT COUNT(t.meta) - FROM (VALUES (p.metadata_1), (p.metadata_2), (p.metadata_3), (p.metadata_4), (p.metadata_5), - (p.metadata_6), (p.metadata_7), (p.metadata_8), (p.metadata_9), (p.metadata_10), - (sessions.user_id)) AS t(meta) - WHERE t.meta NOTNULL)) - FROM public.projects AS p - LEFT JOIN LATERAL ( SELECT 'defined' - FROM public.sessions - WHERE sessions.project_id=p.project_id AND sessions.user_id IS NOT NULL - LIMIT 1) AS sessions(user_id) ON(TRUE) - WHERE p.tenant_id = %(tenant_id)s - AND p.deleted_at ISNULL;""" + cur.mogrify("""SELECT EXISTS((SELECT 1 + FROM public.projects AS p + LEFT JOIN LATERAL ( SELECT 1 + FROM public.sessions + WHERE sessions.project_id = p.project_id + AND sessions.user_id IS NOT NULL + LIMIT 1) AS sessions(user_id) ON (TRUE) + WHERE p.tenant_id = %(tenant_id)s AND p.deleted_at ISNULL + AND ( sessions.user_id IS NOT NULL OR p.metadata_1 IS NOT NULL + OR p.metadata_2 IS NOT NULL OR p.metadata_3 IS NOT NULL + OR p.metadata_4 IS NOT NULL OR p.metadata_5 IS NOT NULL + OR p.metadata_6 IS NOT NULL OR p.metadata_7 IS NOT NULL + OR p.metadata_8 IS NOT NULL OR p.metadata_9 IS NOT NULL + OR p.metadata_10 IS NOT NULL ) + )) AS exists;""" , {"tenant_id": tenant_id})) - meta = cur.fetchone()["sum"] > 0 + meta = cur.fetchone()["exists"] return {"task": "Identify Users", "done": meta, diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index 75a3a31d0..0255c8c8c 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -324,7 +324,7 @@ def is_authorized_batch(project_ids, tenant_id): query = cur.mogrify("""\ SELECT project_id FROM public.projects - where tenant_id =%(tenant_id)s + WHERE tenant_id =%(tenant_id)s AND project_id IN %(project_ids)s AND deleted_at IS NULL;""", {"tenant_id": tenant_id, "project_ids": tuple(project_ids)}) @@ -334,3 +334,13 @@ def is_authorized_batch(project_ids, tenant_id): ) rows = cur.fetchall() return [r["project_id"] for r in rows] + + +def get_projects_ids(tenant_id): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify("""SELECT s.project_id + FROM public.projects AS s + WHERE tenant_id =%(tenant_id)s AND s.deleted_at IS NULL + ORDER BY s.project_id;""", {"tenant_id": tenant_id})) + rows = cur.fetchall() + return [r["project_id"] for r in rows] From 6360712aebe2c5f129693aaf890fa874e9883223 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 4 Apr 2022 15:21:55 +0200 Subject: [PATCH 032/125] feat(api): fixed update custom_metrics with missing index --- api/chalicelib/core/custom_metrics.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 839ba4d72..cc457a0e2 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -148,10 +148,11 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche "metric_value": data.metric_value, "metric_format": data.metric_format} for i, s in enumerate(data.series): prefix = "u_" + if s.index is None: + s.index = i if s.series_id is None or s.series_id not in series_ids: n_series.append({"i": i, "s": s}) prefix = "n_" - s.index = i else: u_series.append({"i": i, "s": s}) u_series_ids.append(s.series_id) @@ -198,9 +199,7 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche AND project_id = %(project_id)s AND (user_id = %(user_id)s OR is_public) RETURNING metric_id;""", params) - cur.execute( - query - ) + cur.execute(query) return get(metric_id=metric_id, project_id=project_id, user_id=user_id) From e75f7ddb4fb5e24dc7a820e352f3e4c2b6d73a1e Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 4 Apr 2022 16:36:51 +0200 Subject: [PATCH 033/125] feat(api): fixed sort sessions --- api/chalicelib/core/sessions.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 1903cc08b..8f619f66b 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -233,20 +233,19 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, e data.order = "DESC" sort = 'session_id' if data.sort is not None and data.sort != "session_id": - sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) - else: - sort = 'session_id' + # sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) + sort = helper.key_to_snake_case(data.sort) meta_keys = metadata.get(project_id=project_id) main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, COALESCE(JSONB_AGG(full_sessions) FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions - FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY issue_score DESC, {sort} {data.order}, session_id desc) AS rn + FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY {sort} {data.order}, issue_score DESC) AS rn FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} {query_part} ORDER BY s.session_id desc) AS filtred_sessions - ORDER BY issue_score DESC, {sort} {data.order}) AS full_sessions;""", + ORDER BY {sort} {data.order}, issue_score DESC) AS full_sessions;""", full_args) # print("--------------------") # print(main_query) @@ -280,9 +279,9 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, e for i, s in enumerate(sessions): sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ if sessions[i][f'metadata_{k["index"]}'] is not None} - if not data.group_by_user and data.sort is not None and data.sort != "session_id": - sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data.sort)], - reverse=data.order.upper() == "DESC") + # if not data.group_by_user and data.sort is not None and data.sort != "session_id": + # sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data.sort)], + # reverse=data.order.upper() == "DESC") return { 'total': total, 'sessions': helper.list_to_camel_case(sessions) From 011cbd006bb3ae5cc1556bb52fb6bf87b2dc8074 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Mon, 4 Apr 2022 20:40:10 +0200 Subject: [PATCH 034/125] fix(ackend-assets): use cachePath in timeout map instead of original url --- backend/services/assets/cacher/cacher.go | 47 ++++++++++---------- backend/services/assets/cacher/timeoutMap.go | 12 ++--- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/backend/services/assets/cacher/cacher.go b/backend/services/assets/cacher/cacher.go index 59b09449f..70ea31928 100644 --- a/backend/services/assets/cacher/cacher.go +++ b/backend/services/assets/cacher/cacher.go @@ -1,31 +1,31 @@ package cacher import ( + "crypto/tls" "fmt" "io" "io/ioutil" "mime" "net/http" - "crypto/tls" "path/filepath" "strings" "time" "github.com/pkg/errors" - - "openreplay/backend/pkg/url/assets" + "openreplay/backend/pkg/storage" + "openreplay/backend/pkg/url/assets" ) const MAX_CACHE_DEPTH = 5 type cacher struct { - timeoutMap *timeoutMap // Concurrency implemented - s3 *storage.S3 // AWS Docs: "These clients are safe to use concurrently." - httpClient *http.Client // Docs: "Clients are safe for concurrent use by multiple goroutines." - rewriter *assets.Rewriter // Read only - Errors chan error - sizeLimit int + timeoutMap *timeoutMap // Concurrency implemented + s3 *storage.S3 // AWS Docs: "These clients are safe to use concurrently." + httpClient *http.Client // Docs: "Clients are safe for concurrent use by multiple goroutines." + rewriter *assets.Rewriter // Read only + Errors chan error + sizeLimit int } func NewCacher(region string, bucket string, origin string, sizeLimit int) *cacher { @@ -36,26 +36,26 @@ func NewCacher(region string, bucket string, origin string, sizeLimit int) *cach httpClient: &http.Client{ Timeout: time.Duration(6) * time.Second, Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, }, - rewriter: rewriter, - Errors: make(chan error), - sizeLimit: sizeLimit, + rewriter: rewriter, + Errors: make(chan error), + sizeLimit: sizeLimit, } } func (c *cacher) cacheURL(requestURL string, sessionID uint64, depth byte, context string, isJS bool) { - if c.timeoutMap.contains(requestURL) { - return - } - c.timeoutMap.add(requestURL) var cachePath string - if (isJS) { + if isJS { cachePath = assets.GetCachePathForJS(requestURL) } else { cachePath = assets.GetCachePathForAssets(sessionID, requestURL) } + if c.timeoutMap.contains(cachePath) { + return + } + c.timeoutMap.add(cachePath) if c.s3.Exists(cachePath) { return } @@ -94,20 +94,19 @@ func (c *cacher) cacheURL(requestURL string, sessionID uint64, depth byte, conte if isCSS { strData = c.rewriter.RewriteCSS(sessionID, requestURL, strData) // TODO: one method for reqrite and return list } - - // TODO: implement in streams + + // TODO: implement in streams err = c.s3.Upload(strings.NewReader(strData), cachePath, contentType, false) if err != nil { c.Errors <- errors.Wrap(err, context) return } - c.timeoutMap.add(requestURL) if isCSS { if depth > 0 { for _, extractedURL := range assets.ExtractURLsFromCSS(string(data)) { - if fullURL, cachable := assets.GetFullCachableURL(requestURL, extractedURL); cachable { - go c.cacheURL(fullURL, sessionID, depth-1, context + "\n -> " + fullURL, false) + if fullURL, cachable := assets.GetFullCachableURL(requestURL, extractedURL); cachable { + go c.cacheURL(fullURL, sessionID, depth-1, context+"\n -> "+fullURL, false) } } if err != nil { diff --git a/backend/services/assets/cacher/timeoutMap.go b/backend/services/assets/cacher/timeoutMap.go index 36fc4ee4d..5a8e31424 100644 --- a/backend/services/assets/cacher/timeoutMap.go +++ b/backend/services/assets/cacher/timeoutMap.go @@ -5,30 +5,30 @@ import ( "time" ) -const MAX_STORAGE_TIME = 18 * time.Hour +const MAX_STORAGE_TIME = 24 * time.Hour // If problem with cache contention (>=4 core) look at sync.Map type timeoutMap struct { mx sync.RWMutex - m map[string]time.Time + m map[string]time.Time } func newTimeoutMap() *timeoutMap { return &timeoutMap{ m: make(map[string]time.Time), } -} +} func (tm *timeoutMap) add(key string) { tm.mx.Lock() - defer tm.mx.Unlock() + defer tm.mx.Unlock() tm.m[key] = time.Now() } func (tm *timeoutMap) contains(key string) bool { tm.mx.RLock() - defer tm.mx.RUnlock() + defer tm.mx.RUnlock() _, ok := tm.m[key] return ok } @@ -36,7 +36,7 @@ func (tm *timeoutMap) contains(key string) bool { func (tm *timeoutMap) deleteOutdated() { now := time.Now() tm.mx.Lock() - defer tm.mx.Unlock() + defer tm.mx.Unlock() for key, t := range tm.m { if now.Sub(t) > MAX_STORAGE_TIME { delete(tm.m, key) From ef4c9a5bf5def1ccc8f09e3054217725a37aa726 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 11:28:47 +0200 Subject: [PATCH 035/125] feat(db): added edited_at to metrics feat(db): defined all template metrics feat(db): support delta multi-execution feat(api): get edited_at with metric data feat(api): get owner_email with metric data feat(api): support of metrics areaChart --- api/chalicelib/core/custom_metrics.py | 15 +++- api/schemas.py | 1 + .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 87 ++++++++----------- .../db/init_dbs/postgresql/init_schema.sql | 1 + 4 files changed, 53 insertions(+), 51 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index cc457a0e2..51f8655a4 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -194,7 +194,8 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche SET name = %(name)s, is_public= %(is_public)s, view_type= %(view_type)s, metric_type= %(metric_type)s, metric_of= %(metric_of)s, metric_value= %(metric_value)s, - metric_format= %(metric_format)s + metric_format= %(metric_format)s, + edited_at = timezone('utc'::text, now()) WHERE metric_id = %(metric_id)s AND project_id = %(project_id)s AND (user_id = %(user_id)s OR is_public) @@ -224,6 +225,11 @@ def get_all(project_id, user_id, include_series=False): AND project_id = %(project_id)s AND ((user_id = %(user_id)s OR is_public))) AS connected_dashboards ) AS connected_dashboards ON (TRUE) + LEFT JOIN LATERAL (SELECT email AS owner_email + FROM users + WHERE deleted_at ISNULL + AND users.user_id = metrics.user_id + ) AS owner ON (TRUE) WHERE metrics.project_id = %(project_id)s AND metrics.deleted_at ISNULL AND (user_id = %(user_id)s OR metrics.is_public) @@ -246,7 +252,7 @@ def delete(project_id, metric_id, user_id): cur.execute( cur.mogrify("""\ UPDATE public.metrics - SET deleted_at = timezone('utc'::text, now()) + SET deleted_at = timezone('utc'::text, now()), edited_at = timezone('utc'::text, now()) WHERE project_id = %(project_id)s AND metric_id = %(metric_id)s AND (user_id = %(user_id)s OR is_public);""", @@ -274,6 +280,11 @@ def get(metric_id, project_id, user_id, flatten=True): AND project_id = %(project_id)s AND ((user_id = %(user_id)s OR is_public))) AS connected_dashboards ) AS connected_dashboards ON (TRUE) + LEFT JOIN LATERAL (SELECT email AS owner_email + FROM users + WHERE deleted_at ISNULL + AND users.user_id = metrics.user_id + ) AS owner ON (TRUE) WHERE metrics.project_id = %(project_id)s AND metrics.deleted_at ISNULL AND (metrics.user_id = %(user_id)s OR metrics.is_public) diff --git a/api/schemas.py b/api/schemas.py index b707a53e1..a7fd003b7 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -776,6 +776,7 @@ class CustomMetricCreateSeriesSchema(BaseModel): class MetricTimeseriesViewType(str, Enum): line_chart = "lineChart" progress = "progress" + area_chart = "areaChart" class MetricTableViewType(str, Enum): diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 3080f8dbb..d4d8575ae 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -18,50 +18,23 @@ CREATE TABLE IF NOT EXISTS dashboards deleted_at timestamp NULL DEFAULT NULL ); --- CREATE TABLE templates --- ( --- template_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, --- template_key text, --- name text NOT NULL, --- category text NOT NULL, --- series jsonb NOT NULL, --- config jsonb NOT NULL, --- predefined boolean DEFAULT TRUE --- ); - --- CREATE TABLE dashboard_widgets --- ( --- widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, --- dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, --- metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, --- -- template_id integer NOT NULL REFERENCES templates (template_id) ON DELETE CASCADE, --- user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, --- created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), --- configuration jsonb NOT NULL DEFAULT '{}'::jsonb, --- name text --- ); - --- INSERT INTO public.templates (name, category, series, config, predefined, template_key) --- VALUES ('captured sessions', 'overview', '[]', '{}', true, 'count_sessions'), --- ('request load time', 'overview', '[]', '{}', true, 'avg_request_load_time'), --- ('page load time', 'overview', '[]', '{}', true, 'avg_page_load_time'), --- ('image load time', 'overview', '[]', '{}', true, 'avg_image_load_time'); - -ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; ALTER TABLE IF EXISTS metrics - DROP CONSTRAINT IF EXISTS null_project_id_for_template_only; + DROP CONSTRAINT IF EXISTS null_project_id_for_template_only, + DROP CONSTRAINT IF EXISTS unique_key; ALTER TABLE IF EXISTS metrics - ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', - ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS key text NULL DEFAULT NULL, - ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', + ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS key text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{}'::jsonb, ALTER COLUMN project_id DROP NOT NULL, ADD CONSTRAINT null_project_id_for_template_only - CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ); + CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), + ADD CONSTRAINT unique_key UNIQUE (key); @@ -76,17 +49,33 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets name text ); --- INSERT INTO public.templates (name, category, series, config, predefined, template_key) --- VALUES ('captured sessions', 'overview', '[]', '{}', true, 'count_sessions'), --- ('request load time', 'overview', '[]', '{}', true, 'avg_request_load_time'), --- ('page load time', 'overview', '[]', '{}', true, 'avg_page_load_time'), --- ('image load time', 'overview', '[]', '{}', true, 'avg_image_load_time'); INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) -VALUES ('captured sessions', 'overview', '{}', true, true, true, 'count_sessions'), - ('request load time', 'overview', '{}', true, true, true, 'avg_request_load_time'), - ('page load time', 'overview', '{}', true, true, true, 'avg_page_load_time'), - ('image load time', 'overview', '{}', true, true, true, 'avg_image_load_time'); +VALUES ('sessions count', 'overview', '{}', true, true, true, 'count_sessions'), + ('avg request load time', 'overview', '{}', true, true, true, 'avg_request_load_time'), + ('avg page load time', 'overview', '{}', true, true, true, 'avg_page_load_time'), + ('avg image load time', 'overview', '{}', true, true, true, 'avg_image_load_time'), + ('avg dom content load start', 'overview', '{}', true, true, true, 'avg_dom_content_load_start'), + ('avg first contentful pixel', 'overview', '{}', true, true, true, 'avg_first_contentful_pixel'), + ('avg visited pages count', 'overview', '{}', true, true, true, 'avg_visited_pages'), + ('avg session duration', 'overview', '{}', true, true, true, 'avg_session_duration'), + ('avg pages dom buildtime', 'overview', '{}', true, true, true, 'avg_pages_dom_buildtime'), + ('avg pages response time', 'overview', '{}', true, true, true, 'avg_pages_response_time'), + ('avg response time', 'overview', '{}', true, true, true, 'avg_response_time'), + ('avg first paint', 'overview', '{}', true, true, true, 'avg_first_paint'), + ('avg dom content loaded', 'overview', '{}', true, true, true, 'avg_dom_content_loaded'), + ('avg time till first bit', 'overview', '{}', true, true, true, 'avg_till_first_bit'), + ('avg time to interactive', 'overview', '{}', true, true, true, 'avg_time_to_interactive'), + ('requests count', 'overview', '{}', true, true, true, 'count_requests'), + ('avg time to render', 'overview', '{}', true, true, true, 'avg_time_to_render'), + ('avg used js heap size', 'overview', '{}', true, true, true, 'avg_used_js_heap_size'), + ('avg cpu', 'overview', '{}', true, true, true, 'avg_cpu') +ON CONFLICT (key) DO UPDATE SET name=excluded.name, + category=excluded.category, + config=excluded.config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public; - -COMMIT \ No newline at end of file +COMMIT; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 35926bc96..c6f628e32 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -947,6 +947,7 @@ $$ active boolean NOT NULL DEFAULT TRUE, created_at timestamp default timezone('utc'::text, now()) not null, deleted_at timestamp, + edited_at timestamp, metric_type metric_type NOT NULL DEFAULT 'timeseries', view_type metric_view_type NOT NULL DEFAULT 'lineChart', metric_of text NOT NULL DEFAULT 'sessionCount', From 92fb75b228ae5e6cf32c635bb10a8c9c61a341c9 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 14:06:30 +0200 Subject: [PATCH 036/125] feat(db): include default metrics in init_schema.sql feat(api): refactored overview --- api/build_local.sh | 3 --- api/routers/subs/dashboard.py | 6 +++-- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 2 +- .../db/init_dbs/postgresql/init_schema.sql | 27 +++++++++++++++++++ 4 files changed, 32 insertions(+), 6 deletions(-) delete mode 100644 api/build_local.sh diff --git a/api/build_local.sh b/api/build_local.sh deleted file mode 100644 index 4253fd812..000000000 --- a/api/build_local.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -docker build -f ./Dockerfile --build-arg envarg="default-foss" -t chalice:local . \ No newline at end of file diff --git a/api/routers/subs/dashboard.py b/api/routers/subs/dashboard.py index fdb5d641a..0832be4b8 100644 --- a/api/routers/subs/dashboard.py +++ b/api/routers/subs/dashboard.py @@ -388,8 +388,10 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body "data": dashboard.get_time_to_render(project_id=projectId, **data.dict())}, {"key": schemas.TemplateKeys.avg_used_js_heap_size, "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_cpu, "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_fps, "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} + {"key": schemas.TemplateKeys.avg_cpu, + "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, + {"key": schemas.TemplateKeys.avg_fps, + "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} ] results = sorted(results, key=lambda r: r["key"]) return {"data": results} diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index d4d8575ae..6c563bfdc 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -59,7 +59,7 @@ VALUES ('sessions count', 'overview', '{}', true, true, true, 'count_sessions'), ('avg first contentful pixel', 'overview', '{}', true, true, true, 'avg_first_contentful_pixel'), ('avg visited pages count', 'overview', '{}', true, true, true, 'avg_visited_pages'), ('avg session duration', 'overview', '{}', true, true, true, 'avg_session_duration'), - ('avg pages dom buildtime', 'overview', '{}', true, true, true, 'avg_pages_dom_buildtime'), + ('avg pages dom build time', 'overview', '{}', true, true, true, 'avg_pages_dom_buildtime'), ('avg pages response time', 'overview', '{}', true, true, true, 'avg_pages_response_time'), ('avg response time', 'overview', '{}', true, true, true, 'avg_response_time'), ('avg first paint', 'overview', '{}', true, true, true, 'avg_first_paint'), diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index c6f628e32..5b5bd7044 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1025,4 +1025,31 @@ $$ $$ LANGUAGE plpgsql; +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) +VALUES ('sessions count', 'overview', '{}', true, true, true, 'count_sessions'), + ('avg request load time', 'overview', '{}', true, true, true, 'avg_request_load_time'), + ('avg page load time', 'overview', '{}', true, true, true, 'avg_page_load_time'), + ('avg image load time', 'overview', '{}', true, true, true, 'avg_image_load_time'), + ('avg dom content load start', 'overview', '{}', true, true, true, 'avg_dom_content_load_start'), + ('avg first contentful pixel', 'overview', '{}', true, true, true, 'avg_first_contentful_pixel'), + ('avg visited pages count', 'overview', '{}', true, true, true, 'avg_visited_pages'), + ('avg session duration', 'overview', '{}', true, true, true, 'avg_session_duration'), + ('avg pages dom build time', 'overview', '{}', true, true, true, 'avg_pages_dom_buildtime'), + ('avg pages response time', 'overview', '{}', true, true, true, 'avg_pages_response_time'), + ('avg response time', 'overview', '{}', true, true, true, 'avg_response_time'), + ('avg first paint', 'overview', '{}', true, true, true, 'avg_first_paint'), + ('avg dom content loaded', 'overview', '{}', true, true, true, 'avg_dom_content_loaded'), + ('avg time till first bit', 'overview', '{}', true, true, true, 'avg_till_first_bit'), + ('avg time to interactive', 'overview', '{}', true, true, true, 'avg_time_to_interactive'), + ('requests count', 'overview', '{}', true, true, true, 'count_requests'), + ('avg time to render', 'overview', '{}', true, true, true, 'avg_time_to_render'), + ('avg used js heap size', 'overview', '{}', true, true, true, 'avg_used_js_heap_size'), + ('avg cpu', 'overview', '{}', true, true, true, 'avg_cpu') +ON CONFLICT (key) DO UPDATE SET name=excluded.name, + category=excluded.category, + config=excluded.config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public; + COMMIT; \ No newline at end of file From effe39d36d7a24bcdeafff1766676efed0abea7d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 17:29:29 +0200 Subject: [PATCH 037/125] feat(api): pg helper support application name feat(api): removed widget name feat(db): removed widget name feat(api): changed edit dashboard to support list of metrics feat(api): get metrics/widgets cast created_at feat(api): get metrics/widgets cast edited_at feat(api): create dashboard with metrics, support default config feat(api): create dashboard with metrics, support widget position feat(api): edit dashboard with metrics, support default config feat(api): edit dashboard with metrics, support widget position --- api/Dockerfile | 1 + api/Dockerfile.alerts | 1 + api/chalicelib/core/custom_metrics.py | 7 +++- api/chalicelib/core/dashboards2.py | 39 +++++++++++++----- api/chalicelib/utils/pg_client.py | 5 ++- api/schemas.py | 14 +++---- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 41 +++++++++---------- .../db/init_dbs/postgresql/init_schema.sql | 38 ++++++++--------- 8 files changed, 84 insertions(+), 62 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 780518ff3..4526c32bd 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /work COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env +ENV APP_NAME chalice # Add Tini # Startup daemon diff --git a/api/Dockerfile.alerts b/api/Dockerfile.alerts index ed8f06eac..bdd1772ba 100644 --- a/api/Dockerfile.alerts +++ b/api/Dockerfile.alerts @@ -6,6 +6,7 @@ COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env && mv app_alerts.py app.py ENV pg_minconn 2 +ENV APP_NAME alerts # Add Tini # Startup daemon diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 51f8655a4..6ca72f453 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -233,7 +233,7 @@ def get_all(project_id, user_id, include_series=False): WHERE metrics.project_id = %(project_id)s AND metrics.deleted_at ISNULL AND (user_id = %(user_id)s OR metrics.is_public) - ORDER BY created_at;""", + ORDER BY metrics.edited_at, metrics.created_at;""", {"project_id": project_id, "user_id": user_id} ) ) @@ -243,6 +243,10 @@ def get_all(project_id, user_id, include_series=False): # r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) for s in r["series"]: s["filter"] = helper.old_search_payload_to_flat(s["filter"]) + else: + for r in rows: + r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r["edited_at"] = TimeUTC.datetime_to_timestamp(r["edited_at"]) rows = helper.list_to_camel_case(rows) return rows @@ -297,6 +301,7 @@ def get(metric_id, project_id, user_id, flatten=True): if row is None: return None row["created_at"] = TimeUTC.datetime_to_timestamp(row["created_at"]) + row["edited_at"] = TimeUTC.datetime_to_timestamp(row["edited_at"]) if flatten: for s in row["series"]: s["filter"] = helper.old_search_payload_to_flat(s["filter"]) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 7e06f52ac..c0bb3017f 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -32,11 +32,15 @@ def create_dashboard(project_id, user_id, data: schemas.CreateDashboardSchema): params = {"userId": user_id, "projectId": project_id, **data.dict()} if data.metrics is not None and len(data.metrics) > 0: pg_query = f"""WITH dash AS ({pg_query}) - INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id) - VALUES {",".join([f"((SELECT dashboard_id FROM dash),%(metric_id_{i})s, %(userId)s)" for i in range(len(data.metrics))])} + INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) + VALUES {",".join([f"((SELECT dashboard_id FROM dash),%(metric_id_{i})s, %(userId)s, %(config_{i})s)" for i in range(len(data.metrics))])} RETURNING (SELECT dashboard_id FROM dash)""" - for i, m in enumerate(data.metrics): - params[f"metric_id_{i}"] = m + for i, m in enumerate(data.metrics): + params[f"metric_id_{i}"] = m + params[f"config_{i}"] = schemas.AddWidgetToDashboardPayloadSchema.schema() \ + .get("properties", {}).get("config", {}).get("default", {}) + params[f"config_{i}"]["position"] = i + params[f"config_{i}"] = json.dumps(params[f"config_{i}"]) cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() if row is None: @@ -98,16 +102,29 @@ def delete_dashboard(project_id, user_id, dashboard_id): def update_dashboard(project_id, user_id, dashboard_id, data: schemas.EditDashboardSchema): with pg_client.PostgresClient() as cur: - pg_query = """UPDATE dashboards - SET name = %(name)s, is_public = %(is_public)s + pg_query = f"""UPDATE dashboards + SET name = %(name)s + {", is_public = %(is_public)s" if data.is_public is not None else ""} + {", is_pinned = %(is_pinned)s" if data.is_pinned is not None else ""} WHERE dashboards.project_id = %(projectId)s AND dashboard_id = %(dashboard_id)s - AND (dashboards.user_id = %(userId)s OR is_public) - RETURNING *;""" + AND (dashboards.user_id = %(userId)s OR is_public)""" params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, **data.dict()} + if data.metrics is not None and len(data.metrics) > 0: + pg_query = f"""WITH dash AS ({pg_query}) + INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) + VALUES {",".join([f"(%(dashboard_id)s, %(metric_id_{i})s, %(userId)s, %(config_{i})s)" for i in range(len(data.metrics))])} + RETURNING (SELECT dashboard_id FROM dash)""" + for i, m in enumerate(data.metrics): + params[f"metric_id_{i}"] = m + params[f"config_{i}"] = schemas.AddWidgetToDashboardPayloadSchema.schema() \ + .get("properties", {}).get("config", {}).get("default", {}) + params[f"config_{i}"]["position"] = i + params[f"config_{i}"] = json.dumps(params[f"config_{i}"]) + cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - return helper.dict_to_camel_case(row) + + return get_dashboard(project_id=project_id, user_id=user_id, dashboard_id=dashboard_id) def get_widget(project_id, user_id, dashboard_id, widget_id): @@ -154,7 +171,7 @@ def add_widget(project_id, user_id, dashboard_id, data: schemas.AddWidgetToDashb def update_widget(project_id, user_id, dashboard_id, widget_id, data: schemas.AddWidgetToDashboardPayloadSchema): with pg_client.PostgresClient() as cur: pg_query = """UPDATE dashboard_widgets - SET name= %(name)s, config= %(config)s + SET config= %(config)s WHERE dashboard_id=%(dashboard_id)s AND widget_id=%(widget_id)s RETURNINIG *;""" params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, diff --git a/api/chalicelib/utils/pg_client.py b/api/chalicelib/utils/pg_client.py index 6e4118689..6379bad1e 100644 --- a/api/chalicelib/utils/pg_client.py +++ b/api/chalicelib/utils/pg_client.py @@ -9,7 +9,8 @@ _PG_CONFIG = {"host": config("pg_host"), "database": config("pg_dbname"), "user": config("pg_user"), "password": config("pg_password"), - "port": config("pg_port", cast=int)} + "port": config("pg_port", cast=int), + "application_name": config("APP_NAME", default="PY")} PG_CONFIG = dict(_PG_CONFIG) if config("pg_timeout", cast=int, default=0) > 0: PG_CONFIG["options"] = f"-c statement_timeout={config('pg_timeout', cast=int) * 1000}" @@ -64,6 +65,8 @@ class PostgresClient: def __init__(self, long_query=False): self.long_query = long_query if long_query: + long_config = dict(_PG_CONFIG) + long_config["application_name"] += "-LONG" self.connection = psycopg2.connect(**_PG_CONFIG) else: self.connection = postgreSQL_pool.getconn() diff --git a/api/schemas.py b/api/schemas.py index a7fd003b7..1f1bec739 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -888,18 +888,15 @@ class CreateDashboardSchema(BaseModel): alias_generator = attribute_to_camel_case -class EditDashboardSchema(BaseModel): - name: str = Field(..., min_length=1) - is_public: bool = Field(default=False) - - class Config: - alias_generator = attribute_to_camel_case +class EditDashboardSchema(CreateDashboardSchema): + is_public: Optional[bool] = Field(default=None) + is_pinned: Optional[bool] = Field(default=None) class AddWidgetToDashboardPayloadSchema(BaseModel): metric_id: int = Field(default=None) - name: Optional[str] = Field(default=None) - config: dict = Field(default={}) + # if you change the config attribute name, please make sure to update it in dashboard2.py + config: dict = Field(default={"col": 1, "row": 1, "position": 0}) class Config: alias_generator = attribute_to_camel_case @@ -929,7 +926,6 @@ class TemplateKeys(str, Enum): avg_fps = "avg_fps" -# class CustomMetricAndTemplate(CreateCustomMetricsSchema): class CustomMetricAndTemplate(BaseModel): is_template: bool = Field(...) project_id: Optional[int] = Field(...) diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 6c563bfdc..1a3a918ed 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -45,31 +45,30 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - config jsonb NOT NULL DEFAULT '{}'::jsonb, - name text + config jsonb NOT NULL DEFAULT '{}'::jsonb ); INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) -VALUES ('sessions count', 'overview', '{}', true, true, true, 'count_sessions'), - ('avg request load time', 'overview', '{}', true, true, true, 'avg_request_load_time'), - ('avg page load time', 'overview', '{}', true, true, true, 'avg_page_load_time'), - ('avg image load time', 'overview', '{}', true, true, true, 'avg_image_load_time'), - ('avg dom content load start', 'overview', '{}', true, true, true, 'avg_dom_content_load_start'), - ('avg first contentful pixel', 'overview', '{}', true, true, true, 'avg_first_contentful_pixel'), - ('avg visited pages count', 'overview', '{}', true, true, true, 'avg_visited_pages'), - ('avg session duration', 'overview', '{}', true, true, true, 'avg_session_duration'), - ('avg pages dom build time', 'overview', '{}', true, true, true, 'avg_pages_dom_buildtime'), - ('avg pages response time', 'overview', '{}', true, true, true, 'avg_pages_response_time'), - ('avg response time', 'overview', '{}', true, true, true, 'avg_response_time'), - ('avg first paint', 'overview', '{}', true, true, true, 'avg_first_paint'), - ('avg dom content loaded', 'overview', '{}', true, true, true, 'avg_dom_content_loaded'), - ('avg time till first bit', 'overview', '{}', true, true, true, 'avg_till_first_bit'), - ('avg time to interactive', 'overview', '{}', true, true, true, 'avg_time_to_interactive'), - ('requests count', 'overview', '{}', true, true, true, 'count_requests'), - ('avg time to render', 'overview', '{}', true, true, true, 'avg_time_to_render'), - ('avg used js heap size', 'overview', '{}', true, true, true, 'avg_used_js_heap_size'), - ('avg cpu', 'overview', '{}', true, true, true, 'avg_cpu') +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu') ON CONFLICT (key) DO UPDATE SET name=excluded.name, category=excluded.category, config=excluded.config, diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 5b5bd7044..5114d2433 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1026,25 +1026,25 @@ $$ LANGUAGE plpgsql; INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) -VALUES ('sessions count', 'overview', '{}', true, true, true, 'count_sessions'), - ('avg request load time', 'overview', '{}', true, true, true, 'avg_request_load_time'), - ('avg page load time', 'overview', '{}', true, true, true, 'avg_page_load_time'), - ('avg image load time', 'overview', '{}', true, true, true, 'avg_image_load_time'), - ('avg dom content load start', 'overview', '{}', true, true, true, 'avg_dom_content_load_start'), - ('avg first contentful pixel', 'overview', '{}', true, true, true, 'avg_first_contentful_pixel'), - ('avg visited pages count', 'overview', '{}', true, true, true, 'avg_visited_pages'), - ('avg session duration', 'overview', '{}', true, true, true, 'avg_session_duration'), - ('avg pages dom build time', 'overview', '{}', true, true, true, 'avg_pages_dom_buildtime'), - ('avg pages response time', 'overview', '{}', true, true, true, 'avg_pages_response_time'), - ('avg response time', 'overview', '{}', true, true, true, 'avg_response_time'), - ('avg first paint', 'overview', '{}', true, true, true, 'avg_first_paint'), - ('avg dom content loaded', 'overview', '{}', true, true, true, 'avg_dom_content_loaded'), - ('avg time till first bit', 'overview', '{}', true, true, true, 'avg_till_first_bit'), - ('avg time to interactive', 'overview', '{}', true, true, true, 'avg_time_to_interactive'), - ('requests count', 'overview', '{}', true, true, true, 'count_requests'), - ('avg time to render', 'overview', '{}', true, true, true, 'avg_time_to_render'), - ('avg used js heap size', 'overview', '{}', true, true, true, 'avg_used_js_heap_size'), - ('avg cpu', 'overview', '{}', true, true, true, 'avg_cpu') +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu') ON CONFLICT (key) DO UPDATE SET name=excluded.name, category=excluded.category, config=excluded.config, From 6685eb1ac9d53d0c56815f9bd4d4e74b6664d203 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 18:10:19 +0200 Subject: [PATCH 038/125] feat(api): fixed create custom metric and add to dashboard feat(api): EE refactored sub-router feat(api): EE upgrade fastapi feat(api): EE refactored metrics feat(api): EE split overview --- api/chalicelib/core/dashboards2.py | 4 +- ee/api/.gitignore | 9 +- ee/api/Dockerfile | 1 + ee/api/Dockerfile.alerts | 1 + ee/api/_clickhouse_upgrade.sh | 10 - ee/api/app.py | 4 +- ee/api/chalicelib/core/dashboard.py | 708 +++++++++++++++++----- ee/api/requirements.txt | 2 +- ee/api/routers/{app => subs}/v1_api_ee.py | 0 9 files changed, 579 insertions(+), 160 deletions(-) delete mode 100644 ee/api/_clickhouse_upgrade.sh rename ee/api/routers/{app => subs}/v1_api_ee.py (100%) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index c0bb3017f..3efd1e968 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -153,9 +153,9 @@ def get_widget(project_id, user_id, dashboard_id, widget_id): def add_widget(project_id, user_id, dashboard_id, data: schemas.AddWidgetToDashboardPayloadSchema): with pg_client.PostgresClient() as cur: - pg_query = """INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config, name) + pg_query = """INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) SELECT %(dashboard_id)s AS dashboard_id, %(metric_id)s AS metric_id, - %(userId)s AS user_id, %(config)s::jsonb AS config, %(name)s AS name + %(userId)s AS user_id, %(config)s::jsonb AS config WHERE EXISTS(SELECT 1 FROM dashboards WHERE dashboards.deleted_at ISNULL AND dashboards.project_id = %(projectId)s AND dashboard_id = %(dashboard_id)s diff --git a/ee/api/.gitignore b/ee/api/.gitignore index f1ff9550b..af91e1919 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -180,9 +180,6 @@ Pipfile /chalicelib/core/alerts.py /chalicelib/core/alerts_processor.py /chalicelib/core/announcements.py -/chalicelib/blueprints/bp_app_api.py -/chalicelib/blueprints/bp_core.py -/chalicelib/blueprints/bp_core_crons.py /chalicelib/core/collaboration_slack.py /chalicelib/core/errors_favorite_viewed.py /chalicelib/core/events.py @@ -237,7 +234,6 @@ Pipfile /chalicelib/utils/smtp.py /chalicelib/utils/strings.py /chalicelib/utils/TimeUTC.py -/chalicelib/blueprints/app/__init__.py /routers/app/__init__.py /routers/crons/__init__.py /routers/subs/__init__.py @@ -245,7 +241,6 @@ Pipfile /chalicelib/core/assist.py /auth/auth_apikey.py /auth/auth_jwt.py -/chalicelib/blueprints/subs/bp_insights.py /build.sh /routers/core.py /routers/crons/core_crons.py @@ -257,10 +252,10 @@ Pipfile /chalicelib/core/heatmaps.py /routers/subs/insights.py /schemas.py -/chalicelib/blueprints/app/v1_api.py -/routers/app/v1_api.py /chalicelib/core/custom_metrics.py /chalicelib/core/performance_event.py /chalicelib/core/saved_search.py /app_alerts.py /build_alerts.sh +/routers/subs/metrics.py +/routers/subs/v1_api.py diff --git a/ee/api/Dockerfile b/ee/api/Dockerfile index cca6e6806..ee88ee22c 100644 --- a/ee/api/Dockerfile +++ b/ee/api/Dockerfile @@ -6,6 +6,7 @@ WORKDIR /work COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env +ENV APP_NAME chalice # Add Tini # Startup daemon diff --git a/ee/api/Dockerfile.alerts b/ee/api/Dockerfile.alerts index 9be6ebc93..230514918 100644 --- a/ee/api/Dockerfile.alerts +++ b/ee/api/Dockerfile.alerts @@ -7,6 +7,7 @@ COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env && mv app_alerts.py app.py ENV pg_minconn 2 +ENV APP_NAME alerts # Add Tini # Startup daemon diff --git a/ee/api/_clickhouse_upgrade.sh b/ee/api/_clickhouse_upgrade.sh deleted file mode 100644 index 9b656a584..000000000 --- a/ee/api/_clickhouse_upgrade.sh +++ /dev/null @@ -1,10 +0,0 @@ -sudo yum update -sudo yum install yum-utils -sudo rpm --import https://repo.clickhouse.com/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/stable/x86_64 -sudo yum update -sudo service clickhouse-server restart - - -#later mus use in clickhouse-client: -#SET allow_experimental_window_functions = 1; \ No newline at end of file diff --git a/ee/api/app.py b/ee/api/app.py index deae9d78d..ed2c01aa4 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -11,10 +11,10 @@ from starlette.responses import StreamingResponse, JSONResponse from chalicelib.utils import helper from chalicelib.utils import pg_client from routers import core, core_dynamic, ee, saml -from routers.app import v1_api, v1_api_ee +from routers.subs import v1_api from routers.crons import core_crons from routers.crons import core_dynamic_crons -from routers.subs import dashboard, insights +from routers.subs import dashboard, insights, v1_api_ee app = FastAPI() diff --git a/ee/api/chalicelib/core/dashboard.py b/ee/api/chalicelib/core/dashboard.py index c5c373c78..c834c852b 100644 --- a/ee/api/chalicelib/core/dashboard.py +++ b/ee/api/chalicelib/core/dashboard.py @@ -169,7 +169,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) ch_query = f"""\ SELECT toUnixTimestamp(toStartOfInterval(sessions.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(sessions.session_id) AS count + COUNT(sessions.session_id) AS value FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -181,19 +181,17 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) rows = ch.execute(query=ch_query, params=params) results = { - "count": sum([r["count"] for r in rows]), + "value": sum([r["value"] for r in rows]), "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, - neutral={"count": 0}) + neutral={"value": 0}) } diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff - ch_query = f"""\ - SELECT - COUNT(sessions.session_id) AS count + ch_query = f""" SELECT COUNT(sessions.session_id) AS count FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)};""" params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, @@ -203,7 +201,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) count = count[0]["count"] - results["countProgress"] = helper.__progress(old_val=count, new_val=results["count"]) + results["progress"] = helper.__progress(old_val=count, new_val=results["value"]) return results @@ -222,9 +220,8 @@ def get_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimesta with ch_client.ClickHouseClient() as ch: ch_query = f"""\ - SELECT - toUnixTimestamp(toStartOfInterval(errors.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(DISTINCT errors.session_id) AS count + SELECT toUnixTimestamp(toStartOfInterval(errors.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, + COUNT(DISTINCT errors.session_id) AS count FROM errors {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -304,9 +301,8 @@ def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), errors = {} for error_id in error_ids: ch_query = f"""\ - SELECT - toUnixTimestamp(toStartOfInterval(errors.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(errors.session_id) AS count + SELECT toUnixTimestamp(toStartOfInterval(errors.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, + COUNT(errors.session_id) AS count FROM errors {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -348,10 +344,8 @@ def __get_page_metrics(ch, project_id, startTimestamp, endTimestamp, **args): ch_sub_query += meta_condition # changed dom_content_loaded_event_start to dom_content_loaded_event_end ch_query = f"""\ - SELECT - COALESCE(AVG(NULLIF(pages.dom_content_loaded_event_end ,0)),0) AS avg_dom_content_load_start, --- COALESCE(AVG(NULLIF(pages.dom_content_loaded_event_start ,0)),0) AS avg_dom_content_load_start, - COALESCE(AVG(NULLIF(pages.first_contentful_paint,0)),0) AS avg_first_contentful_pixel + SELECT COALESCE(AVG(NULLIF(pages.dom_content_loaded_event_end ,0)),0) AS avg_dom_content_load_start, + COALESCE(AVG(NULLIF(pages.first_contentful_paint,0)),0) AS avg_first_contentful_pixel FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)};""" params = {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, @@ -395,8 +389,7 @@ def __get_application_activity(ch, project_id, startTimestamp, endTimestamp, **a ch_sub_query += meta_condition ch_sub_query.append("resources.type= %(type)s") ch_query = f"""\ - SELECT - AVG(NULLIF(resources.duration,0)) AS avg + SELECT AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)};""" row = ch.execute(query=ch_query, @@ -443,9 +436,8 @@ def __get_user_activity(cur, project_id, startTimestamp, endTimestamp, **args): ch_sub_query += meta_condition ch_query = f"""\ - SELECT - COALESCE(CEIL(AVG(NULLIF(sessions.pages_count,0))),0) AS avg_visited_pages, - COALESCE(AVG(NULLIF(sessions.duration,0)),0) AS avg_session_duration + SELECT COALESCE(CEIL(AVG(NULLIF(sessions.pages_count,0))),0) AS avg_visited_pages, + COALESCE(AVG(NULLIF(sessions.duration,0)),0) AS avg_session_duration FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)};""" params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, @@ -471,8 +463,8 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT resources.url, - AVG(NULLIF(resources.duration,0)) AS avg, - COUNT(resources.session_id) AS count + AVG(NULLIF(resources.duration,0)) AS avg, + COUNT(resources.session_id) AS count FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY resources.url ORDER BY avg DESC LIMIT 10;""" @@ -489,7 +481,7 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), for url in urls: ch_query = f"""\ SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(resources.duration,0)) AS avg + AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -544,9 +536,8 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(resources.duration,0)) AS avg + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} AND resources.type = 'img' @@ -558,9 +549,8 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi __complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, neutral={"avg": 0})] - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(resources.duration,0)) AS avg + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} AND resources.type = 'fetch' @@ -577,9 +567,8 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi data=args) ch_sub_query_chart += meta_condition - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(pages.load_event_end ,0)) AS avg + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(pages.load_event_end ,0)) AS avg FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} @@ -648,9 +637,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, if resource_type == "ALL" and not pages_only and not events_only: ch_sub_query.append("positionUTF8(url_hostpath,%(value)s)!=0") with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - arrayJoin(arraySlice(arrayReverseSort(arrayDistinct(groupArray(url))), 1, 5)) AS value, - type AS key + ch_query = f"""SELECT arrayJoin(arraySlice(arrayReverseSort(arrayDistinct(groupArray(url))), 1, 5)) AS value, + type AS key FROM resources WHERE {" AND ".join(ch_sub_query)} GROUP BY type @@ -685,9 +673,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, ch_sub_query.append(f"resources.type = '{__get_resource_db_type_from_type(resource_type)}'") with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - DISTINCT url_hostpath AS value, - %(resource_type)s AS key + ch_query = f"""SELECT DISTINCT url_hostpath AS value, + %(resource_type)s AS key FROM resources WHERE {" AND ".join(ch_sub_query)} LIMIT 10;""" @@ -787,34 +774,6 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, return [helper.dict_to_camel_case(row) for row in rows] -# def frustration_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1), -# endTimestamp=TimeUTC.now(), **args): -# with pg_client.PostgresClient() as cur: -# sub_q = "" -# if platform == 'mobile': -# sub_q = "AND s.user_device_type = 'mobile' AND s.project_id = %(project_id)s AND s.start_ts >= %(startTimestamp)s AND s.start_ts < %(endTimestamp)s" -# elif platform == 'desktop': -# sub_q = "AND s.user_device_type = 'desktop' AND s.project_id = %(project_id)s AND s.start_ts >= %(startTimestamp)s AND s.start_ts < %(endTimestamp)s" -# -# cur.execute(cur.mogrify(f"""\ -# SELECT s.project_id, -# s.session_id::text AS session_id, -# s.* -# FROM public.sessions AS s -# LEFT JOIN public.session_watchdogs AS sw ON s.session_id=sw.session_id -# LEFT JOIN public.watchdogs AS w ON w.watchdog_id=sw.watchdog_id -# WHERE s.project_id = %(project_id)s -# AND w.type='clickrage' -# AND s.start_ts>=%(startTimestamp)s -# AND s.start_ts<=%(endTimestamp)s -# {sub_q} -# ORDER BY s.session_id DESC -# LIMIT 5;""", -# {"project_id": project_id, "startTimestamp": startTimestamp, -# "endTimestamp": endTimestamp})) -# return helper.list_to_camel_case(cur.fetchall()) - - def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -826,9 +785,8 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day ch_sub_query += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - resources.url_hostpath AS key, - COUNT(resources.session_id) AS doc_count + ch_query = f"""SELECT resources.url_hostpath AS key, + COUNT(resources.session_id) AS doc_count FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY url_hostpath @@ -841,10 +799,9 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day if len(rows) == 0: return [] ch_sub_query.append("resources.url_hostpath = %(value)s") - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COUNT(resources.session_id) AS doc_count, - toUnixTimestamp(MAX(resources.datetime))*1000 AS max_datatime + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + COUNT(resources.session_id) AS doc_count, + toUnixTimestamp(MAX(resources.datetime))*1000 AS max_datatime FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY timestamp @@ -879,9 +836,8 @@ def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - resources.url_hostpath, COUNT(resources.session_id) AS doc_count + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + resources.url_hostpath, COUNT(resources.session_id) AS doc_count FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp, resources.url_hostpath @@ -935,9 +891,8 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(resources.duration,0)) AS avg + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -969,9 +924,8 @@ def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=- ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(pages.dom_building_time) AS avg + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(pages.dom_building_time) AS value FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -985,10 +939,10 @@ def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=- FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avg": avg, + return {"value": avg, "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, - density=density, neutral={"avg": 0})} + density=density, neutral={"value": 0})} def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1009,9 +963,8 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query_chart.append("isNotNull(resources.duration)") ch_sub_query_chart.append("resources.duration>0") with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - splitByChar('/', resources.url_hostpath)[-1] AS name, - AVG(NULLIF(resources.duration,0)) AS avg + ch_query = f"""SELECT splitByChar('/', resources.url_hostpath)[-1] AS name, + AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY name @@ -1030,9 +983,8 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), # r["url"] = r["url"].decode("utf-8") # except UnicodeDecodeError: # continue - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(resources.duration) AS avg + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(resources.duration) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1119,7 +1071,7 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 ch_sub_query_chart.append(f"url_path = %(value)s") with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(pages.response_time) AS avg + AVG(pages.response_time) AS value FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1134,10 +1086,10 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avg": avg, + return {"value": avg, "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, - density=density, neutral={"avg": 0})} + density=density, neutral={"value": 0})} def get_pages_response_time_distribution(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1268,9 +1220,8 @@ def get_busiest_time_of_day(project_id, startTimestamp=TimeUTC.now(delta_days=-1 ch_sub_query += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - intDiv(toHour(sessions.datetime),2)*2 AS hour, - COUNT(sessions.session_id) AS count + ch_query = f"""SELECT intDiv(toHour(sessions.datetime),2)*2 AS hour, + COUNT(sessions.session_id) AS count FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY hour @@ -1320,7 +1271,7 @@ def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(pages.visually_complete) AS avg + AVG(pages.visually_complete) AS value FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1334,9 +1285,9 @@ def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avg": avg, "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, + return {"value": avg, "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, - neutral={"avg": 0})} + neutral={"value": 0})} def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1353,7 +1304,7 @@ def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(d with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(DISTINCT pages.session_id) AS count + COUNT(DISTINCT pages.session_id) AS count FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} AND (pages.response_time)>(SELECT AVG(pages.response_time) @@ -1382,7 +1333,7 @@ def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1) with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(performance.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(performance.avg_used_js_heap_size) AS avg_used_js_heap_size + AVG(performance.avg_used_js_heap_size) AS value FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1396,11 +1347,11 @@ def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1) FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avgUsedJsHeapSize": avg, + return {"value": avg, "chart": helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, - neutral={"avg_used_js_heap_size": 0}))} + neutral={"value": 0}))} def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1413,7 +1364,7 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(performance.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(performance.avg_cpu) AS avg_cpu + AVG(performance.avg_cpu) AS value FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1427,11 +1378,11 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avgCpu": avg, + return {"value": avg, "chart": helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, - neutral={"avg_cpu": 0}))} + neutral={"value": 0}))} def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1444,7 +1395,7 @@ def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(performance.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(performance.avg_fps) AS avg_fps + AVG(performance.avg_fps) AS value FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1458,11 +1409,11 @@ def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avgFps": avg, + return {"value": avg, "chart": helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, - neutral={"avg_fps": 0}))} + neutral={"value": 0}))} def __get_crashed_sessions_ids(project_id, startTimestamp, endTimestamp): @@ -1698,9 +1649,8 @@ def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - resources.url_host AS domain, - AVG(resources.duration) AS avg + ch_query = f"""SELECT resources.url_host AS domain, + AVG(resources.duration) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY resources.url_host @@ -1747,15 +1697,13 @@ def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=- ch_sub_query += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - b.user_browser AS browser, - b.count, - groupArray([bv.user_browser_version, toString(bv.count)]) AS versions + ch_query = f"""SELECT b.user_browser AS browser, + b.count, + groupArray([bv.user_browser_version, toString(bv.count)]) AS versions FROM ( - SELECT - sessions.user_browser, - COUNT(sessions.session_id) AS count + SELECT sessions.user_browser, + COUNT(sessions.session_id) AS count FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY sessions.user_browser @@ -1764,10 +1712,9 @@ def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=- ) AS b INNER JOIN ( - SELECT - sessions.user_browser, - sessions.user_browser_version, - COUNT(sessions.session_id) AS count + SELECT sessions.user_browser, + sessions.user_browser_version, + COUNT(sessions.session_id) AS count FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY @@ -1934,8 +1881,8 @@ def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_d "endTimestamp": endTimestamp, **__get_constraint_values(args)} with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(resources.session_id) AS total, - SUM(if(resources.type='fetch',1,0)) AS xhr + COUNT(resources.session_id) AS total, + SUM(if(resources.type='fetch',1,0)) AS xhr FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1946,7 +1893,7 @@ def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_d density=density, neutral={"total": 0, "xhr": 0}) ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(pages.response_end) AS avg_response_end + AVG(pages.response_end) AS avg_response_end FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart_response_end)} GROUP BY timestamp @@ -1969,8 +1916,8 @@ def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(de with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(errors.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(DISTINCT errors.session_id) AS sessions_count, - COUNT(DISTINCT errors.error_id) AS errors_count + COUNT(DISTINCT errors.session_id) AS sessions_count, + COUNT(DISTINCT errors.error_id) AS errors_count FROM errors {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -2008,15 +1955,13 @@ def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(de ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(s.base_datetime, toIntervalSecond(%(step_size)s))) * 1000 AS timestamp, - AVG(NULLIF(s.count,0)) AS avg, - groupArray([toString(t.type), toString(t.xavg)]) AS types + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(s.base_datetime, toIntervalSecond(%(step_size)s))) * 1000 AS timestamp, + AVG(NULLIF(s.count,0)) AS avg, + groupArray([toString(t.type), toString(t.xavg)]) AS types FROM - ( SELECT - resources.session_id, - MIN(resources.datetime) AS base_datetime, - COUNT(resources.url) AS count + ( SELECT resources.session_id, + MIN(resources.datetime) AS base_datetime, + COUNT(resources.url) AS count FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY resources.session_id @@ -2137,3 +2082,490 @@ def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1) density=density, neutral={"first_party": 0, "third_party": 0})) + + +def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with ch_client.ClickHouseClient() as ch: + row = __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + results["chart"] = get_performance_avg_page_load_time(project_id, startTimestamp, endTimestamp, **args) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + ch_query = f"""\ + SELECT AVG(NULLIF(pages.load_event_end ,0)) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + row = ch.execute(query=ch_query, params=params)[0] + result = row + for k in result: + if result[k] is None: + result[k] = 0 + return result + + +def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, resources=None, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) + location_constraints = [] + meta_condition = __get_meta_constraint(args) + + location_constraints_vals = {} + + if resources and len(resources) > 0: + for r in resources: + if r["type"] == "LOCATION": + location_constraints.append(f"pages.url_path = %(val_{len(location_constraints)})s") + location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value'] + + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + with ch_client.ClickHouseClient() as ch: + ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, + data=args) + ch_sub_query_chart += meta_condition + + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(pages.load_event_end ,0)) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + GROUP BY timestamp + ORDER BY timestamp;""" + + rows = ch.execute(query=ch_query, + params={**params, **location_constraints_vals, **__get_constraint_values(args)}) + pages = [{"timestamp": i["timestamp"], "value": i["value"]} for i in + __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, + density=density, neutral={"value": 0})] + + for s in pages: + for k in s: + if s[k] is None: + s[k] = 0 + return pages + + +def get_application_activity_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with ch_client.ClickHouseClient() as ch: + row = __get_application_activity_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + results["chart"] = get_performance_avg_image_load_time(project_id, startTimestamp, endTimestamp, **args) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_application_activity_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="resources", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + ch_sub_query.append("resources.type= %(type)s") + ch_query = f"""\ + SELECT AVG(NULLIF(resources.duration,0)) AS value + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + row = ch.execute(query=ch_query, + params={"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})[0] + result = row + for k in result: + if result[k] is None: + result[k] = 0 + return result + + +def get_performance_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, resources=None, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) + img_constraints = [] + ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query_chart += meta_condition + + img_constraints_vals = {} + + if resources and len(resources) > 0: + for r in resources: + if r["type"] == "IMG": + img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s") + img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value'] + + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS value + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + AND resources.type = 'img' + {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} + GROUP BY timestamp + ORDER BY timestamp;""" + rows = ch.execute(query=ch_query, params={**params, **img_constraints_vals, **__get_constraint_values(args)}) + images = [{"timestamp": i["timestamp"], "value": i["value"]} for i in + __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, + density=density, neutral={"value": 0})] + + for s in images: + for k in s: + if s[k] is None: + s[k] = 0 + return images + + +def get_application_activity_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with ch_client.ClickHouseClient() as ch: + row = __get_application_activity_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + results["chart"] = get_performance_avg_request_load_time(project_id, startTimestamp, endTimestamp, **args) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_application_activity_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="resources", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + ch_sub_query.append("resources.type= %(type)s") + ch_query = f"""\ + SELECT AVG(NULLIF(resources.duration,0)) AS value + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + row = ch.execute(query=ch_query, + params={"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})[0] + result = row + for k in result: + if result[k] is None: + result[k] = 0 + return result + + +def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, resources=None, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) + request_constraints = [] + ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query_chart += meta_condition + + request_constraints_vals = {} + + if resources and len(resources) > 0: + for r in resources: + if r["type"] != "IMG" and r["type"] == "LOCATION": + request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s") + request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS value + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + AND resources.type = 'fetch' + {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} + GROUP BY timestamp + ORDER BY timestamp;""" + rows = ch.execute(query=ch_query, + params={**params, **request_constraints_vals, **__get_constraint_values(args)}) + requests = [{"timestamp": i["timestamp"], "value": i["value"]} for i in + __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, density=density, + neutral={"value": 0})] + + for s in requests: + for k in s: + if s[k] is None: + s[k] = 0 + return requests + + +def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with ch_client.ClickHouseClient() as ch: + rows = __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + ch_query = f"""\ + SELECT COALESCE(AVG(NULLIF(pages.dom_content_loaded_event_end ,0)),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + params = {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + rows = ch.execute(query=ch_query, params=params) + return rows + + +def get_page_metrics_avg_first_contentful_pixel(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with ch_client.ClickHouseClient() as ch: + rows = __get_page_metrics_avg_first_contentful_pixel(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_page_metrics_avg_first_contentful_pixel(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_page_metrics_avg_first_contentful_pixel(ch, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + # changed dom_content_loaded_event_start to dom_content_loaded_event_end + ch_query = f"""\ + SELECT COALESCE(AVG(NULLIF(pages.first_contentful_paint,0)),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + params = {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + rows = ch.execute(query=ch_query, params=params) + return rows + + +def get_user_activity_avg_visited_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + results = {} + + with ch_client.ClickHouseClient() as ch: + rows = __get_user_activity_avg_visited_pages(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + for key in results: + if isnan(results[key]): + results[key] = 0 + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_user_activity_avg_visited_pages(ch, project_id, startTimestamp, endTimestamp, **args) + + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_user_activity_avg_visited_pages(cur, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="sessions", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + ch_query = f"""\ + SELECT COALESCE(CEIL(AVG(NULLIF(sessions.pages_count,0))),0) AS value + FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + rows = cur.execute(query=ch_query, params=params) + + return rows + + +def get_user_activity_avg_session_duration(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + results = {} + + with ch_client.ClickHouseClient() as ch: + rows = __get_user_activity_avg_session_duration(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + for key in results: + if isnan(results[key]): + results[key] = 0 + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_user_activity_avg_session_duration(ch, project_id, startTimestamp, endTimestamp, **args) + + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="sessions", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + ch_query = f"""\ + SELECT COALESCE(AVG(NULLIF(sessions.duration,0)),0) AS value + FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + rows = cur.execute(query=ch_query, params=params) + + return rows + + +def get_top_metrics_avg_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COALESCE(AVG(pages.response_time),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.response_time) AND pages.response_time>0;""" + rows = ch.execute(query=ch_query, + params={"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)}) + return helper.dict_to_camel_case(rows[0]) + + +def get_top_metrics_count_requests(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COUNT(pages.session_id) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + rows = ch.execute(query=ch_query, + params={"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)}) + return helper.dict_to_camel_case(rows[0]) + + +def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COALESCE(AVG(pages.first_paint),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.first_paint) AND pages.first_paint>0;""" + rows = ch.execute(query=ch_query, + params={"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)}) + return helper.dict_to_camel_case(rows[0]) + + +def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COALESCE(AVG(pages.dom_content_loaded_event_time),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.dom_content_loaded_event_time) AND pages.dom_content_loaded_event_time>0;""" + rows = ch.execute(query=ch_query, + params={"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)}) + return helper.dict_to_camel_case(rows[0]) + + +def get_top_metrics_avg_till_first_bit(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COALESCE(AVG(pages.ttfb),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.ttfb) AND pages.ttfb>0;""" + rows = ch.execute(query=ch_query, + params={"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)}) + return helper.dict_to_camel_case(rows[0]) + + +def get_top_metrics_avg_time_to_interactive(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COALESCE(AVG(pages.time_to_interactive),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.time_to_interactive) AND pages.time_to_interactive >0;""" + rows = ch.execute(query=ch_query, + params={"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)}) + return helper.dict_to_camel_case(rows[0]) diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index ef143038b..5909d31c1 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -8,7 +8,7 @@ jira==3.1.1 clickhouse-driver==0.2.2 python3-saml==1.12.0 -fastapi==0.74.1 +fastapi==0.75.0 python-multipart==0.0.5 uvicorn[standard]==0.17.5 python-decouple==3.6 diff --git a/ee/api/routers/app/v1_api_ee.py b/ee/api/routers/subs/v1_api_ee.py similarity index 100% rename from ee/api/routers/app/v1_api_ee.py rename to ee/api/routers/subs/v1_api_ee.py From 2fe3f88a7d651b3d8e10cfc390f236c7b441994a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 18:41:05 +0200 Subject: [PATCH 039/125] feat(api): EE metrics get slowest images optimized (from 30s to 4s) --- ee/api/chalicelib/core/dashboard.py | 47 +++++++++++++++++------------ 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/ee/api/chalicelib/core/dashboard.py b/ee/api/chalicelib/core/dashboard.py index c834c852b..826be538a 100644 --- a/ee/api/chalicelib/core/dashboard.py +++ b/ee/api/chalicelib/core/dashboard.py @@ -456,7 +456,7 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query.append("resources.type = 'img'") ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) ch_sub_query_chart.append("resources.type = 'img'") - ch_sub_query_chart.append("resources.url = %(url)s") + ch_sub_query_chart.append("resources.url IN %(url)s") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition ch_sub_query_chart += meta_condition @@ -468,35 +468,42 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY resources.url ORDER BY avg DESC LIMIT 10;""" - - rows = ch.execute(query=ch_query, - params={"project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)}) + params = {"project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + # print(ch.client().substitute_params(ch_query, params)) + rows = ch.execute(query=ch_query, params=params) rows = [{"url": i["url"], "avgDuration": i["avg"], "sessions": i["count"]} for i in rows] - + if len(rows) == 0: + return [] urls = [row["url"] for row in rows] charts = {} + ch_query = f"""\ + SELECT url, + toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS avg + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + GROUP BY url, timestamp + ORDER BY url, timestamp;""" + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, "url": urls, **__get_constraint_values(args)} + print(ch.client().substitute_params(ch_query, params)) + u_rows = ch.execute(query=ch_query, params=params) for url in urls: - ch_query = f"""\ - SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(resources.duration,0)) AS avg - FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, "url": url, **__get_constraint_values(args)} - r = ch.execute(query=ch_query, params=params) + sub_rows = [] + for r in u_rows: + if r["url"] == url: + sub_rows.append(r) charts[url] = [{"timestamp": int(i["timestamp"]), "avgDuration": i["avg"]} - for i in __complete_missing_steps(rows=r, start_time=startTimestamp, + for i in __complete_missing_steps(rows=sub_rows, start_time=startTimestamp, end_time=endTimestamp, density=density, neutral={"avg": 0})] for i in range(len(rows)): rows[i] = helper.dict_to_camel_case(rows[i]) - rows[i]["chart"] = [helper.dict_to_camel_case(chart) for chart in charts[rows[i]["url"]]] + rows[i]["chart"] = helper.list_to_camel_case(charts[rows[i]["url"]]) return sorted(rows, key=lambda k: k["sessions"], reverse=True) @@ -1286,8 +1293,8 @@ def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 return {"value": avg, "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, density=density, - neutral={"value": 0})} + end_time=endTimestamp, density=density, + neutral={"value": 0})} def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), From 3d2ae11ce16a4f4a674e005cc3939d5ff650d8f9 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 18:48:42 +0200 Subject: [PATCH 040/125] feat(api): fixed get metrics with dashboards --- api/chalicelib/core/custom_metrics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 6ca72f453..cb1e371c0 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -220,10 +220,11 @@ def get_all(project_id, user_id, include_series=False): {sub_join} LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards FROM (SELECT dashboard_id, name, is_public - FROM dashboards + FROM dashboards INNER JOIN dashboard_widgets USING (dashboard_id) WHERE deleted_at ISNULL + AND dashboard_widgets.metric_id = metrics.metric_id AND project_id = %(project_id)s - AND ((user_id = %(user_id)s OR is_public))) AS connected_dashboards + AND ((dashboards.user_id = %(user_id)s OR is_public))) AS connected_dashboards ) AS connected_dashboards ON (TRUE) LEFT JOIN LATERAL (SELECT email AS owner_email FROM users From 02209a7716b6ca057ca3903e087e60351724281e Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 18:53:41 +0200 Subject: [PATCH 041/125] feat(api): fixed update dashboard with metrics list --- api/chalicelib/core/dashboards2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 3efd1e968..dd259a077 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -113,8 +113,7 @@ def update_dashboard(project_id, user_id, dashboard_id, data: schemas.EditDashbo if data.metrics is not None and len(data.metrics) > 0: pg_query = f"""WITH dash AS ({pg_query}) INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) - VALUES {",".join([f"(%(dashboard_id)s, %(metric_id_{i})s, %(userId)s, %(config_{i})s)" for i in range(len(data.metrics))])} - RETURNING (SELECT dashboard_id FROM dash)""" + VALUES {",".join([f"(%(dashboard_id)s, %(metric_id_{i})s, %(userId)s, %(config_{i})s)" for i in range(len(data.metrics))])};""" for i, m in enumerate(data.metrics): params[f"metric_id_{i}"] = m params[f"config_{i}"] = schemas.AddWidgetToDashboardPayloadSchema.schema() \ From 5582f0ac36a7ed3522ec14725f277aa25674471f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 19:18:29 +0200 Subject: [PATCH 042/125] feat(api): fixed get dashboard with templates --- api/chalicelib/core/dashboards2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index dd259a077..0e233d54b 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -76,7 +76,7 @@ def get_dashboard(project_id, user_id, dashboard_id): ) AS metric_series ON (TRUE) WHERE dashboard_widgets.dashboard_id = dashboards.dashboard_id AND metrics.deleted_at ISNULL - AND metrics.project_id = %(projectId)s) AS raw_metrics + AND (metrics.project_id = %(projectId)s OR metrics.project_id ISNULL)) AS raw_metrics ) AS all_metric_widgets ON (TRUE) WHERE dashboards.deleted_at ISNULL AND dashboards.project_id = %(projectId)s From ef6ba3b9ce83201a88e3c4e56c799e1361cfa0c0 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 19:40:07 +0200 Subject: [PATCH 043/125] feat(api): EE metrics get slowest resources optimized (from 52s to 7s) --- ee/api/chalicelib/core/dashboard.py | 72 ++++++++++++++--------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/ee/api/chalicelib/core/dashboard.py b/ee/api/chalicelib/core/dashboard.py index 826be538a..9c02e5ae4 100644 --- a/ee/api/chalicelib/core/dashboard.py +++ b/ee/api/chalicelib/core/dashboard.py @@ -489,13 +489,15 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ORDER BY url, timestamp;""" params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, "url": urls, **__get_constraint_values(args)} - print(ch.client().substitute_params(ch_query, params)) + # print(ch.client().substitute_params(ch_query, params)) u_rows = ch.execute(query=ch_query, params=params) for url in urls: sub_rows = [] for r in u_rows: if r["url"] == url: sub_rows.append(r) + elif len(sub_rows) > 0: + break charts[url] = [{"timestamp": int(i["timestamp"]), "avgDuration": i["avg"]} for i in __complete_missing_steps(rows=sub_rows, start_time=startTimestamp, @@ -970,52 +972,50 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query_chart.append("isNotNull(resources.duration)") ch_sub_query_chart.append("resources.duration>0") with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT splitByChar('/', resources.url_hostpath)[-1] AS name, + ch_query = f"""SELECT any(url) AS url, any(type) AS type, + splitByChar('/', resources.url_hostpath)[-1] AS name, AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY name ORDER BY avg DESC LIMIT 10;""" - rows = ch.execute(query=ch_query, - params={"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)}) - ch_sub_query_chart.append("endsWith(resources.url_hostpath, %(url)s)>0") + params = {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + print(ch.format(query=ch_query, params=params)) + rows = ch.execute(query=ch_query, params=params) + # ch_sub_query_chart.append("endsWith(resources.url_hostpath, %(url)s)>0") ch_sub_query.append(ch_sub_query_chart[-1]) results = [] + names = {f"name_{i}": r["name"] for i, r in enumerate(rows)} + ch_query = f"""SELECT splitByChar('/', resources.url_hostpath)[-1] AS name, + toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(resources.duration) AS avg + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + AND ({" OR ".join([f"endsWith(resources.url_hostpath, %(name_{i})s)>0" for i in range(len(names.keys()))])}) + GROUP BY name,timestamp + ORDER BY name,timestamp;""" + params = {"step_size": step_size, "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + **names, **__get_constraint_values(args)} + # print(ch.format(query=ch_query, params=params)) + charts = ch.execute(query=ch_query, params=params) for r in rows: - # if isinstance(r["url"], bytes): - # try: - # r["url"] = r["url"].decode("utf-8") - # except UnicodeDecodeError: - # continue - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(resources.duration) AS avg - FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - chart = ch.execute(query=ch_query, - params={"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "url": r["name"], **__get_constraint_values(args)}) - r["chart"] = __complete_missing_steps(rows=chart, start_time=startTimestamp, + sub_chart = [] + for c in charts: + if c["name"] == r["name"]: + cc = dict(c) + cc.pop("name") + sub_chart.append(cc) + elif len(sub_chart) > 0: + break + r["chart"] = __complete_missing_steps(rows=sub_chart, start_time=startTimestamp, end_time=endTimestamp, density=density, neutral={"avg": 0}) - ch_query = f"""SELECT url, type - FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} - WHERE {" AND ".join(ch_sub_query)} - ORDER BY duration DESC - LIMIT 1;""" - url = ch.execute(query=ch_query, - params={"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "url": r["name"], **__get_constraint_values(args)}) - r["url"] = url[0]["url"] - r["type"] = __get_resource_type_from_db_type(url[0]["type"]) + r["type"] = __get_resource_type_from_db_type(r["type"]) results.append(r) return results From 3f621165d6bb30a7ff3ae50b63dd348a60fcfc7d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 19:43:44 +0200 Subject: [PATCH 044/125] feat(api): EE metrics refactored --- ee/api/chalicelib/core/dashboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/api/chalicelib/core/dashboard.py b/ee/api/chalicelib/core/dashboard.py index 9c02e5ae4..c36c877da 100644 --- a/ee/api/chalicelib/core/dashboard.py +++ b/ee/api/chalicelib/core/dashboard.py @@ -985,7 +985,7 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), "endTimestamp": endTimestamp, **__get_constraint_values(args)} print(ch.format(query=ch_query, params=params)) rows = ch.execute(query=ch_query, params=params) - # ch_sub_query_chart.append("endsWith(resources.url_hostpath, %(url)s)>0") + ch_sub_query.append(ch_sub_query_chart[-1]) results = [] names = {f"name_{i}": r["name"] for i, r in enumerate(rows)} From 07fd1027f8ab2b7b7f9b67b96650aad160a81480 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 19:55:00 +0200 Subject: [PATCH 045/125] feat(api): EE new method for debug in ch_client --- ee/api/chalicelib/utils/ch_client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ee/api/chalicelib/utils/ch_client.py b/ee/api/chalicelib/utils/ch_client.py index babdd669a..aa45699f7 100644 --- a/ee/api/chalicelib/utils/ch_client.py +++ b/ee/api/chalicelib/utils/ch_client.py @@ -25,5 +25,8 @@ class ClickHouseClient: def client(self): return self.__client + def format(self, query, params): + return self.__client.substitute_params(query, params) + def __exit__(self, *args): pass From 584de8d09c5a329ca3a49e70c4a703f71e42f0a7 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 6 Apr 2022 11:10:02 +0200 Subject: [PATCH 046/125] feat(api): jira return Auth-error to UI feat(api): metrics allow empty series --- api/chalicelib/utils/jira_client.py | 1 + api/schemas.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api/chalicelib/utils/jira_client.py b/api/chalicelib/utils/jira_client.py index 8370feb5e..40bc0d66c 100644 --- a/api/chalicelib/utils/jira_client.py +++ b/api/chalicelib/utils/jira_client.py @@ -22,6 +22,7 @@ class JiraManager: except Exception as e: print("!!! JIRA AUTH ERROR") print(e) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: Authentication error") def set_jira_project_id(self, project_id): self._config["JIRA_PROJECT_ID"] = project_id diff --git a/api/schemas.py b/api/schemas.py index 1f1bec739..989ece66e 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -820,7 +820,7 @@ class CustomMetricChartPayloadSchema(CustomMetricSessionsPayloadSchema): class CreateCustomMetricsSchema(CustomMetricChartPayloadSchema): name: str = Field(...) - series: List[CustomMetricCreateSeriesSchema] = Field(..., min_items=1) + series: List[CustomMetricCreateSeriesSchema] = Field(...) is_public: bool = Field(default=True) view_type: Union[MetricTimeseriesViewType, MetricTableViewType] = Field(MetricTimeseriesViewType.line_chart) metric_type: MetricType = Field(MetricType.timeseries) From 3c30ebf76d800d661534f43a68fbbc38ae2f6232 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 6 Apr 2022 14:17:24 +0200 Subject: [PATCH 047/125] feat(api): changed update widget feat(api): FOSS&EE optimized /projects --- api/chalicelib/core/dashboards2.py | 2 +- api/chalicelib/core/projects.py | 25 ++++++++++++------------- api/routers/subs/metrics.py | 2 +- api/schemas.py | 10 ++++++++-- ee/api/.gitignore | 1 + ee/api/chalicelib/core/projects.py | 22 +++++++++++----------- 6 files changed, 34 insertions(+), 28 deletions(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 0e233d54b..afe65d71a 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -167,7 +167,7 @@ def add_widget(project_id, user_id, dashboard_id, data: schemas.AddWidgetToDashb return helper.dict_to_camel_case(row) -def update_widget(project_id, user_id, dashboard_id, widget_id, data: schemas.AddWidgetToDashboardPayloadSchema): +def update_widget(project_id, user_id, dashboard_id, widget_id, data: schemas.UpdateWidgetPayloadSchema): with pg_client.PostgresClient() as cur: pg_query = """UPDATE dashboard_widgets SET config= %(config)s diff --git a/api/chalicelib/core/projects.py b/api/chalicelib/core/projects.py index 0c2fd9e01..3559f645a 100644 --- a/api/chalicelib/core/projects.py +++ b/api/chalicelib/core/projects.py @@ -65,27 +65,26 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st FROM public.projects AS s {'LEFT JOIN LATERAL (SELECT COUNT(*) AS count FROM public.integrations WHERE s.project_id = integrations.project_id LIMIT 1) AS stack_integrations ON TRUE' if stack_integrations else ''} WHERE s.deleted_at IS NULL - ORDER BY s.project_id;""" - ) + ORDER BY s.project_id;""") rows = cur.fetchall() if recording_state: project_ids = [f'({r["project_id"]})' for r in rows] - query = f"""SELECT projects.project_id, COALESCE(MAX(start_ts), 0) AS last - FROM (VALUES {",".join(project_ids)}) AS projects(project_id) - LEFT JOIN sessions USING (project_id) - GROUP BY project_id;""" - cur.execute( - query=query - ) + query = cur.mogrify(f"""SELECT projects.project_id, COALESCE(MAX(start_ts), 0) AS last + FROM (VALUES {",".join(project_ids)}) AS projects(project_id) + LEFT JOIN sessions USING (project_id) + WHERE sessions.start_ts >= %(startDate)s AND sessions.start_ts <= %(endDate)s + GROUP BY project_id;""", + {"startDate": TimeUTC.now(delta_days=-3), "endDate": TimeUTC.now(delta_days=1)}) + + cur.execute(query=query) status = cur.fetchall() for r in rows: + r["status"] = "red" for s in status: if s["project_id"] == r["project_id"]: - if s["last"] < TimeUTC.now(-2): - r["status"] = "red" - elif s["last"] < TimeUTC.now(-1): + if TimeUTC.now(-2) <= s["last"] < TimeUTC.now(-1): r["status"] = "yellow" - else: + elif s["last"] >= TimeUTC.now(-1): r["status"] = "green" break diff --git a/api/routers/subs/metrics.py b/api/routers/subs/metrics.py index fc875e0fd..02eec811d 100644 --- a/api/routers/subs/metrics.py +++ b/api/routers/subs/metrics.py @@ -68,7 +68,7 @@ def create_metric_and_add_to_dashboard(projectId: int, dashboardId: int, @app.post('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) @app.put('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) def update_widget_in_dashboard(projectId: int, dashboardId: int, widgetId: int, - data: schemas.AddWidgetToDashboardPayloadSchema = Body(...), + data: schemas.UpdateWidgetPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): return dashboards2.update_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, widget_id=widgetId, data=data) diff --git a/api/schemas.py b/api/schemas.py index 989ece66e..65db116c0 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -893,8 +893,7 @@ class EditDashboardSchema(CreateDashboardSchema): is_pinned: Optional[bool] = Field(default=None) -class AddWidgetToDashboardPayloadSchema(BaseModel): - metric_id: int = Field(default=None) +class UpdateWidgetPayloadSchema(BaseModel): # if you change the config attribute name, please make sure to update it in dashboard2.py config: dict = Field(default={"col": 1, "row": 1, "position": 0}) @@ -902,6 +901,13 @@ class AddWidgetToDashboardPayloadSchema(BaseModel): alias_generator = attribute_to_camel_case +class AddWidgetToDashboardPayloadSchema(UpdateWidgetPayloadSchema): + metric_id: int = Field(default=None) + + class Config: + alias_generator = attribute_to_camel_case + + # these values should match the keys in metrics table class TemplateKeys(str, Enum): count_sessions = "count_sessions" diff --git a/ee/api/.gitignore b/ee/api/.gitignore index af91e1919..488fab072 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -259,3 +259,4 @@ Pipfile /build_alerts.sh /routers/subs/metrics.py /routers/subs/v1_api.py +/chalicelib/core/dashboards2.py diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index 0255c8c8c..3072f55a0 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -82,22 +82,22 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st rows = cur.fetchall() if recording_state: project_ids = [f'({r["project_id"]})' for r in rows] - query = f"""SELECT projects.project_id, COALESCE(MAX(start_ts), 0) AS last - FROM (VALUES {",".join(project_ids)}) AS projects(project_id) - LEFT JOIN sessions USING (project_id) - GROUP BY project_id;""" - cur.execute( - query=query - ) + query = cur.mogrify(f"""SELECT projects.project_id, COALESCE(MAX(start_ts), 0) AS last + FROM (VALUES {",".join(project_ids)}) AS projects(project_id) + LEFT JOIN sessions USING (project_id) + WHERE sessions.start_ts >= %(startDate)s AND sessions.start_ts <= %(endDate)s + GROUP BY project_id;""", + {"startDate": TimeUTC.now(delta_days=-3), "endDate": TimeUTC.now(delta_days=1)}) + + cur.execute(query=query) status = cur.fetchall() for r in rows: + r["status"] = "red" for s in status: if s["project_id"] == r["project_id"]: - if s["last"] < TimeUTC.now(-2): - r["status"] = "red" - elif s["last"] < TimeUTC.now(-1): + if TimeUTC.now(-2) <= s["last"] < TimeUTC.now(-1): r["status"] = "yellow" - else: + elif s["last"] >= TimeUTC.now(-1): r["status"] = "green" break From 70ef8af0cb81ff507ed2517c5cc8b60308f72166 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 6 Apr 2022 14:25:43 +0200 Subject: [PATCH 048/125] feat(api): fixed duplicate dashboard for metrics list --- api/chalicelib/core/custom_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index cb1e371c0..77f902dcb 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -219,7 +219,7 @@ def get_all(project_id, user_id, include_series=False): FROM metrics {sub_join} LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards - FROM (SELECT dashboard_id, name, is_public + FROM (SELECT DISTINCT dashboard_id, name, is_public FROM dashboards INNER JOIN dashboard_widgets USING (dashboard_id) WHERE deleted_at ISNULL AND dashboard_widgets.metric_id = metrics.metric_id From c30ceb447d54886a132b5f36b07fca749ab4fd0f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 6 Apr 2022 15:28:22 +0200 Subject: [PATCH 049/125] feat(utilities): FOSS-WS send SESSION_RECONNECTED on session's reconnection --- utilities/servers/websocket.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utilities/servers/websocket.js b/utilities/servers/websocket.js index 772bd7315..3cd22f3fc 100644 --- a/utilities/servers/websocket.js +++ b/utilities/servers/websocket.js @@ -12,6 +12,7 @@ const AGENT_DISCONNECT = "AGENT_DISCONNECTED"; const AGENTS_CONNECTED = "AGENTS_CONNECTED"; const NO_SESSIONS = "SESSION_DISCONNECTED"; const SESSION_ALREADY_CONNECTED = "SESSION_ALREADY_CONNECTED"; +const SESSION_RECONNECTED = "SESSION_RECONNECTED" let io; const debug = process.env.debug === "1" || false; @@ -258,6 +259,7 @@ module.exports = { debug && console.log(`notifying new session about agent-existence`); let agents_ids = await get_all_agents_ids(io, socket); io.to(socket.id).emit(AGENTS_CONNECTED, agents_ids); + socket.to(socket.peerId).emit(SESSION_RECONNECTED, socket.id); } } else if (c_sessions <= 0) { From c9f7b6e98034b5472a915b62ddf441f9f9c33fb3 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 6 Apr 2022 16:48:26 +0200 Subject: [PATCH 050/125] feat(api): jira integration handle errors --- api/chalicelib/core/integration_jira_cloud.py | 24 ++++++++++++------- api/chalicelib/core/integrations_manager.py | 5 +++- api/chalicelib/utils/jira_client.py | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/api/chalicelib/core/integration_jira_cloud.py b/api/chalicelib/core/integration_jira_cloud.py index ea9c6c24e..7d8c956cf 100644 --- a/api/chalicelib/core/integration_jira_cloud.py +++ b/api/chalicelib/core/integration_jira_cloud.py @@ -15,10 +15,17 @@ class JIRAIntegration(integration_base.BaseIntegration): # TODO: enable super-constructor when OAuth is done # super(JIRAIntegration, self).__init__(jwt, user_id, JIRACloudIntegrationProxy) self._user_id = user_id - i = self.get() - if i is None: + self.integration = self.get() + if self.integration is None: return - self.issue_handler = JIRACloudIntegrationIssue(token=i["token"], username=i["username"], url=i["url"]) + self.integration["valid"] = True + try: + self.issue_handler = JIRACloudIntegrationIssue(token=self.integration["token"], + username=self.integration["username"], + url=self.integration["url"]) + except Exception as e: + self.issue_handler = None + self.integration["valid"] = False @property def provider(self): @@ -37,10 +44,10 @@ class JIRAIntegration(integration_base.BaseIntegration): return helper.dict_to_camel_case(cur.fetchone()) def get_obfuscated(self): - integration = self.get() - if integration is None: + if self.integration is None: return None - integration["token"] = obfuscate_string(integration["token"]) + integration = dict(self.integration) + integration["token"] = obfuscate_string(self.integration["token"]) integration["provider"] = self.provider.lower() return integration @@ -90,14 +97,13 @@ class JIRAIntegration(integration_base.BaseIntegration): return {"state": "success"} def add_edit(self, data): - s = self.get() - if s is not None: + if self.integration is not None: return self.update( changes={ "username": data["username"], "token": data["token"] \ if data.get("token") and len(data["token"]) > 0 and data["token"].find("***") == -1 \ - else s["token"], + else self.integration["token"], "url": data["url"] }, obfuscate=True diff --git a/api/chalicelib/core/integrations_manager.py b/api/chalicelib/core/integrations_manager.py index fca271870..ef63a7d96 100644 --- a/api/chalicelib/core/integrations_manager.py +++ b/api/chalicelib/core/integrations_manager.py @@ -36,7 +36,10 @@ def get_integration(tenant_id, user_id, tool=None): if tool not in SUPPORTED_TOOLS: return {"errors": [f"issue tracking tool not supported yet, available: {SUPPORTED_TOOLS}"]}, None if tool == integration_jira_cloud.PROVIDER: - return None, integration_jira_cloud.JIRAIntegration(tenant_id=tenant_id, user_id=user_id) + integration = integration_jira_cloud.JIRAIntegration(tenant_id=tenant_id, user_id=user_id) + if integration.integration is not None and not integration.integration.get("valid", True): + return {"errors": ["JIRA: connexion issue/unauthorized"]}, integration + return None, integration elif tool == integration_github.PROVIDER: return None, integration_github.GitHubIntegration(tenant_id=tenant_id, user_id=user_id) return {"errors": ["lost integration"]}, None diff --git a/api/chalicelib/utils/jira_client.py b/api/chalicelib/utils/jira_client.py index 40bc0d66c..b1734660c 100644 --- a/api/chalicelib/utils/jira_client.py +++ b/api/chalicelib/utils/jira_client.py @@ -22,7 +22,7 @@ class JiraManager: except Exception as e: print("!!! JIRA AUTH ERROR") print(e) - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: Authentication error") + raise e def set_jira_project_id(self, project_id): self._config["JIRA_PROJECT_ID"] = project_id From 7ebcbb8b0b1aeda348aafe1323152a4cfd2f8a3c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 6 Apr 2022 16:53:27 +0200 Subject: [PATCH 051/125] feat(utilities): EE-WS send SESSION_RECONNECTED on session's reconnection --- ee/utilities/servers/websocket-cluster.js | 2 ++ ee/utilities/servers/websocket.js | 2 ++ utilities/servers/websocket.js | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ee/utilities/servers/websocket-cluster.js b/ee/utilities/servers/websocket-cluster.js index c044043a5..998a457df 100644 --- a/ee/utilities/servers/websocket-cluster.js +++ b/ee/utilities/servers/websocket-cluster.js @@ -14,6 +14,7 @@ const AGENT_DISCONNECT = "AGENT_DISCONNECTED"; const AGENTS_CONNECTED = "AGENTS_CONNECTED"; const NO_SESSIONS = "SESSION_DISCONNECTED"; const SESSION_ALREADY_CONNECTED = "SESSION_ALREADY_CONNECTED"; +const SESSION_RECONNECTED = "SESSION_RECONNECTED"; const REDIS_URL = process.env.REDIS_URL || "redis://localhost:6379"; const pubClient = createClient({url: REDIS_URL}); const subClient = pubClient.duplicate(); @@ -309,6 +310,7 @@ module.exports = { debug && console.log(`notifying new session about agent-existence`); let agents_ids = await get_all_agents_ids(io, socket); io.to(socket.id).emit(AGENTS_CONNECTED, agents_ids); + socket.to(socket.peerId).emit(SESSION_RECONNECTED, socket.id); } } else if (c_sessions <= 0) { diff --git a/ee/utilities/servers/websocket.js b/ee/utilities/servers/websocket.js index 0bd397d96..256286351 100644 --- a/ee/utilities/servers/websocket.js +++ b/ee/utilities/servers/websocket.js @@ -12,6 +12,7 @@ const AGENT_DISCONNECT = "AGENT_DISCONNECTED"; const AGENTS_CONNECTED = "AGENTS_CONNECTED"; const NO_SESSIONS = "SESSION_DISCONNECTED"; const SESSION_ALREADY_CONNECTED = "SESSION_ALREADY_CONNECTED"; +const SESSION_RECONNECTED = "SESSION_RECONNECTED"; let io; const debug = process.env.debug === "1" || false; @@ -287,6 +288,7 @@ module.exports = { debug && console.log(`notifying new session about agent-existence`); let agents_ids = await get_all_agents_ids(io, socket); io.to(socket.id).emit(AGENTS_CONNECTED, agents_ids); + socket.to(socket.peerId).emit(SESSION_RECONNECTED, socket.id); } } else if (c_sessions <= 0) { diff --git a/utilities/servers/websocket.js b/utilities/servers/websocket.js index 3cd22f3fc..7371f206c 100644 --- a/utilities/servers/websocket.js +++ b/utilities/servers/websocket.js @@ -12,7 +12,7 @@ const AGENT_DISCONNECT = "AGENT_DISCONNECTED"; const AGENTS_CONNECTED = "AGENTS_CONNECTED"; const NO_SESSIONS = "SESSION_DISCONNECTED"; const SESSION_ALREADY_CONNECTED = "SESSION_ALREADY_CONNECTED"; -const SESSION_RECONNECTED = "SESSION_RECONNECTED" +const SESSION_RECONNECTED = "SESSION_RECONNECTED"; let io; const debug = process.env.debug === "1" || false; From 4fa1751a68b518368319c80410669e81da41b547 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 6 Apr 2022 16:53:49 +0200 Subject: [PATCH 052/125] fix(backend-storage): dir clean in routine; MAX_POLL_INTERVAL_MS env var --- backend/Dockerfile | 1 + backend/services/storage/clean.go | 25 +++++++++++++------------ backend/services/storage/main.go | 2 +- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 4c31518ef..b7a494f86 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -27,6 +27,7 @@ ENV TZ=UTC \ HTTP_PORT=80 \ BEACON_SIZE_LIMIT=7000000 \ KAFKA_USE_SSL=true \ + KAFKA_MAX_POLL_INTERVAL_MS=400000 \ REDIS_STREAMS_MAX_LEN=3000 \ TOPIC_RAW_WEB=raw \ TOPIC_RAW_IOS=raw-ios \ diff --git a/backend/services/storage/clean.go b/backend/services/storage/clean.go index 829bc8705..72f5f359c 100644 --- a/backend/services/storage/clean.go +++ b/backend/services/storage/clean.go @@ -1,23 +1,23 @@ package main import ( - "os" - "log" - "time" - "strconv" "io/ioutil" + "log" + "os" + "strconv" + "time" "openreplay/backend/pkg/flakeid" ) -const DELETE_TIMEOUT = 12 * time.Hour; +const DELETE_TIMEOUT = 48 * time.Hour func cleanDir(dirname string) { - files, err := ioutil.ReadDir(dirname) - if err != nil { - log.Printf("Cannot read file directory. %v", err) - return - } + files, err := ioutil.ReadDir(dirname) + if err != nil { + log.Printf("Cannot read file directory. %v", err) + return + } for _, f := range files { name := f.Name() @@ -27,8 +27,9 @@ func cleanDir(dirname string) { continue } ts := int64(flakeid.ExtractTimestamp(id)) - if time.Unix(ts/1000, 0).Add(DELETE_TIMEOUT).Before(time.Now()) { + if time.UnixMilli(ts).Add(DELETE_TIMEOUT).Before(time.Now()) { + // returns a error. Don't log it sinse it can be race condition between worker instances os.Remove(dirname + "/" + name) } } -} \ No newline at end of file +} diff --git a/backend/services/storage/main.go b/backend/services/storage/main.go index cd585f10e..9579fbe4f 100644 --- a/backend/services/storage/main.go +++ b/backend/services/storage/main.go @@ -69,7 +69,7 @@ func main() { consumer.Close() os.Exit(0) case <-cleanTick: - cleanDir(FS_DIR) + go cleanDir(FS_DIR) default: err := consumer.ConsumeNext() if err != nil { From 7e3597521c19b39bb367d7c13dc959a140ab8a2c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 6 Apr 2022 17:14:26 +0200 Subject: [PATCH 053/125] feat(db): dashboard schema for init file feat(db): dashboard schema for EE init file feat(db): dashboard schema for EE delta file --- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 80 +++++++++++++++++++ .../db/init_dbs/postgresql/init_schema.sql | 80 +++++++++++++++++-- .../db/init_dbs/postgresql/init_schema.sql | 27 ++++++- 3 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql new file mode 100644 index 000000000..15d5f314b --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -0,0 +1,80 @@ +BEGIN; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.5.5-ee' +$$ LANGUAGE sql IMMUTABLE; + + +CREATE TABLE IF NOT EXISTS dashboards +( + dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT TRUE, + is_pinned boolean NOT NULL DEFAULT FALSE, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + deleted_at timestamp NULL DEFAULT NULL +); + + +ALTER TABLE IF EXISTS metrics + DROP CONSTRAINT IF EXISTS null_project_id_for_template_only, + DROP CONSTRAINT IF EXISTS unique_key; + +ALTER TABLE IF EXISTS metrics + ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', + ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS key text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{}'::jsonb, + ALTER COLUMN project_id DROP NOT NULL, + ADD CONSTRAINT null_project_id_for_template_only + CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), + ADD CONSTRAINT unique_key UNIQUE (key); + + + +CREATE TABLE IF NOT EXISTS dashboard_widgets +( + widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, + metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + config jsonb NOT NULL DEFAULT '{}'::jsonb +); + + +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu') +ON CONFLICT (key) DO UPDATE SET name=excluded.name, + category=excluded.category, + config=excluded.config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public; + +COMMIT; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 9adab50e0..506853ffc 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -7,7 +7,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.5.4-ee' +SELECT 'v1.5.5-ee' $$ LANGUAGE sql IMMUTABLE; @@ -106,6 +106,8 @@ $$ ('assigned_sessions'), ('autocomplete'), ('basic_authentication'), + ('dashboards'), + ('dashboard_widgets'), ('errors'), ('funnels'), ('integrations'), @@ -787,22 +789,32 @@ $$ CREATE INDEX IF NOT EXISTS traces_tenant_id_idx ON traces (tenant_id); CREATE TYPE metric_type AS ENUM ('timeseries','table'); - CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart'); + CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart'); CREATE TABLE IF NOT EXISTS metrics ( metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE, user_id integer REFERENCES users (user_id) ON DELETE SET NULL, name text NOT NULL, is_public boolean NOT NULL DEFAULT FALSE, active boolean NOT NULL DEFAULT TRUE, - created_at timestamp DEFAULT timezone('utc'::text, now()) not null, + created_at timestamp default timezone('utc'::text, now()) not null, deleted_at timestamp, + edited_at timestamp, metric_type metric_type NOT NULL DEFAULT 'timeseries', view_type metric_view_type NOT NULL DEFAULT 'lineChart', metric_of text NOT NULL DEFAULT 'sessionCount', metric_value text[] NOT NULL DEFAULT '{}'::text[], - metric_format text + metric_format text, + category text NULL DEFAULT 'custom', + is_pinned boolean NOT NULL DEFAULT FALSE, + is_predefined boolean NOT NULL DEFAULT FALSE, + is_template boolean NOT NULL DEFAULT FALSE, + key text NULL DEFAULT NULL, + config jsonb NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT null_project_id_for_template_only + CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), + CONSTRAINT unique_key UNIQUE (key) ); CREATE INDEX IF NOT EXISTS metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); CREATE TABLE IF NOT EXISTS metric_series @@ -817,6 +829,29 @@ $$ ); CREATE INDEX IF NOT EXISTS metric_series_metric_id_idx ON public.metric_series (metric_id); + + CREATE TABLE dashboards + ( + dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT TRUE, + is_pinned boolean NOT NULL DEFAULT FALSE, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + deleted_at timestamp NULL DEFAULT NULL + ); + + CREATE TABLE dashboard_widgets + ( + widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, + metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + config jsonb NOT NULL DEFAULT '{}'::jsonb + ); + CREATE TABLE IF NOT EXISTS searches ( search_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, @@ -948,10 +983,13 @@ $$ CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR - first_paint_time > 0 OR - dom_content_loaded_time > 0 OR + first_paint_time > + 0 OR + dom_content_loaded_time > + 0 OR ttfb > 0 OR - time_to_interactive > 0; + time_to_interactive > + 0; CREATE INDEX IF NOT EXISTS pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); @@ -1219,5 +1257,31 @@ $$ $$ LANGUAGE plpgsql; +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu') +ON CONFLICT (key) DO UPDATE SET name=excluded.name, + category=excluded.category, + config=excluded.config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public; COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 5114d2433..8a0d2e807 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS events; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.5.4' +SELECT 'v1.5.5' $$ LANGUAGE sql IMMUTABLE; -- --- accounts.sql --- @@ -960,7 +960,8 @@ $$ key text NULL DEFAULT NULL, config jsonb NOT NULL DEFAULT '{}'::jsonb, CONSTRAINT null_project_id_for_template_only - CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ) + CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), + CONSTRAINT unique_key UNIQUE (key) ); CREATE INDEX metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); @@ -976,6 +977,28 @@ $$ ); CREATE INDEX metric_series_metric_id_idx ON public.metric_series (metric_id); + CREATE TABLE dashboards + ( + dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT TRUE, + is_pinned boolean NOT NULL DEFAULT FALSE, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + deleted_at timestamp NULL DEFAULT NULL + ); + + CREATE TABLE dashboard_widgets + ( + widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, + metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + config jsonb NOT NULL DEFAULT '{}'::jsonb + ); + CREATE TABLE searches ( From 54ef9a248b15da4d7fa9b59cc5e6bfd89834fcf4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 6 Apr 2022 17:51:07 +0200 Subject: [PATCH 054/125] feat(api): dashboard format created_at and edited_at --- api/chalicelib/core/dashboards2.py | 8 ++++++++ api/routers/core.py | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index afe65d71a..ac382f3d7 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -4,6 +4,7 @@ import schemas from chalicelib.core import custom_metrics, dashboard from chalicelib.utils import helper from chalicelib.utils import pg_client +from chalicelib.utils.TimeUTC import TimeUTC CATEGORY_DESCRIPTION = { 'categ1': 'lorem', @@ -21,6 +22,9 @@ def get_templates(project_id, user_id): rows = cur.fetchall() for r in rows: r["description"] = CATEGORY_DESCRIPTION.get(r["category"], "") + for w in r["widgets"]: + w["created_at"] = TimeUTC.datetime_to_timestamp(w["created_at"]) + w["edited_at"] = TimeUTC.datetime_to_timestamp(w["edited_at"]) return helper.list_to_camel_case(rows) @@ -85,6 +89,10 @@ def get_dashboard(project_id, user_id, dashboard_id): params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id} cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() + if row is not None: + for w in row["widgets"]: + row["created_at"] = TimeUTC.datetime_to_timestamp(w["created_at"]) + row["edited_at"] = TimeUTC.datetime_to_timestamp(w["edited_at"]) return helper.dict_to_camel_case(row) diff --git a/api/routers/core.py b/api/routers/core.py index 20864288d..06743c054 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -394,7 +394,7 @@ def delete_sumologic(projectId: int, context: schemas.CurrentContext = Depends(O def get_integration_status(context: schemas.CurrentContext = Depends(OR_context)): error, integration = integrations_manager.get_integration(tenant_id=context.tenant_id, user_id=context.user_id) - if error is not None: + if error is not None and integration is None: return {"data": {}} return {"data": integration.get_obfuscated()} @@ -406,7 +406,7 @@ def add_edit_jira_cloud(data: schemas.JiraGithubSchema = Body(...), error, integration = integrations_manager.get_integration(tool=integration_jira_cloud.PROVIDER, tenant_id=context.tenant_id, user_id=context.user_id) - if error is not None: + if error is not None and integration is None: return error data.provider = integration_jira_cloud.PROVIDER return {"data": integration.add_edit(data=data.dict())} @@ -429,7 +429,7 @@ def add_edit_github(data: schemas.JiraGithubSchema = Body(...), def delete_default_issue_tracking_tool(context: schemas.CurrentContext = Depends(OR_context)): error, integration = integrations_manager.get_integration(tenant_id=context.tenant_id, user_id=context.user_id) - if error is not None: + if error is not None and integration is None: return error return {"data": integration.delete()} From a04f8c3028ea3717cbf2102a64ba8f59685e6b8d Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 6 Apr 2022 20:13:43 +0200 Subject: [PATCH 055/125] fix(assist-frontend):handle session reconnect --- .../app/player/MessageDistributor/managers/AssistManager.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager.ts b/frontend/app/player/MessageDistributor/managers/AssistManager.ts index b684919f9..e888e18a4 100644 --- a/frontend/app/player/MessageDistributor/managers/AssistManager.ts +++ b/frontend/app/player/MessageDistributor/managers/AssistManager.ts @@ -144,7 +144,6 @@ export default class AssistManager { }) socket.on('messages', messages => { //console.log(messages.filter(m => m._id === 41 || m._id === 44)) - showDisconnectTimeout && clearTimeout(showDisconnectTimeout); jmr.append(messages) // as RawMessage[] if (waitingForMessages) { @@ -168,6 +167,9 @@ export default class AssistManager { socket.on("control_rejected", id => { id === socket.id && this.toggleRemoteControl(false) }) + socket.on('SESSION_RECONNECTED', () => { + showDisconnectTimeout && clearTimeout(showDisconnectTimeout) + }) socket.on('SESSION_DISCONNECTED', e => { waitingForMessages = true showDisconnectTimeout = setTimeout(() => { From c7d42553f23a77bd374bc4e0c0a76d0bf97669ae Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 7 Apr 2022 00:09:28 +0200 Subject: [PATCH 056/125] chore(minio): modularizing components Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/files/minio.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/helmcharts/openreplay/files/minio.sh b/scripts/helmcharts/openreplay/files/minio.sh index 1cc3e0460..903168b1b 100644 --- a/scripts/helmcharts/openreplay/files/minio.sh +++ b/scripts/helmcharts/openreplay/files/minio.sh @@ -12,9 +12,7 @@ mc alias set minio http://minio.db.svc.cluster.local:9000 $MINIO_ACCESS_KEY $MIN function init() { echo "Initializing minio" -for bucket in ${buckets[*]}; do -mc mb minio/${bucket} || true -mc ilm import minio/${bucket} < /tmp/lifecycle.json { "Rules": [ { @@ -27,13 +25,17 @@ mc ilm import minio/${bucket} < Date: Thu, 7 Apr 2022 12:23:02 +0200 Subject: [PATCH 057/125] feat(api): custom metrics & templates try schema --- api/schemas.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/schemas.py b/api/schemas.py index 65db116c0..158850618 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -818,7 +818,7 @@ class CustomMetricChartPayloadSchema(CustomMetricSessionsPayloadSchema): alias_generator = attribute_to_camel_case -class CreateCustomMetricsSchema(CustomMetricChartPayloadSchema): +class TryCustomMetricsPayloadSchema(CustomMetricChartPayloadSchema): name: str = Field(...) series: List[CustomMetricCreateSeriesSchema] = Field(...) is_public: bool = Field(default=True) @@ -859,6 +859,10 @@ class CreateCustomMetricsSchema(CustomMetricChartPayloadSchema): alias_generator = attribute_to_camel_case +class CreateCustomMetricsSchema(TryCustomMetricsPayloadSchema): + series: List[CustomMetricCreateSeriesSchema] = Field(..., min_items=1) + + class CustomMetricUpdateSeriesSchema(CustomMetricCreateSeriesSchema): series_id: Optional[int] = Field(None) From 9fcd55b879b293cc2fe1c1fc1d416fd3853b225b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 12:30:07 +0200 Subject: [PATCH 058/125] feat(api): dashboard fixed create with metrics --- api/chalicelib/core/custom_metrics.py | 4 ++-- api/chalicelib/core/dashboards2.py | 2 +- api/schemas.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 77f902dcb..c6c93a7b1 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -9,7 +9,7 @@ from chalicelib.utils.TimeUTC import TimeUTC PIE_CHART_GROUP = 5 -def __try_live(project_id, data: schemas.CreateCustomMetricsSchema): +def __try_live(project_id, data: schemas.TryCustomMetricsPayloadSchema): results = [] for i, s in enumerate(data.series): s.filter.startDate = data.startDate @@ -42,7 +42,7 @@ def __try_live(project_id, data: schemas.CreateCustomMetricsSchema): return results -def merged_live(project_id, data: schemas.CreateCustomMetricsSchema): +def merged_live(project_id, data: schemas.TryCustomMetricsPayloadSchema): series_charts = __try_live(project_id=project_id, data=data) if data.view_type == schemas.MetricTimeseriesViewType.progress or data.metric_type == schemas.MetricType.table: return series_charts diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index ac382f3d7..d6c05f057 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -215,7 +215,7 @@ def pin_dashboard(project_id, user_id, dashboard_id): def create_metric_add_widget(project_id, user_id, dashboard_id, data: schemas.CreateCustomMetricsSchema): metric_id = custom_metrics.create(project_id=project_id, user_id=user_id, data=data, dashboard=True) return add_widget(project_id=project_id, user_id=user_id, dashboard_id=dashboard_id, - data=schemas.AddWidgetToDashboardPayloadSchema(metric_id=metric_id)) + data=schemas.AddWidgetToDashboardPayloadSchema(metricId=metric_id)) PREDEFINED = {schemas.TemplateKeys.count_sessions: dashboard.get_processed_sessions, diff --git a/api/schemas.py b/api/schemas.py index 158850618..6e54af018 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -906,7 +906,7 @@ class UpdateWidgetPayloadSchema(BaseModel): class AddWidgetToDashboardPayloadSchema(UpdateWidgetPayloadSchema): - metric_id: int = Field(default=None) + metric_id: int = Field(...) class Config: alias_generator = attribute_to_camel_case From de84d88d47ceae729b2307c0bcba7a97078fd55c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 12:30:29 +0200 Subject: [PATCH 059/125] feat(api): dashboard fixed create with metrics --- api/routers/subs/metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/routers/subs/metrics.py b/api/routers/subs/metrics.py index 02eec811d..e2535433a 100644 --- a/api/routers/subs/metrics.py +++ b/api/routers/subs/metrics.py @@ -61,8 +61,7 @@ def create_metric_and_add_to_dashboard(projectId: int, dashboardId: int, data: schemas.CreateCustomMetricsSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): return {"data": dashboards2.create_metric_add_widget(project_id=projectId, user_id=context.user_id, - dashboard_id=dashboardId, - data=data)} + dashboard_id=dashboardId, data=data)} @app.post('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) From 06b8366fab1cc1acaba758721f104eab695dacd2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 12:48:02 +0200 Subject: [PATCH 060/125] feat(api): fixed /templates time conversion --- api/chalicelib/utils/TimeUTC.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/chalicelib/utils/TimeUTC.py b/api/chalicelib/utils/TimeUTC.py index bac7a027f..7befebb27 100644 --- a/api/chalicelib/utils/TimeUTC.py +++ b/api/chalicelib/utils/TimeUTC.py @@ -95,6 +95,8 @@ class TimeUTC: def datetime_to_timestamp(date): if date is None: return None + if isinstance(date, str): + return TimeUTC.human_to_timestamp(date, "%Y-%m-%dT%H:%M:%S.%f") return int(datetime.timestamp(date) * 1000) @staticmethod From f09a251c4478930b0b3ca472e71012e146a5229d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 13:28:01 +0200 Subject: [PATCH 061/125] feat(api): dashboard include series with templates --- api/chalicelib/core/dashboards2.py | 11 +++++++++-- api/chalicelib/utils/TimeUTC.py | 7 +++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index d6c05f057..37ec7a85e 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -14,8 +14,15 @@ CATEGORY_DESCRIPTION = { def get_templates(project_id, user_id): with pg_client.PostgresClient() as cur: pg_query = cur.mogrify(f"""SELECT category, jsonb_agg(metrics ORDER BY name) AS widgets - FROM metrics - WHERE deleted_at IS NULL AND (project_id ISNULL OR (project_id = %(project_id)s AND (is_public OR user_id= %(userId)s))) + FROM (SELECT * + FROM metrics LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(metric_series.* ORDER BY index), '[]'::jsonb) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + WHERE deleted_at IS NULL + AND (project_id ISNULL OR (project_id = %(project_id)s AND (is_public OR user_id= %(userId)s))) + ) AS metrics GROUP BY category ORDER BY category;""", {"project_id": project_id, "userId": user_id}) cur.execute(pg_query) diff --git a/api/chalicelib/utils/TimeUTC.py b/api/chalicelib/utils/TimeUTC.py index 7befebb27..d399e1651 100644 --- a/api/chalicelib/utils/TimeUTC.py +++ b/api/chalicelib/utils/TimeUTC.py @@ -88,7 +88,7 @@ class TimeUTC: return datetime.utcfromtimestamp(ts // 1000).strftime(fmt) @staticmethod - def human_to_timestamp(ts, pattern): + def human_to_timestamp(ts, pattern="%Y-%m-%dT%H:%M:%S.%f"): return int(datetime.strptime(ts, pattern).timestamp() * 1000) @staticmethod @@ -96,7 +96,10 @@ class TimeUTC: if date is None: return None if isinstance(date, str): - return TimeUTC.human_to_timestamp(date, "%Y-%m-%dT%H:%M:%S.%f") + fp = date.find(".") + if fp > 0: + date += '0' * (6 - len(date[fp + 1:])) + date = datetime.fromisoformat(date) return int(datetime.timestamp(date) * 1000) @staticmethod From 14e8dde249ae42233fad6bc9e0fea9e33262d3f5 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 13:44:43 +0200 Subject: [PATCH 062/125] feat(api): pg_client reconnect if PG is not available --- api/chalicelib/utils/pg_client.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/api/chalicelib/utils/pg_client.py b/api/chalicelib/utils/pg_client.py index 6379bad1e..3d60dda5c 100644 --- a/api/chalicelib/utils/pg_client.py +++ b/api/chalicelib/utils/pg_client.py @@ -1,3 +1,4 @@ +import time from threading import Semaphore import psycopg2 @@ -37,9 +38,14 @@ class ORThreadedConnectionPool(psycopg2.pool.ThreadedConnectionPool): postgreSQL_pool: ORThreadedConnectionPool = None +RETRY_MAX = config("PG_RETRY_MAX", cast=int, default=50) +RETRY_INTERVAL = config("PG_RETRY_INTERVAL", cast=int, default=2) +RETRY = 0 + def make_pool(): global postgreSQL_pool + global RETRY if postgreSQL_pool is not None: try: postgreSQL_pool.closeall() @@ -51,7 +57,13 @@ def make_pool(): print("Connection pool created successfully") except (Exception, psycopg2.DatabaseError) as error: print("Error while connecting to PostgreSQL", error) - raise error + if RETRY < RETRY_MAX: + RETRY += 1 + print(f"waiting for {RETRY_INTERVAL}s before retry n°{RETRY}") + time.sleep(RETRY_INTERVAL) + make_pool() + else: + raise error make_pool() From f423e9c9cfb30a635e33820c24fef40ca0a9ff0d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 13:48:07 +0200 Subject: [PATCH 063/125] feat(api): pg_client reconnect config --- api/.env.default | 2 ++ ee/api/.env.default | 2 ++ 2 files changed, 4 insertions(+) diff --git a/api/.env.default b/api/.env.default index 6ae959a7d..0d4f8130c 100644 --- a/api/.env.default +++ b/api/.env.default @@ -37,6 +37,8 @@ pg_port=5432 pg_user=postgres pg_timeout=30 pg_minconn=45 +PG_RETRY_MAX=50 +PG_RETRY_INTERVAL=2 put_S3_TTL=20 sentryURL= sessions_bucket=mobs diff --git a/ee/api/.env.default b/ee/api/.env.default index 28f46f273..778a8f32c 100644 --- a/ee/api/.env.default +++ b/ee/api/.env.default @@ -46,6 +46,8 @@ pg_port=5432 pg_user=postgres pg_timeout=30 pg_minconn=45 +PG_RETRY_MAX=50 +PG_RETRY_INTERVAL=2 put_S3_TTL=20 sentryURL= sessions_bucket=mobs From ed51fc09cb950b53b2f49d77ac2f1488d9c4a0c2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 14:21:50 +0200 Subject: [PATCH 064/125] feat(api): dashboard category description feat(db): changed metrics to support overview metric-type --- api/chalicelib/core/authorizers.py | 4 +- api/chalicelib/core/dashboards2.py | 3 +- api/routers/subs/metrics.py | 2 +- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 49 ++++++++++--------- .../db/init_dbs/postgresql/init_schema.sql | 45 ++++++++--------- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 49 ++++++++++--------- .../db/init_dbs/postgresql/init_schema.sql | 45 ++++++++--------- 7 files changed, 101 insertions(+), 96 deletions(-) diff --git a/api/chalicelib/core/authorizers.py b/api/chalicelib/core/authorizers.py index 33a859cc8..899fd046f 100644 --- a/api/chalicelib/core/authorizers.py +++ b/api/chalicelib/core/authorizers.py @@ -13,9 +13,9 @@ def jwt_authorizer(token): try: payload = jwt.decode( token[1], - config("jwt_secret"), + "", algorithms=config("jwt_algorithm"), - audience=[f"plugin:{helper.get_stage_name()}", f"front:{helper.get_stage_name()}"] + audience=[ f"front:default-foss"] ) except jwt.ExpiredSignatureError: print("! JWT Expired signature") diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 37ec7a85e..868bb9c4a 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -7,7 +7,8 @@ from chalicelib.utils import pg_client from chalicelib.utils.TimeUTC import TimeUTC CATEGORY_DESCRIPTION = { - 'categ1': 'lorem', + 'overview': 'lorem ipsum', + 'custom': 'lorem cusipsum', } diff --git a/api/routers/subs/metrics.py b/api/routers/subs/metrics.py index e2535433a..0a806b146 100644 --- a/api/routers/subs/metrics.py +++ b/api/routers/subs/metrics.py @@ -100,7 +100,7 @@ def get_templates(projectId: int, context: schemas.CurrentContext = Depends(OR_c @app.put('/{projectId}/metrics/try', tags=["dashboard"]) @app.post('/{projectId}/custom_metrics/try', tags=["customMetrics"]) @app.put('/{projectId}/custom_metrics/try', tags=["customMetrics"]) -def try_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), +def try_custom_metric(projectId: int, data: schemas.TryCustomMetricsPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): return {"data": custom_metrics.merged_live(project_id=projectId, data=data)} diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 15d5f314b..1472e1779 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -48,33 +48,34 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets config jsonb NOT NULL DEFAULT '{}'::jsonb ); +COMMIT; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; +ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'overview'; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu') +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key, metric_type) +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'overview'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'overview'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'overview'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'overview'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'overview'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'overview'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'overview'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'overview'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'overview'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'overview'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'overview'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'overview'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'overview'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'overview'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'overview'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'overview'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'overview'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'overview'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'overview') ON CONFLICT (key) DO UPDATE SET name=excluded.name, category=excluded.category, config=excluded.config, is_predefined=excluded.is_predefined, is_template=excluded.is_template, - is_public=excluded.is_public; - -COMMIT; -ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; \ No newline at end of file + is_public=excluded.is_public, + metric_type=excluded.metric_type; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 506853ffc..ea60c70ec 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -788,7 +788,7 @@ $$ CREATE INDEX IF NOT EXISTS traces_user_id_idx ON traces (user_id); CREATE INDEX IF NOT EXISTS traces_tenant_id_idx ON traces (tenant_id); - CREATE TYPE metric_type AS ENUM ('timeseries','table'); + CREATE TYPE metric_type AS ENUM ('timeseries','table', 'overview'); CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart'); CREATE TABLE IF NOT EXISTS metrics ( @@ -1257,31 +1257,32 @@ $$ $$ LANGUAGE plpgsql; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu') +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key, metric_type) +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'overview'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'overview'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'overview'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'overview'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'overview'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'overview'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'overview'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'overview'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'overview'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'overview'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'overview'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'overview'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'overview'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'overview'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'overview'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'overview'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'overview'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'overview'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'overview') ON CONFLICT (key) DO UPDATE SET name=excluded.name, category=excluded.category, config=excluded.config, is_predefined=excluded.is_predefined, is_template=excluded.is_template, - is_public=excluded.is_public; + is_public=excluded.is_public, + metric_type=excluded.metric_type; COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 1a3a918ed..a511090d1 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -48,33 +48,34 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets config jsonb NOT NULL DEFAULT '{}'::jsonb ); +COMMIT; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; +ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'overview'; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu') +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key, metric_type) +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'overview'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'overview'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'overview'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'overview'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'overview'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'overview'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'overview'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'overview'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'overview'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'overview'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'overview'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'overview'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'overview'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'overview'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'overview'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'overview'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'overview'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'overview'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'overview') ON CONFLICT (key) DO UPDATE SET name=excluded.name, category=excluded.category, config=excluded.config, is_predefined=excluded.is_predefined, is_template=excluded.is_template, - is_public=excluded.is_public; - -COMMIT; -ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; \ No newline at end of file + is_public=excluded.is_public, + metric_type=excluded.metric_type; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 8a0d2e807..838aa53cd 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -935,7 +935,7 @@ $$ CREATE INDEX jobs_start_at_idx ON jobs (start_at); CREATE INDEX jobs_project_id_idx ON jobs (project_id); - CREATE TYPE metric_type AS ENUM ('timeseries','table'); + CREATE TYPE metric_type AS ENUM ('timeseries','table', 'overview'); CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart'); CREATE TABLE metrics ( @@ -1048,31 +1048,32 @@ $$ $$ LANGUAGE plpgsql; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu') +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key, metric_type) +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'overview'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'overview'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'overview'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'overview'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'overview'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'overview'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'overview'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'overview'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'overview'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'overview'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'overview'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'overview'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'overview'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'overview'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'overview'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'overview'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'overview'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'overview'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'overview') ON CONFLICT (key) DO UPDATE SET name=excluded.name, category=excluded.category, config=excluded.config, is_predefined=excluded.is_predefined, is_template=excluded.is_template, - is_public=excluded.is_public; + is_public=excluded.is_public, + metric_type=excluded.metric_type; COMMIT; \ No newline at end of file From 48052317954c576fe8fae302553ac2b0ace7f2df Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 15:39:51 +0200 Subject: [PATCH 065/125] feat(api): dashboard defined new categories feat(api): changed Templates schemas feat(db): changed metrics structure feat(db): changed metric_view_type --- api/chalicelib/core/dashboards2.py | 45 ++++---- api/routers/subs/dashboard.py | 42 ++++---- api/schemas.py | 4 +- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 80 +++++++------- .../db/init_dbs/postgresql/init_schema.sql | 102 +++++++++--------- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 79 +++++++------- .../db/init_dbs/postgresql/init_schema.sql | 102 +++++++++--------- 7 files changed, 238 insertions(+), 216 deletions(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 868bb9c4a..ed9a17d76 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -9,6 +9,9 @@ from chalicelib.utils.TimeUTC import TimeUTC CATEGORY_DESCRIPTION = { 'overview': 'lorem ipsum', 'custom': 'lorem cusipsum', + 'errors': 'lorem erripsum', + 'performance': 'lorem perfipsum', + 'resources': 'lorem resipsum' } @@ -226,29 +229,29 @@ def create_metric_add_widget(project_id, user_id, dashboard_id, data: schemas.Cr data=schemas.AddWidgetToDashboardPayloadSchema(metricId=metric_id)) -PREDEFINED = {schemas.TemplateKeys.count_sessions: dashboard.get_processed_sessions, - schemas.TemplateKeys.avg_image_load_time: dashboard.get_application_activity_avg_image_load_time, - schemas.TemplateKeys.avg_page_load_time: dashboard.get_application_activity_avg_page_load_time, - schemas.TemplateKeys.avg_request_load_time: dashboard.get_application_activity_avg_request_load_time, - schemas.TemplateKeys.avg_dom_content_load_start: dashboard.get_page_metrics_avg_dom_content_load_start, - schemas.TemplateKeys.avg_first_contentful_pixel: dashboard.get_page_metrics_avg_first_contentful_pixel, - schemas.TemplateKeys.avg_visited_pages: dashboard.get_user_activity_avg_visited_pages, - schemas.TemplateKeys.avg_session_duration: dashboard.get_user_activity_avg_session_duration, - schemas.TemplateKeys.avg_pages_dom_buildtime: dashboard.get_pages_dom_build_time, - schemas.TemplateKeys.avg_pages_response_time: dashboard.get_pages_response_time, - schemas.TemplateKeys.avg_response_time: dashboard.get_top_metrics_avg_response_time, - schemas.TemplateKeys.avg_first_paint: dashboard.get_top_metrics_avg_first_paint, - schemas.TemplateKeys.avg_dom_content_loaded: dashboard.get_top_metrics_avg_dom_content_loaded, - schemas.TemplateKeys.avg_till_first_bit: dashboard.get_top_metrics_avg_till_first_bit, - schemas.TemplateKeys.avg_time_to_interactive: dashboard.get_top_metrics_avg_time_to_interactive, - schemas.TemplateKeys.count_requests: dashboard.get_top_metrics_count_requests, - schemas.TemplateKeys.avg_time_to_render: dashboard.get_time_to_render, - schemas.TemplateKeys.avg_used_js_heap_size: dashboard.get_memory_consumption, - schemas.TemplateKeys.avg_cpu: dashboard.get_avg_cpu, - schemas.TemplateKeys.avg_fps: dashboard.get_avg_fps} +PREDEFINED = {schemas.TemplatePredefinedKeys.count_sessions: dashboard.get_processed_sessions, + schemas.TemplatePredefinedKeys.avg_image_load_time: dashboard.get_application_activity_avg_image_load_time, + schemas.TemplatePredefinedKeys.avg_page_load_time: dashboard.get_application_activity_avg_page_load_time, + schemas.TemplatePredefinedKeys.avg_request_load_time: dashboard.get_application_activity_avg_request_load_time, + schemas.TemplatePredefinedKeys.avg_dom_content_load_start: dashboard.get_page_metrics_avg_dom_content_load_start, + schemas.TemplatePredefinedKeys.avg_first_contentful_pixel: dashboard.get_page_metrics_avg_first_contentful_pixel, + schemas.TemplatePredefinedKeys.avg_visited_pages: dashboard.get_user_activity_avg_visited_pages, + schemas.TemplatePredefinedKeys.avg_session_duration: dashboard.get_user_activity_avg_session_duration, + schemas.TemplatePredefinedKeys.avg_pages_dom_buildtime: dashboard.get_pages_dom_build_time, + schemas.TemplatePredefinedKeys.avg_pages_response_time: dashboard.get_pages_response_time, + schemas.TemplatePredefinedKeys.avg_response_time: dashboard.get_top_metrics_avg_response_time, + schemas.TemplatePredefinedKeys.avg_first_paint: dashboard.get_top_metrics_avg_first_paint, + schemas.TemplatePredefinedKeys.avg_dom_content_loaded: dashboard.get_top_metrics_avg_dom_content_loaded, + schemas.TemplatePredefinedKeys.avg_till_first_bit: dashboard.get_top_metrics_avg_till_first_bit, + schemas.TemplatePredefinedKeys.avg_time_to_interactive: dashboard.get_top_metrics_avg_time_to_interactive, + schemas.TemplatePredefinedKeys.count_requests: dashboard.get_top_metrics_count_requests, + schemas.TemplatePredefinedKeys.avg_time_to_render: dashboard.get_time_to_render, + schemas.TemplatePredefinedKeys.avg_used_js_heap_size: dashboard.get_memory_consumption, + schemas.TemplatePredefinedKeys.avg_cpu: dashboard.get_avg_cpu, + schemas.TemplatePredefinedKeys.avg_fps: dashboard.get_avg_fps} -def get_predefined_metric(key: schemas.TemplateKeys, project_id: int, data: dict): +def get_predefined_metric(key: schemas.TemplatePredefinedKeys, project_id: int, data: dict): return PREDEFINED.get(key, lambda *args: None)(project_id=project_id, **data) diff --git a/api/routers/subs/dashboard.py b/api/routers/subs/dashboard.py index 0832be4b8..e2d4ba268 100644 --- a/api/routers/subs/dashboard.py +++ b/api/routers/subs/dashboard.py @@ -342,7 +342,7 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body {"key": "avg_time_to_render", "data": dashboard.get_time_to_render(project_id=projectId, **data.dict())}, {"key": "avg_used_js_heap_size", "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, {"key": "avg_cpu", "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_fps, "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} + {"key": schemas.TemplatePredefinedKeys.avg_fps, "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} ] results = sorted(results, key=lambda r: r["key"]) return {"data": results} @@ -352,45 +352,45 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body @app.get('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body(...)): results = [ - {"key": schemas.TemplateKeys.count_sessions, + {"key": schemas.TemplatePredefinedKeys.count_sessions, "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_image_load_time, + {"key": schemas.TemplatePredefinedKeys.avg_image_load_time, "data": dashboard.get_application_activity_avg_image_load_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_page_load_time, + {"key": schemas.TemplatePredefinedKeys.avg_page_load_time, "data": dashboard.get_application_activity_avg_page_load_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_request_load_time, + {"key": schemas.TemplatePredefinedKeys.avg_request_load_time, "data": dashboard.get_application_activity_avg_request_load_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_dom_content_load_start, + {"key": schemas.TemplatePredefinedKeys.avg_dom_content_load_start, "data": dashboard.get_page_metrics_avg_dom_content_load_start(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_first_contentful_pixel, + {"key": schemas.TemplatePredefinedKeys.avg_first_contentful_pixel, "data": dashboard.get_page_metrics_avg_first_contentful_pixel(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_visited_pages, + {"key": schemas.TemplatePredefinedKeys.avg_visited_pages, "data": dashboard.get_user_activity_avg_visited_pages(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_session_duration, + {"key": schemas.TemplatePredefinedKeys.avg_session_duration, "data": dashboard.get_user_activity_avg_session_duration(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_pages_dom_buildtime, + {"key": schemas.TemplatePredefinedKeys.avg_pages_dom_buildtime, "data": dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_pages_response_time, + {"key": schemas.TemplatePredefinedKeys.avg_pages_response_time, "data": dashboard.get_pages_response_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_response_time, + {"key": schemas.TemplatePredefinedKeys.avg_response_time, "data": dashboard.get_top_metrics_avg_response_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_first_paint, + {"key": schemas.TemplatePredefinedKeys.avg_first_paint, "data": dashboard.get_top_metrics_avg_first_paint(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_dom_content_loaded, + {"key": schemas.TemplatePredefinedKeys.avg_dom_content_loaded, "data": dashboard.get_top_metrics_avg_dom_content_loaded(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_till_first_bit, + {"key": schemas.TemplatePredefinedKeys.avg_till_first_bit, "data": dashboard.get_top_metrics_avg_till_first_bit(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_time_to_interactive, + {"key": schemas.TemplatePredefinedKeys.avg_time_to_interactive, "data": dashboard.get_top_metrics_avg_time_to_interactive(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.count_requests, + {"key": schemas.TemplatePredefinedKeys.count_requests, "data": dashboard.get_top_metrics_count_requests(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_time_to_render, + {"key": schemas.TemplatePredefinedKeys.avg_time_to_render, "data": dashboard.get_time_to_render(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_used_js_heap_size, + {"key": schemas.TemplatePredefinedKeys.avg_used_js_heap_size, "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_cpu, + {"key": schemas.TemplatePredefinedKeys.avg_cpu, "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, - {"key": schemas.TemplateKeys.avg_fps, + {"key": schemas.TemplatePredefinedKeys.avg_fps, "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} ] results = sorted(results, key=lambda r: r["key"]) diff --git a/api/schemas.py b/api/schemas.py index 6e54af018..7dd7774ba 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -913,7 +913,7 @@ class AddWidgetToDashboardPayloadSchema(UpdateWidgetPayloadSchema): # these values should match the keys in metrics table -class TemplateKeys(str, Enum): +class TemplatePredefinedKeys(str, Enum): count_sessions = "count_sessions" avg_request_load_time = "avg_request_load_time" avg_page_load_time = "avg_page_load_time" @@ -939,7 +939,7 @@ class TemplateKeys(str, Enum): class CustomMetricAndTemplate(BaseModel): is_template: bool = Field(...) project_id: Optional[int] = Field(...) - key: Optional[TemplateKeys] = Field(...) + key: Optional[TemplatePredefinedKeys] = Field(...) class Config: alias_generator = attribute_to_camel_case diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 1472e1779..1ba9cc0af 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -24,17 +24,17 @@ ALTER TABLE IF EXISTS metrics DROP CONSTRAINT IF EXISTS unique_key; ALTER TABLE IF EXISTS metrics - ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, - ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', - ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS key text NULL DEFAULT NULL, - ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', + ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{}'::jsonb, ALTER COLUMN project_id DROP NOT NULL, ADD CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), - ADD CONSTRAINT unique_key UNIQUE (key); + ADD CONSTRAINT unique_key UNIQUE (predefined_key); @@ -50,32 +50,40 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets COMMIT; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; -ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'overview'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'barChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarLineChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'overview'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'map'; +ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'predefined'; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key, metric_type) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'overview'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'overview'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'overview'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'overview'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'overview'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'overview'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'overview'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'overview'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'overview'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'overview'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'overview'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'overview'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'overview'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'overview'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'overview'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'overview'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'overview'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'overview') -ON CONFLICT (key) DO UPDATE SET name=excluded.name, - category=excluded.category, - config=excluded.config, - is_predefined=excluded.is_predefined, - is_template=excluded.is_template, - is_public=excluded.is_public, - metric_type=excluded.metric_type; \ No newline at end of file + +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'predefined', 'overview'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview') +ON CONFLICT (predefined_key) DO UPDATE + SET name=excluded.name, + category=excluded.category, + config=excluded.config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public, + metric_type=excluded.metric_type, + view_type=excluded.view_type; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index ea60c70ec..6a0258af7 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -788,33 +788,33 @@ $$ CREATE INDEX IF NOT EXISTS traces_user_id_idx ON traces (user_id); CREATE INDEX IF NOT EXISTS traces_tenant_id_idx ON traces (tenant_id); - CREATE TYPE metric_type AS ENUM ('timeseries','table', 'overview'); - CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart'); + CREATE TYPE metric_type AS ENUM ('timeseries','table', 'predefined'); + CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart','barChart','stackedBarChart','stackedBarLineChart','overview','map'); CREATE TABLE IF NOT EXISTS metrics ( - metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE, - user_id integer REFERENCES users (user_id) ON DELETE SET NULL, - name text NOT NULL, - is_public boolean NOT NULL DEFAULT FALSE, - active boolean NOT NULL DEFAULT TRUE, - created_at timestamp default timezone('utc'::text, now()) not null, - deleted_at timestamp, - edited_at timestamp, - metric_type metric_type NOT NULL DEFAULT 'timeseries', - view_type metric_view_type NOT NULL DEFAULT 'lineChart', - metric_of text NOT NULL DEFAULT 'sessionCount', - metric_value text[] NOT NULL DEFAULT '{}'::text[], - metric_format text, - category text NULL DEFAULT 'custom', - is_pinned boolean NOT NULL DEFAULT FALSE, - is_predefined boolean NOT NULL DEFAULT FALSE, - is_template boolean NOT NULL DEFAULT FALSE, - key text NULL DEFAULT NULL, - config jsonb NOT NULL DEFAULT '{}'::jsonb, + metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT FALSE, + active boolean NOT NULL DEFAULT TRUE, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp, + edited_at timestamp, + metric_type metric_type NOT NULL DEFAULT 'timeseries', + view_type metric_view_type NOT NULL DEFAULT 'lineChart', + metric_of text NOT NULL DEFAULT 'sessionCount', + metric_value text[] NOT NULL DEFAULT '{}'::text[], + metric_format text, + category text NULL DEFAULT 'custom', + is_pinned boolean NOT NULL DEFAULT FALSE, + is_predefined boolean NOT NULL DEFAULT FALSE, + is_template boolean NOT NULL DEFAULT FALSE, + predefined_key text NULL DEFAULT NULL, + config jsonb NOT NULL DEFAULT '{}'::jsonb, CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), - CONSTRAINT unique_key UNIQUE (key) + CONSTRAINT unique_key UNIQUE (predefined_key) ); CREATE INDEX IF NOT EXISTS metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); CREATE TABLE IF NOT EXISTS metric_series @@ -1257,32 +1257,34 @@ $$ $$ LANGUAGE plpgsql; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key, metric_type) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'overview'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'overview'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'overview'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'overview'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'overview'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'overview'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'overview'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'overview'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'overview'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'overview'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'overview'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'overview'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'overview'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'overview'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'overview'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'overview'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'overview'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'overview') -ON CONFLICT (key) DO UPDATE SET name=excluded.name, - category=excluded.category, - config=excluded.config, - is_predefined=excluded.is_predefined, - is_template=excluded.is_template, - is_public=excluded.is_public, - metric_type=excluded.metric_type; +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'predefined', 'overview'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview') +ON CONFLICT (predefined_key) DO UPDATE + SET name=excluded.name, + category=excluded.category, + config=excluded.config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public, + metric_type=excluded.metric_type, + view_type=excluded.view_type; COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index a511090d1..5afd50a77 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -24,17 +24,17 @@ ALTER TABLE IF EXISTS metrics DROP CONSTRAINT IF EXISTS unique_key; ALTER TABLE IF EXISTS metrics - ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, - ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', - ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS key text NULL DEFAULT NULL, - ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', + ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{}'::jsonb, ALTER COLUMN project_id DROP NOT NULL, ADD CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), - ADD CONSTRAINT unique_key UNIQUE (key); + ADD CONSTRAINT unique_key UNIQUE (predefined_key); @@ -50,32 +50,39 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets COMMIT; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; -ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'overview'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'barChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarLineChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'overview'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'map'; +ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'predefined'; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key, metric_type) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'overview'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'overview'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'overview'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'overview'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'overview'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'overview'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'overview'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'overview'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'overview'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'overview'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'overview'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'overview'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'overview'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'overview'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'overview'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'overview'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'overview'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'overview') -ON CONFLICT (key) DO UPDATE SET name=excluded.name, - category=excluded.category, - config=excluded.config, - is_predefined=excluded.is_predefined, - is_template=excluded.is_template, - is_public=excluded.is_public, - metric_type=excluded.metric_type; \ No newline at end of file +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'predefined', 'overview'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview') +ON CONFLICT (predefined_key) DO UPDATE + SET name=excluded.name, + category=excluded.category, + config=excluded.config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public, + metric_type=excluded.metric_type, + view_type=excluded.view_type; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 838aa53cd..3278a48b7 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -935,33 +935,33 @@ $$ CREATE INDEX jobs_start_at_idx ON jobs (start_at); CREATE INDEX jobs_project_id_idx ON jobs (project_id); - CREATE TYPE metric_type AS ENUM ('timeseries','table', 'overview'); - CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart'); + CREATE TYPE metric_type AS ENUM ('timeseries','table', 'predefined'); + CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart','barChart','stackedBarChart','stackedBarLineChart','overview','map'); CREATE TABLE metrics ( - metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE, - user_id integer REFERENCES users (user_id) ON DELETE SET NULL, - name text NOT NULL, - is_public boolean NOT NULL DEFAULT FALSE, - active boolean NOT NULL DEFAULT TRUE, - created_at timestamp default timezone('utc'::text, now()) not null, - deleted_at timestamp, - edited_at timestamp, - metric_type metric_type NOT NULL DEFAULT 'timeseries', - view_type metric_view_type NOT NULL DEFAULT 'lineChart', - metric_of text NOT NULL DEFAULT 'sessionCount', - metric_value text[] NOT NULL DEFAULT '{}'::text[], - metric_format text, - category text NULL DEFAULT 'custom', - is_pinned boolean NOT NULL DEFAULT FALSE, - is_predefined boolean NOT NULL DEFAULT FALSE, - is_template boolean NOT NULL DEFAULT FALSE, - key text NULL DEFAULT NULL, - config jsonb NOT NULL DEFAULT '{}'::jsonb, + metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT FALSE, + active boolean NOT NULL DEFAULT TRUE, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp, + edited_at timestamp, + metric_type metric_type NOT NULL DEFAULT 'timeseries', + view_type metric_view_type NOT NULL DEFAULT 'lineChart', + metric_of text NOT NULL DEFAULT 'sessionCount', + metric_value text[] NOT NULL DEFAULT '{}'::text[], + metric_format text, + category text NULL DEFAULT 'custom', + is_pinned boolean NOT NULL DEFAULT FALSE, + is_predefined boolean NOT NULL DEFAULT FALSE, + is_template boolean NOT NULL DEFAULT FALSE, + predefined_key text NULL DEFAULT NULL, + config jsonb NOT NULL DEFAULT '{}'::jsonb, CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), - CONSTRAINT unique_key UNIQUE (key) + CONSTRAINT unique_key UNIQUE (predefined_key) ); CREATE INDEX metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); @@ -1048,32 +1048,34 @@ $$ $$ LANGUAGE plpgsql; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key, metric_type) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'overview'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'overview'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'overview'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'overview'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'overview'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'overview'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'overview'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'overview'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'overview'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'overview'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'overview'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'overview'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'overview'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'overview'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'overview'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'overview'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'overview'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'overview') -ON CONFLICT (key) DO UPDATE SET name=excluded.name, - category=excluded.category, - config=excluded.config, - is_predefined=excluded.is_predefined, - is_template=excluded.is_template, - is_public=excluded.is_public, - metric_type=excluded.metric_type; +INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'predefined', 'overview'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview') +ON CONFLICT (predefined_key) DO UPDATE + SET name=excluded.name, + category=excluded.category, + config=excluded.config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public, + metric_type=excluded.metric_type, + view_type=excluded.view_type; COMMIT; \ No newline at end of file From 91e7a4103e62ddde31283683fc269dc3e8798c01 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 15:45:44 +0200 Subject: [PATCH 066/125] feat(api): dashboard fixed new predefined key issue --- api/chalicelib/core/custom_metrics.py | 21 ++++++++++++--------- api/chalicelib/core/dashboards2.py | 8 +++++--- api/schemas.py | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index c6c93a7b1..a43de6903 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -309,24 +309,27 @@ def get(metric_id, project_id, user_id, flatten=True): return helper.dict_to_camel_case(row) -def get_with_template(metric_id, project_id, user_id): +def get_with_template(metric_id, project_id, user_id, include_dashboard=True): with pg_client.PostgresClient() as cur: + sub_query="" + if include_dashboard: + sub_query="""LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards + FROM (SELECT dashboard_id, name, is_public + FROM dashboards + WHERE deleted_at ISNULL + AND project_id = %(project_id)s + AND ((user_id = %(user_id)s OR is_public))) AS connected_dashboards + ) AS connected_dashboards ON (TRUE)""" cur.execute( cur.mogrify( - """SELECT * + f"""SELECT * FROM metrics LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(metric_series.* ORDER BY index),'[]'::jsonb) AS series FROM metric_series WHERE metric_series.metric_id = metrics.metric_id AND metric_series.deleted_at ISNULL ) AS metric_series ON (TRUE) - LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards - FROM (SELECT dashboard_id, name, is_public - FROM dashboards - WHERE deleted_at ISNULL - AND project_id = %(project_id)s - AND ((user_id = %(user_id)s OR is_public))) AS connected_dashboards - ) AS connected_dashboards ON (TRUE) + {sub_query} WHERE (metrics.project_id = %(project_id)s OR metrics.project_id ISNULL) AND metrics.deleted_at ISNULL AND (metrics.user_id = %(user_id)s OR metrics.is_public) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index ed9a17d76..aa188bb1f 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -256,12 +256,14 @@ def get_predefined_metric(key: schemas.TemplatePredefinedKeys, project_id: int, def make_chart_metrics(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema): - raw_metric = custom_metrics.get_with_template(metric_id=metric_id, project_id=project_id, user_id=user_id) + raw_metric = custom_metrics.get_with_template(metric_id=metric_id, project_id=project_id, user_id=user_id, + include_dashboard=False) if raw_metric is None: return None + print(raw_metric) metric = schemas.CustomMetricAndTemplate = schemas.CustomMetricAndTemplate.parse_obj(raw_metric) if metric.is_template: - return get_predefined_metric(key=metric.key, project_id=project_id, data=data.dict()) + return get_predefined_metric(key=metric.predefined_key, project_id=project_id, data=data.dict()) else: return custom_metrics.make_chart(project_id=project_id, user_id=user_id, metric_id=metric_id, data=data, metric=raw_metric) @@ -273,7 +275,7 @@ def make_chart_widget(dashboard_id, project_id, user_id, widget_id, data: schema return None metric = schemas.CustomMetricAndTemplate = schemas.CustomMetricAndTemplate.parse_obj(raw_metric) if metric.is_template: - return get_predefined_metric(key=metric.key, project_id=project_id, data=data.dict()) + return get_predefined_metric(key=metric.predefined_key, project_id=project_id, data=data.dict()) else: return custom_metrics.make_chart(project_id=project_id, user_id=user_id, metric_id=raw_metric["metricId"], data=data, metric=raw_metric) diff --git a/api/schemas.py b/api/schemas.py index 7dd7774ba..e518b9914 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -939,7 +939,7 @@ class TemplatePredefinedKeys(str, Enum): class CustomMetricAndTemplate(BaseModel): is_template: bool = Field(...) project_id: Optional[int] = Field(...) - key: Optional[TemplatePredefinedKeys] = Field(...) + predefined_key: Optional[TemplatePredefinedKeys] = Field(...) class Config: alias_generator = attribute_to_camel_case From f26fbc9a71a40b4509575e487e4e570a5ba00314 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 16:25:50 +0200 Subject: [PATCH 067/125] feat(db): defined new predefined templates (errors) feat(api): support for errors predefined templates feat(api): fixed update widget config --- api/chalicelib/core/dashboard.py | 27 ++++++++++++++----- api/chalicelib/core/dashboards2.py | 14 +++++++--- api/schemas.py | 16 +++++++++++ .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 9 ++++++- .../db/init_dbs/postgresql/init_schema.sql | 9 ++++++- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 9 ++++++- .../db/init_dbs/postgresql/init_schema.sql | 9 ++++++- 7 files changed, 80 insertions(+), 13 deletions(-) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index 6de949047..362e4c491 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -170,7 +170,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) count = cur.fetchone()["count"] results["progress"] = helper.__progress(old_val=count, new_val=results["value"]) - + results["unit"] = schemas.TemplatePredefinedUnits.count return results @@ -998,6 +998,7 @@ def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=- cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond return row @@ -1156,7 +1157,7 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"value": avg, "chart": rows} + return {"value": avg, "chart": rows, "unit": schemas.TemplatePredefinedUnits.millisecond} def get_pages_response_time_distribution(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1378,6 +1379,7 @@ def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), "endTimestamp": endTimestamp, "value": url, **__get_constraint_values(args)} cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond return row @@ -1448,7 +1450,7 @@ def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1) WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"value": avg, "chart": helper.list_to_camel_case(rows)} + return {"value": avg, "chart": helper.list_to_camel_case(rows), "unit": schemas.TemplatePredefinedUnits.memory} def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1480,7 +1482,8 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"value": avg, "chart": helper.list_to_camel_case(rows)} + return {"value": avg, "chart": helper.list_to_camel_case(rows), + "unit": schemas.TemplatePredefinedUnits.percentage} def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1512,7 +1515,7 @@ def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"value": avg, "chart": helper.list_to_camel_case(rows)} + return {"value": avg, "chart": helper.list_to_camel_case(rows), "unit": schemas.TemplatePredefinedUnits.frame} def get_crashes(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -2269,7 +2272,7 @@ def get_application_activity_avg_image_load_time(project_id, startTimestamp=Time row = __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) previous = helper.dict_to_camel_case(row) results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) - + results["unit"] = schemas.TemplatePredefinedUnits.millisecond return results @@ -2345,6 +2348,7 @@ def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeU row = __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) previous = helper.dict_to_camel_case(row) results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond return results @@ -2414,6 +2418,7 @@ def get_application_activity_avg_request_load_time(project_id, startTimestamp=Ti row = __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) previous = helper.dict_to_camel_case(row) results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond return results @@ -2475,6 +2480,7 @@ def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeU if len(rows) > 0: previous = helper.dict_to_camel_case(rows[0]) results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond return results @@ -2509,6 +2515,7 @@ def get_page_metrics_avg_first_contentful_pixel(project_id, startTimestamp=TimeU if len(rows) > 0: previous = helper.dict_to_camel_case(rows[0]) results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond return results @@ -2542,6 +2549,7 @@ def get_user_activity_avg_visited_pages(project_id, startTimestamp=TimeUTC.now(d previous = helper.dict_to_camel_case(row) results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.count return results @@ -2572,6 +2580,7 @@ def get_user_activity_avg_session_duration(project_id, startTimestamp=TimeUTC.no previous = helper.dict_to_camel_case(row) results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond return results @@ -2609,6 +2618,7 @@ def get_top_metrics_avg_response_time(project_id, startTimestamp=TimeUTC.now(del "endTimestamp": endTimestamp, "value": value, **__get_constraint_values(args)})) row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond return helper.dict_to_camel_case(row) @@ -2631,6 +2641,7 @@ def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta "endTimestamp": endTimestamp, "value": value, **__get_constraint_values(args)})) row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond return helper.dict_to_camel_case(row) @@ -2653,6 +2664,7 @@ def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.no "endTimestamp": endTimestamp, "value": value, **__get_constraint_values(args)})) row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond return helper.dict_to_camel_case(row) @@ -2675,6 +2687,7 @@ def get_top_metrics_avg_till_first_bit(project_id, startTimestamp=TimeUTC.now(de "endTimestamp": endTimestamp, "value": value, **__get_constraint_values(args)})) row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond return helper.dict_to_camel_case(row) @@ -2697,6 +2710,7 @@ def get_top_metrics_avg_time_to_interactive(project_id, startTimestamp=TimeUTC.n "endTimestamp": endTimestamp, "value": value, **__get_constraint_values(args)})) row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond return helper.dict_to_camel_case(row) @@ -2715,4 +2729,5 @@ def get_top_metrics_count_requests(project_id, startTimestamp=TimeUTC.now(delta_ "endTimestamp": endTimestamp, "value": value, **__get_constraint_values(args)})) row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.count return helper.dict_to_camel_case(row) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index aa188bb1f..594db2fd0 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -191,9 +191,10 @@ def update_widget(project_id, user_id, dashboard_id, widget_id, data: schemas.Up pg_query = """UPDATE dashboard_widgets SET config= %(config)s WHERE dashboard_id=%(dashboard_id)s AND widget_id=%(widget_id)s - RETURNINIG *;""" + RETURNING *;""" params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, "widget_id": widget_id, **data.dict()} + params["config"] = json.dumps(data.config) cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() return helper.dict_to_camel_case(row) @@ -248,7 +249,15 @@ PREDEFINED = {schemas.TemplatePredefinedKeys.count_sessions: dashboard.get_proce schemas.TemplatePredefinedKeys.avg_time_to_render: dashboard.get_time_to_render, schemas.TemplatePredefinedKeys.avg_used_js_heap_size: dashboard.get_memory_consumption, schemas.TemplatePredefinedKeys.avg_cpu: dashboard.get_avg_cpu, - schemas.TemplatePredefinedKeys.avg_fps: dashboard.get_avg_fps} + schemas.TemplatePredefinedKeys.avg_fps: dashboard.get_avg_fps, + schemas.TemplatePredefinedKeys.impacted_sessions_by_js_errors: dashboard.get_impacted_sessions_by_js_errors, + schemas.TemplatePredefinedKeys.domains_errors_4xx: dashboard.get_domains_errors_4xx, + schemas.TemplatePredefinedKeys.domains_errors_5xx: dashboard.get_domains_errors_5xx, + schemas.TemplatePredefinedKeys.errors_per_domains: dashboard.get_errors_per_domains, + schemas.TemplatePredefinedKeys.calls_errors: dashboard.get_calls_errors, + schemas.TemplatePredefinedKeys.errors_by_type: dashboard.get_errors_per_type, + schemas.TemplatePredefinedKeys.errors_by_origin: dashboard.get_resources_by_party, + } def get_predefined_metric(key: schemas.TemplatePredefinedKeys, project_id: int, data: dict): @@ -260,7 +269,6 @@ def make_chart_metrics(project_id, user_id, metric_id, data: schemas.CustomMetri include_dashboard=False) if raw_metric is None: return None - print(raw_metric) metric = schemas.CustomMetricAndTemplate = schemas.CustomMetricAndTemplate.parse_obj(raw_metric) if metric.is_template: return get_predefined_metric(key=metric.predefined_key, project_id=project_id, data=data.dict()) diff --git a/api/schemas.py b/api/schemas.py index e518b9914..aa9adc4fa 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -934,6 +934,22 @@ class TemplatePredefinedKeys(str, Enum): avg_used_js_heap_size = "avg_used_js_heap_size" avg_cpu = "avg_cpu" avg_fps = "avg_fps" + impacted_sessions_by_js_errors = "impacted_sessions_by_js_errors" + domains_errors_4xx = "domains_errors_4xx" + domains_errors_5xx = "domains_errors_5xx" + errors_per_domains = "errors_per_domains" + calls_errors = "calls_errors" + errors_by_type = "errors_per_type" + errors_by_origin = "resources_by_party" + + +class TemplatePredefinedUnits(str, Enum): + millisecond = "ms" + minute = "min" + memory = "mb" + frame = "f/s" + percentage = "%" + count = "count" class CustomMetricAndTemplate(BaseModel): diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 1ba9cc0af..14a31349a 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -77,7 +77,14 @@ VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, tr ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview') + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('sessions affected by js errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('top domains with 4xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('top domains with 5xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), + ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 6a0258af7..53b51972e 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1276,7 +1276,14 @@ VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, tr ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview') + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('sessions affected by js errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('top domains with 4xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('top domains with 5xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), + ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 5afd50a77..8d28f814f 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -76,7 +76,14 @@ VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, tr ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview') + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('sessions affected by js errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('top domains with 4xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('top domains with 5xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), + ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 3278a48b7..f955833c9 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1067,7 +1067,14 @@ VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, tr ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview') + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('sessions affected by js errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('top domains with 4xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('top domains with 5xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), + ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, From eb842085ea5831645b334c790e787c6fb5edeffe Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 16:37:27 +0200 Subject: [PATCH 068/125] feat(api): changed startDate/endDate to startTimestamp/endTimestamp for custom metrics --- api/chalicelib/core/custom_metrics.py | 12 ++++++------ api/schemas.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index a43de6903..15c2ffc49 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -12,8 +12,8 @@ PIE_CHART_GROUP = 5 def __try_live(project_id, data: schemas.TryCustomMetricsPayloadSchema): results = [] for i, s in enumerate(data.series): - s.filter.startDate = data.startDate - s.filter.endDate = data.endDate + s.filter.startDate = data.startTimestamp + s.filter.endDate = data.endTimestamp results.append(sessions.search2_series(data=s.filter, project_id=project_id, density=data.density, view_type=data.view_type, metric_type=data.metric_type, metric_of=data.metric_of, metric_value=data.metric_value)) @@ -93,8 +93,8 @@ def get_sessions(project_id, user_id, metric_id, data: schemas.CustomMetricSessi return None results = [] for s in metric.series: - s.filter.startDate = data.startDate - s.filter.endDate = data.endDate + s.filter.startDate = data.startTimestamp + s.filter.endDate = data.endTimestamp results.append({"seriesId": s.series_id, "seriesName": s.name, **sessions.search2_pg(data=s.filter, project_id=project_id, user_id=user_id)}) @@ -311,9 +311,9 @@ def get(metric_id, project_id, user_id, flatten=True): def get_with_template(metric_id, project_id, user_id, include_dashboard=True): with pg_client.PostgresClient() as cur: - sub_query="" + sub_query = "" if include_dashboard: - sub_query="""LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards + sub_query = """LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards FROM (SELECT dashboard_id, name, is_public FROM dashboards WHERE deleted_at ISNULL diff --git a/api/schemas.py b/api/schemas.py index aa9adc4fa..a5bc349a5 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -804,8 +804,8 @@ class TimeseriesMetricOfType(str, Enum): class CustomMetricSessionsPayloadSchema(FlatSessionsSearch): - startDate: int = Field(TimeUTC.now(-7)) - endDate: int = Field(TimeUTC.now()) + startTimestamp: int = Field(TimeUTC.now(-7)) + endTimestamp: int = Field(TimeUTC.now()) class Config: alias_generator = attribute_to_camel_case From badae1249b48b1267dd142c46b64721f80147e53 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 17:06:57 +0200 Subject: [PATCH 069/125] feat(api): dashboard support for Performance predefined templates --- api/chalicelib/core/dashboards2.py | 13 +++++++++++++ api/schemas.py | 13 +++++++++++++ .../helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql | 15 ++++++++++++++- .../helm/db/init_dbs/postgresql/init_schema.sql | 15 ++++++++++++++- .../helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql | 15 ++++++++++++++- .../helm/db/init_dbs/postgresql/init_schema.sql | 15 ++++++++++++++- 6 files changed, 82 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 594db2fd0..fdbd9c7c1 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -257,6 +257,19 @@ PREDEFINED = {schemas.TemplatePredefinedKeys.count_sessions: dashboard.get_proce schemas.TemplatePredefinedKeys.calls_errors: dashboard.get_calls_errors, schemas.TemplatePredefinedKeys.errors_by_type: dashboard.get_errors_per_type, schemas.TemplatePredefinedKeys.errors_by_origin: dashboard.get_resources_by_party, + schemas.TemplatePredefinedKeys.speed_index_by_location: dashboard.get_speed_index_location, + schemas.TemplatePredefinedKeys.slowest_domains: dashboard.get_slowest_domains, + schemas.TemplatePredefinedKeys.sessions_per_browser: dashboard.get_sessions_per_browser, + schemas.TemplatePredefinedKeys.time_to_render: dashboard.get_time_to_render, + schemas.TemplatePredefinedKeys.impacted_sessions_by_slow_pages: dashboard.get_impacted_sessions_by_slow_pages, + schemas.TemplatePredefinedKeys.memory_consumption: dashboard.get_memory_consumption, + schemas.TemplatePredefinedKeys.cpu_load: dashboard.get_avg_cpu, + schemas.TemplatePredefinedKeys.frame_rate: dashboard.get_avg_fps, + schemas.TemplatePredefinedKeys.crashes: dashboard.get_crashes, + schemas.TemplatePredefinedKeys.resources_vs_visually_complete: dashboard.get_resources_vs_visually_complete, + schemas.TemplatePredefinedKeys.pages_dom_buildtime: dashboard.get_pages_dom_build_time, + schemas.TemplatePredefinedKeys.pages_response_time: dashboard.get_pages_response_time, + schemas.TemplatePredefinedKeys.pages_response_time_distribution: dashboard.get_pages_response_time_distribution, } diff --git a/api/schemas.py b/api/schemas.py index a5bc349a5..35f5fc9a0 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -941,6 +941,19 @@ class TemplatePredefinedKeys(str, Enum): calls_errors = "calls_errors" errors_by_type = "errors_per_type" errors_by_origin = "resources_by_party" + speed_index_by_location="speed_location" + slowest_domains="slowest_domains" + sessions_per_browser="sessions_per_browser" + time_to_render="time_to_render" + impacted_sessions_by_slow_pages="impacted_sessions_by_slow_pages" + memory_consumption="memory_consumption" + cpu_load="cpu" + frame_rate="fps" + crashes="crashes" + resources_vs_visually_complete="resources_vs_visually_complete" + pages_dom_buildtime="pages_dom_buildtime" + pages_response_time="pages_response_time" + pages_response_time_distribution="pages_response_time_distribution" class TemplatePredefinedUnits(str, Enum): diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 14a31349a..4b9d91892 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -84,7 +84,20 @@ VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, tr ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), - ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart') + ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + ('speed index by location', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), + ('slowest domains', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('sessions per browser', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('time to render', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('sessions impacted by slow pages', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('memory consumption', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('cpu load', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('frame rate', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), + ('crashes', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('resources vs visually complete', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('dom build time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('pages response time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('pages response time distribution', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 53b51972e..bcc1d005c 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1283,7 +1283,20 @@ VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, tr ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), - ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart') + ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + ('speed index by location', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), + ('slowest domains', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('sessions per browser', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('time to render', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('sessions impacted by slow pages', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('memory consumption', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('cpu load', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('frame rate', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), + ('crashes', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('resources vs visually complete', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('dom build time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('pages response time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('pages response time distribution', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 8d28f814f..781fb3be5 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -83,7 +83,20 @@ VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, tr ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), - ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart') + ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + ('speed index by location', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), + ('slowest domains', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('sessions per browser', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('time to render', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('sessions impacted by slow pages', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('memory consumption', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('cpu load', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('frame rate', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), + ('crashes', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('resources vs visually complete', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('dom build time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('pages response time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('pages response time distribution', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index f955833c9..0737fb8b1 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1074,7 +1074,20 @@ VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, tr ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), - ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart') + ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + ('speed index by location', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), + ('slowest domains', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('sessions per browser', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('time to render', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('sessions impacted by slow pages', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('memory consumption', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('cpu load', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('frame rate', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), + ('crashes', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('resources vs visually complete', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('dom build time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('pages response time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('pages response time distribution', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, From c26b77ca897831bd808a251f407e36f07d19bf1c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 17:59:54 +0200 Subject: [PATCH 070/125] feat(api): dashboard support for Resources predefined templates --- api/chalicelib/core/dashboards2.py | 5 ++ api/schemas.py | 33 ++++--- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 89 ++++++++++--------- .../db/init_dbs/postgresql/init_schema.sql | 89 ++++++++++--------- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 89 ++++++++++--------- .../db/init_dbs/postgresql/init_schema.sql | 89 ++++++++++--------- 6 files changed, 220 insertions(+), 174 deletions(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index fdbd9c7c1..d4053e82b 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -270,6 +270,11 @@ PREDEFINED = {schemas.TemplatePredefinedKeys.count_sessions: dashboard.get_proce schemas.TemplatePredefinedKeys.pages_dom_buildtime: dashboard.get_pages_dom_build_time, schemas.TemplatePredefinedKeys.pages_response_time: dashboard.get_pages_response_time, schemas.TemplatePredefinedKeys.pages_response_time_distribution: dashboard.get_pages_response_time_distribution, + schemas.TemplatePredefinedKeys.missing_resources: dashboard.get_missing_resources_trend, + schemas.TemplatePredefinedKeys.slowest_resources: dashboard.get_slowest_resources, + schemas.TemplatePredefinedKeys.resources_fetch_time: dashboard.get_resources_loading_time, + schemas.TemplatePredefinedKeys.resource_type_vs_response_end: dashboard.resource_type_vs_response_end, + schemas.TemplatePredefinedKeys.resources_count_by_type: dashboard.get_resources_count_by_type, } diff --git a/api/schemas.py b/api/schemas.py index 35f5fc9a0..e97135f0c 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -927,7 +927,7 @@ class TemplatePredefinedKeys(str, Enum): avg_response_time = "avg_response_time" avg_first_paint = "avg_first_paint" avg_dom_content_loaded = "avg_dom_content_loaded" - avg_till_first_bit = "avg_till_first_bit" + avg_till_first_bit = "avg_till_first_byte" avg_time_to_interactive = "avg_time_to_interactive" count_requests = "count_requests" avg_time_to_render = "avg_time_to_render" @@ -941,19 +941,24 @@ class TemplatePredefinedKeys(str, Enum): calls_errors = "calls_errors" errors_by_type = "errors_per_type" errors_by_origin = "resources_by_party" - speed_index_by_location="speed_location" - slowest_domains="slowest_domains" - sessions_per_browser="sessions_per_browser" - time_to_render="time_to_render" - impacted_sessions_by_slow_pages="impacted_sessions_by_slow_pages" - memory_consumption="memory_consumption" - cpu_load="cpu" - frame_rate="fps" - crashes="crashes" - resources_vs_visually_complete="resources_vs_visually_complete" - pages_dom_buildtime="pages_dom_buildtime" - pages_response_time="pages_response_time" - pages_response_time_distribution="pages_response_time_distribution" + speed_index_by_location = "speed_location" + slowest_domains = "slowest_domains" + sessions_per_browser = "sessions_per_browser" + time_to_render = "time_to_render" + impacted_sessions_by_slow_pages = "impacted_sessions_by_slow_pages" + memory_consumption = "memory_consumption" + cpu_load = "cpu" + frame_rate = "fps" + crashes = "crashes" + resources_vs_visually_complete = "resources_vs_visually_complete" + pages_dom_buildtime = "pages_dom_buildtime" + pages_response_time = "pages_response_time" + pages_response_time_distribution = "pages_response_time_distribution" + missing_resources = "missing_resources" + slowest_resources = "slowest_resources" + resources_fetch_time = "resources_loading_time" + resource_type_vs_response_end = "resource_type_vs_response_end" + resources_count_by_type = "resources_count_by_type" class TemplatePredefinedUnits(str, Enum): diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 4b9d91892..113069f24 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -30,7 +30,7 @@ ALTER TABLE IF EXISTS metrics ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, - ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{"col":2,"row":2,"position":0}'::jsonb, ALTER COLUMN project_id DROP NOT NULL, ADD CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), @@ -59,45 +59,54 @@ ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'predefined'; INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'predefined', 'overview'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), - ('sessions affected by js errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), - ('top domains with 4xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), - ('top domains with 5xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), - ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), - ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), - ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), - ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), - ('speed index by location', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), - ('slowest domains', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), - ('sessions per browser', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), - ('time to render', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), - ('sessions impacted by slow pages', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), - ('memory consumption', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), - ('cpu load', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), - ('frame rate', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), - ('crashes', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), - ('resources vs visually complete', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), - ('dom build time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), - ('pages response time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), - ('pages response time distribution', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart') +VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('Image Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('DOM Content Load Start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('First Meaningful paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('No. of Visited Pages', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('Session Duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('DOM Build Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('Pages Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('First Paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('DOM Content Loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('Time Till First byte', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), + ('Time To Interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('Captured requests', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), + ('Time To Render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('Memory Consumption', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('CPU Load', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('Frame rate', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_fps', 'predefined', 'overview'), + + ('Sessions Affected by JS Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('Top Domains with 4xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('Top Domains with 5xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('Errors per Domain', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('Fetch Calls with Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), + ('Errors by Type', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('Errors by Origin', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + + ('Speed Index by Location', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), + ('Slowest Domains', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('Sessions per Browser', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('Time To Render', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('Sessions Impacted by Slow Pages', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('Memory Consumption', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('CPU Load', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('Frame Rate', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), + ('Crashes', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('Resources Loaded vs Visually Complete', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('DOM Build Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), + + ('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), + ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), + ('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index bcc1d005c..c5303db96 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -811,7 +811,7 @@ $$ is_predefined boolean NOT NULL DEFAULT FALSE, is_template boolean NOT NULL DEFAULT FALSE, predefined_key text NULL DEFAULT NULL, - config jsonb NOT NULL DEFAULT '{}'::jsonb, + config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), CONSTRAINT unique_key UNIQUE (predefined_key) @@ -1258,45 +1258,54 @@ $$ LANGUAGE plpgsql; INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'predefined', 'overview'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), - ('sessions affected by js errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), - ('top domains with 4xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), - ('top domains with 5xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), - ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), - ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), - ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), - ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), - ('speed index by location', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), - ('slowest domains', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), - ('sessions per browser', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), - ('time to render', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), - ('sessions impacted by slow pages', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), - ('memory consumption', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), - ('cpu load', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), - ('frame rate', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), - ('crashes', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), - ('resources vs visually complete', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), - ('dom build time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), - ('pages response time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), - ('pages response time distribution', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart') +VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('Image Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('DOM Content Load Start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('First Meaningful paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('No. of Visited Pages', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('Session Duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('DOM Build Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('Pages Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('First Paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('DOM Content Loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('Time Till First byte', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), + ('Time To Interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('Captured requests', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), + ('Time To Render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('Memory Consumption', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('CPU Load', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('Frame rate', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_fps', 'predefined', 'overview'), + + ('Sessions Affected by JS Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('Top Domains with 4xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('Top Domains with 5xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('Errors per Domain', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('Fetch Calls with Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), + ('Errors by Type', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('Errors by Origin', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + + ('Speed Index by Location', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), + ('Slowest Domains', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('Sessions per Browser', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('Time To Render', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('Sessions Impacted by Slow Pages', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('Memory Consumption', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('CPU Load', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('Frame Rate', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), + ('Crashes', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('Resources Loaded vs Visually Complete', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('DOM Build Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), + + ('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), + ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), + ('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 781fb3be5..ba502c493 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -30,7 +30,7 @@ ALTER TABLE IF EXISTS metrics ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, - ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{"col":2,"row":2,"position":0}'::jsonb, ALTER COLUMN project_id DROP NOT NULL, ADD CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), @@ -58,45 +58,54 @@ ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'map'; ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'predefined'; INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'predefined', 'overview'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), - ('sessions affected by js errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), - ('top domains with 4xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), - ('top domains with 5xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), - ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), - ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), - ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), - ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), - ('speed index by location', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), - ('slowest domains', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), - ('sessions per browser', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), - ('time to render', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), - ('sessions impacted by slow pages', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), - ('memory consumption', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), - ('cpu load', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), - ('frame rate', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), - ('crashes', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), - ('resources vs visually complete', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), - ('dom build time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), - ('pages response time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), - ('pages response time distribution', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart') +VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('Image Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('DOM Content Load Start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('First Meaningful paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('No. of Visited Pages', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('Session Duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('DOM Build Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('Pages Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('First Paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('DOM Content Loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('Time Till First byte', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), + ('Time To Interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('Captured requests', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), + ('Time To Render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('Memory Consumption', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('CPU Load', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('Frame rate', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_fps', 'predefined', 'overview'), + + ('Sessions Affected by JS Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('Top Domains with 4xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('Top Domains with 5xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('Errors per Domain', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('Fetch Calls with Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), + ('Errors by Type', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('Errors by Origin', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + + ('Speed Index by Location', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), + ('Slowest Domains', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('Sessions per Browser', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('Time To Render', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('Sessions Impacted by Slow Pages', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('Memory Consumption', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('CPU Load', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('Frame Rate', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), + ('Crashes', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('Resources Loaded vs Visually Complete', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('DOM Build Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), + + ('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), + ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), + ('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 0737fb8b1..c8a1e41fb 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -958,7 +958,7 @@ $$ is_predefined boolean NOT NULL DEFAULT FALSE, is_template boolean NOT NULL DEFAULT FALSE, predefined_key text NULL DEFAULT NULL, - config jsonb NOT NULL DEFAULT '{}'::jsonb, + config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), CONSTRAINT unique_key UNIQUE (predefined_key) @@ -1049,45 +1049,54 @@ $$ LANGUAGE plpgsql; INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) -VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), - ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), - ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), - ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), - ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), - ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), - ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), - ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), - ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), - ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), - ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), - ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), - ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), - ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit', 'predefined', 'overview'), - ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), - ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), - ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), - ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), - ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), - ('sessions affected by js errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), - ('top domains with 4xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), - ('top domains with 5xx fetch errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), - ('errors per domain', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), - ('fetch calls with errors', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), - ('errors by type', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), - ('errors by origin', 'errors', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), - ('speed index by location', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), - ('slowest domains', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), - ('sessions per browser', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), - ('time to render', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), - ('sessions impacted by slow pages', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), - ('memory consumption', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), - ('cpu load', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), - ('frame rate', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), - ('crashes', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), - ('resources vs visually complete', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), - ('dom build time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), - ('pages response time', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), - ('pages response time distribution', 'performance', '{"col":1,"row":1,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart') +VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('Image Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('DOM Content Load Start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('First Meaningful paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('No. of Visited Pages', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('Session Duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('DOM Build Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('Pages Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('First Paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('DOM Content Loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('Time Till First byte', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), + ('Time To Interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('Captured requests', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), + ('Time To Render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('Memory Consumption', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('CPU Load', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('Frame rate', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_fps', 'predefined', 'overview'), + + ('Sessions Affected by JS Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('Top Domains with 4xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('Top Domains with 5xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('Errors per Domain', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('Fetch Calls with Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), + ('Errors by Type', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('Errors by Origin', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + + ('Speed Index by Location', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), + ('Slowest Domains', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('Sessions per Browser', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('Time To Render', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('Sessions Impacted by Slow Pages', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('Memory Consumption', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('CPU Load', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('Frame Rate', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), + ('Crashes', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('Resources Loaded vs Visually Complete', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('DOM Build Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), + + ('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), + ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), + ('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, From 577185c304c81e87dca55d15512f2d0ad853b163 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 7 Apr 2022 18:24:08 +0200 Subject: [PATCH 071/125] feat(api): dashboard separated config and default config --- api/chalicelib/core/dashboards2.py | 2 +- api/schemas.py | 2 +- ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql | 10 +++++----- ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql | 4 ++-- scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql | 10 +++++----- scripts/helm/db/init_dbs/postgresql/init_schema.sql | 4 ++-- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index d4053e82b..651500aaa 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -18,7 +18,7 @@ CATEGORY_DESCRIPTION = { def get_templates(project_id, user_id): with pg_client.PostgresClient() as cur: pg_query = cur.mogrify(f"""SELECT category, jsonb_agg(metrics ORDER BY name) AS widgets - FROM (SELECT * + FROM (SELECT * , default_config AS config FROM metrics LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(metric_series.* ORDER BY index), '[]'::jsonb) AS series FROM metric_series WHERE metric_series.metric_id = metrics.metric_id diff --git a/api/schemas.py b/api/schemas.py index e97135f0c..7ba9c0413 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -899,7 +899,7 @@ class EditDashboardSchema(CreateDashboardSchema): class UpdateWidgetPayloadSchema(BaseModel): # if you change the config attribute name, please make sure to update it in dashboard2.py - config: dict = Field(default={"col": 1, "row": 1, "position": 0}) + config: dict = Field(default={"col": 2, "row": 2, "position": 0}) class Config: alias_generator = attribute_to_camel_case diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 113069f24..f1d3cdc9a 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -24,13 +24,13 @@ ALTER TABLE IF EXISTS metrics DROP CONSTRAINT IF EXISTS unique_key; ALTER TABLE IF EXISTS metrics - ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', + ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, - ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{"col":2,"row":2,"position":0}'::jsonb, + ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS default_config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, ALTER COLUMN project_id DROP NOT NULL, ADD CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), @@ -58,7 +58,7 @@ ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'map'; ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'predefined'; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) +INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index c5303db96..f2cb95b76 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -811,7 +811,7 @@ $$ is_predefined boolean NOT NULL DEFAULT FALSE, is_template boolean NOT NULL DEFAULT FALSE, predefined_key text NULL DEFAULT NULL, - config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, + default_config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), CONSTRAINT unique_key UNIQUE (predefined_key) @@ -1257,7 +1257,7 @@ $$ $$ LANGUAGE plpgsql; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) +INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index ba502c493..c97cd76ac 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -24,13 +24,13 @@ ALTER TABLE IF EXISTS metrics DROP CONSTRAINT IF EXISTS unique_key; ALTER TABLE IF EXISTS metrics - ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', + ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, - ADD COLUMN IF NOT EXISTS config jsonb NOT NULL DEFAULT '{"col":2,"row":2,"position":0}'::jsonb, + ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS default_config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, ALTER COLUMN project_id DROP NOT NULL, ADD CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), @@ -57,7 +57,7 @@ ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'overview'; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'map'; ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'predefined'; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) +INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index c8a1e41fb..3d5239153 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -958,7 +958,7 @@ $$ is_predefined boolean NOT NULL DEFAULT FALSE, is_template boolean NOT NULL DEFAULT FALSE, predefined_key text NULL DEFAULT NULL, - config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, + default_config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), CONSTRAINT unique_key UNIQUE (predefined_key) @@ -1048,7 +1048,7 @@ $$ $$ LANGUAGE plpgsql; -INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) +INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), From c6a5ec42369030805023fb07767997ae48d5744e Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Fri, 8 Apr 2022 11:39:23 +0200 Subject: [PATCH 072/125] chore(helm): update note Signed-off-by: rjshrjndrn --- .../helmcharts/openreplay/templates/NOTES.txt | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/scripts/helmcharts/openreplay/templates/NOTES.txt b/scripts/helmcharts/openreplay/templates/NOTES.txt index 6881933f1..1479a07c3 100644 --- a/scripts/helmcharts/openreplay/templates/NOTES.txt +++ b/scripts/helmcharts/openreplay/templates/NOTES.txt @@ -1,22 +1 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openreplay.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openreplay.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openreplay.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openreplay.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} +OpenReplay Installation is complete. Follow along with the SSL configuration from [doc](https://docs.openreplay.com/deployment/deploy-aws#configuretls/ssl). From 46199bbdcab5d21f2af7979b8180363691036dc8 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 8 Apr 2022 17:41:36 +0200 Subject: [PATCH 073/125] feat(db): changed dashboard template --- ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql | 4 ++-- ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql | 4 ++-- scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql | 4 ++-- scripts/helm/db/init_dbs/postgresql/init_schema.sql | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index f1d3cdc9a..8cafd767d 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -102,7 +102,7 @@ VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, ('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), - ('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Missing Resources', 'resources', '{"col":4,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), ('Slowest Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), @@ -110,7 +110,7 @@ VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, - config=excluded.config, + default_config=excluded.default_config, is_predefined=excluded.is_predefined, is_template=excluded.is_template, is_public=excluded.is_public, diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index f2cb95b76..5abd6f026 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1302,14 +1302,14 @@ VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), ('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), - ('Slowest Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{"col":4,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), ('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, - config=excluded.config, + default_config=excluded.default_config, is_predefined=excluded.is_predefined, is_template=excluded.is_template, is_public=excluded.is_public, diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index c97cd76ac..c50f6fc42 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -102,14 +102,14 @@ VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), ('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), - ('Slowest Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{"col":4,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), ('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, - config=excluded.config, + default_config=excluded.default_config, is_predefined=excluded.is_predefined, is_template=excluded.is_template, is_public=excluded.is_public, diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 3d5239153..d78d0754b 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1093,14 +1093,14 @@ VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), ('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), - ('Slowest Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{"col":4,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), ('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, - config=excluded.config, + default_config=excluded.default_config, is_predefined=excluded.is_predefined, is_template=excluded.is_template, is_public=excluded.is_public, From e362cde1a99bac9e7d40b4dd32ee4523c0ec2c90 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 8 Apr 2022 17:54:13 +0200 Subject: [PATCH 074/125] feat(api): dashboard merge config if not present in payload feat(api): dashboard use metrics' default config for batch create --- api/chalicelib/core/dashboards2.py | 13 +++++++------ api/schemas.py | 3 +-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 651500aaa..c82a38b5f 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -48,14 +48,15 @@ def create_dashboard(project_id, user_id, data: schemas.CreateDashboardSchema): if data.metrics is not None and len(data.metrics) > 0: pg_query = f"""WITH dash AS ({pg_query}) INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) - VALUES {",".join([f"((SELECT dashboard_id FROM dash),%(metric_id_{i})s, %(userId)s, %(config_{i})s)" for i in range(len(data.metrics))])} + VALUES {",".join([f"((SELECT dashboard_id FROM dash),%(metric_id_{i})s, %(userId)s, (SELECT default_config FROM metrics WHERE metric_id=%(metric_id_{i})s)||%(config_{i})s))" for i in range(len(data.metrics))])} RETURNING (SELECT dashboard_id FROM dash)""" for i, m in enumerate(data.metrics): params[f"metric_id_{i}"] = m - params[f"config_{i}"] = schemas.AddWidgetToDashboardPayloadSchema.schema() \ - .get("properties", {}).get("config", {}).get("default", {}) - params[f"config_{i}"]["position"] = i - params[f"config_{i}"] = json.dumps(params[f"config_{i}"]) + # params[f"config_{i}"] = schemas.AddWidgetToDashboardPayloadSchema.schema() \ + # .get("properties", {}).get("config", {}).get("default", {}) + # params[f"config_{i}"]["position"] = i + # params[f"config_{i}"] = json.dumps(params[f"config_{i}"]) + params[f"config_{i}"] = json.dumps({"position": i}) cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() if row is None: @@ -173,7 +174,7 @@ def add_widget(project_id, user_id, dashboard_id, data: schemas.AddWidgetToDashb with pg_client.PostgresClient() as cur: pg_query = """INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) SELECT %(dashboard_id)s AS dashboard_id, %(metric_id)s AS metric_id, - %(userId)s AS user_id, %(config)s::jsonb AS config + %(userId)s AS user_id, (SELECT default_config FROM metrics WHERE metric_id=%(metric_id)s)||%(config)s::jsonb AS config WHERE EXISTS(SELECT 1 FROM dashboards WHERE dashboards.deleted_at ISNULL AND dashboards.project_id = %(projectId)s AND dashboard_id = %(dashboard_id)s diff --git a/api/schemas.py b/api/schemas.py index 7ba9c0413..b01553cc8 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -898,8 +898,7 @@ class EditDashboardSchema(CreateDashboardSchema): class UpdateWidgetPayloadSchema(BaseModel): - # if you change the config attribute name, please make sure to update it in dashboard2.py - config: dict = Field(default={"col": 2, "row": 2, "position": 0}) + config: dict = Field(default={}) class Config: alias_generator = attribute_to_camel_case From 7dd96fde02ee61068cfeb951132e14d30465d532 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 8 Apr 2022 18:09:44 +0200 Subject: [PATCH 075/125] feat(api): dashboard fixed SQL typo --- api/chalicelib/core/dashboards2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index c82a38b5f..231f20519 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -48,7 +48,7 @@ def create_dashboard(project_id, user_id, data: schemas.CreateDashboardSchema): if data.metrics is not None and len(data.metrics) > 0: pg_query = f"""WITH dash AS ({pg_query}) INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) - VALUES {",".join([f"((SELECT dashboard_id FROM dash),%(metric_id_{i})s, %(userId)s, (SELECT default_config FROM metrics WHERE metric_id=%(metric_id_{i})s)||%(config_{i})s))" for i in range(len(data.metrics))])} + VALUES {",".join([f"((SELECT dashboard_id FROM dash),%(metric_id_{i})s, %(userId)s, (SELECT default_config FROM metrics WHERE metric_id=%(metric_id_{i})s)||%(config_{i})s)" for i in range(len(data.metrics))])} RETURNING (SELECT dashboard_id FROM dash)""" for i, m in enumerate(data.metrics): params[f"metric_id_{i}"] = m From f52a61babc3775609f9a39035a803873c24672ba Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 8 Apr 2022 18:21:58 +0200 Subject: [PATCH 076/125] Update frontend.yaml --- .github/workflows/frontend.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 8c5038f5f..990ce3c8a 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -1,5 +1,6 @@ name: Frontend FOSS Deployment on: + workflow_dispatch: push: branches: - dev From ea6e21b94c12c5611102c648764947edf9ede1f8 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 8 Apr 2022 18:41:42 +0200 Subject: [PATCH 077/125] feat(api): dashboard fixed update with empty config payload --- api/chalicelib/core/dashboards2.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 231f20519..a66324532 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -133,13 +133,14 @@ def update_dashboard(project_id, user_id, dashboard_id, data: schemas.EditDashbo if data.metrics is not None and len(data.metrics) > 0: pg_query = f"""WITH dash AS ({pg_query}) INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) - VALUES {",".join([f"(%(dashboard_id)s, %(metric_id_{i})s, %(userId)s, %(config_{i})s)" for i in range(len(data.metrics))])};""" + VALUES {",".join([f"(%(dashboard_id)s, %(metric_id_{i})s, %(userId)s, (SELECT default_config FROM metrics WHERE metric_id=%(metric_id_{i})s)||%(config_{i})s)" for i in range(len(data.metrics))])};""" for i, m in enumerate(data.metrics): params[f"metric_id_{i}"] = m - params[f"config_{i}"] = schemas.AddWidgetToDashboardPayloadSchema.schema() \ - .get("properties", {}).get("config", {}).get("default", {}) - params[f"config_{i}"]["position"] = i - params[f"config_{i}"] = json.dumps(params[f"config_{i}"]) + # params[f"config_{i}"] = schemas.AddWidgetToDashboardPayloadSchema.schema() \ + # .get("properties", {}).get("config", {}).get("default", {}) + # params[f"config_{i}"]["position"] = i + # params[f"config_{i}"] = json.dumps(params[f"config_{i}"]) + params[f"config_{i}"] = json.dumps({"position": i}) cur.execute(cur.mogrify(pg_query, params)) From 89b63cae273c4385a21b31071e94f3a0fa3fb1da Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 10 Apr 2022 15:53:24 +0200 Subject: [PATCH 078/125] feat-fix(tracker):3.5.5 * Fix behaviour when initialised inside iframe * Capture select element changes * Retry /i when response is not 200 (and not 401) * Capture initial element scroll on start * Fallback Label vallue to name, id or class --- tracker/tracker/package.json | 2 +- tracker/tracker/src/main/app/context.ts | 4 +- tracker/tracker/src/main/app/index.ts | 3 +- tracker/tracker/src/main/modules/input.ts | 38 ++++++++---- tracker/tracker/src/main/modules/mouse.ts | 49 +++++++++------- tracker/tracker/src/main/modules/scroll.ts | 12 ++-- tracker/tracker/src/webworker/index.ts | 67 +++++++++++----------- 7 files changed, 101 insertions(+), 74 deletions(-) diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 6df50033b..9c7920ad7 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "3.5.4", + "version": "3.5.5", "keywords": [ "logging", "replay" diff --git a/tracker/tracker/src/main/app/context.ts b/tracker/tracker/src/main/app/context.ts index 781f91ea8..fd7ec11dd 100644 --- a/tracker/tracker/src/main/app/context.ts +++ b/tracker/tracker/src/main/app/context.ts @@ -41,13 +41,13 @@ export function isInstance(node: Node, constr: Cons // @ts-ignore (for EI, Safary) doc.parentWindow || doc.defaultView; // TODO: smart global typing for Window object - while((context.parent || context.top) && context.parent !== context) { + while(context !== window) { // @ts-ignore if (node instanceof context[constr.name]) { return true } // @ts-ignore - context = context.parent || context.top + context = context.parent || window } // @ts-ignore return node instanceof context[constr.name] diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 59129cbec..de9518af6 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -234,10 +234,11 @@ export default class App { ); } + // TODO: full correct semantic checkRequiredVersion(version: string): boolean { const reqVer = version.split('.') const ver = this.version.split('.') - for (let i = 0; i < ver.length; i++) { + for (let i = 0; i < 3; i++) { if (Number(ver[i]) < Number(reqVer[i]) || isNaN(Number(ver[i])) || isNaN(Number(reqVer[i]))) { return false } diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index ad8cda673..0cad3c58b 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -1,4 +1,9 @@ -import { normSpaces, IN_BROWSER, getLabelAttribute, hasOpenreplayAttribute } from "../utils.js"; +import { + normSpaces, + IN_BROWSER, + getLabelAttribute, + hasOpenreplayAttribute, +} from "../utils.js"; import App from "../app/index.js"; import { SetInputTarget, SetInputValue, SetInputChecked } from "../../messages/index.js"; @@ -31,14 +36,14 @@ function isCheckable(node: any): node is HTMLInputElement { } const labelElementFor: ( - node: TextEditableElement, + element: TextEditableElement, ) => HTMLLabelElement | undefined = IN_BROWSER && 'labels' in HTMLInputElement.prototype - ? (node): HTMLLabelElement | undefined => { + ? (node) => { let p: Node | null = node; while ((p = p.parentNode) !== null) { - if (p.nodeName === 'LABEL') { - return p as HTMLLabelElement; + if (p instanceof HTMLLabelElement) { + return p } } const labels = node.labels; @@ -46,10 +51,10 @@ const labelElementFor: ( return labels[0]; } } - : (node): HTMLLabelElement | undefined => { + : (node) => { let p: Node | null = node; while ((p = p.parentNode) !== null) { - if (p.nodeName === 'LABEL') { + if (p instanceof HTMLLabelElement) { return p as HTMLLabelElement; } } @@ -66,10 +71,12 @@ export function getInputLabel(node: TextEditableElement): string { let label = getLabelAttribute(node); if (label === null) { const labelElement = labelElementFor(node); - label = - labelElement === undefined - ? node.placeholder || node.name - : labelElement.innerText; + label = (labelElement && labelElement.innerText) + || node.placeholder + || node.name + || node.id + || node.className + || node.type } return normSpaces(label).slice(0, 100); } @@ -101,7 +108,7 @@ export default function (app: App, opts: Partial): void { app.send(new SetInputTarget(id, label)); } } - function sendInputValue(id: number, node: TextEditableElement): void { + function sendInputValue(id: number, node: TextEditableElement | HTMLSelectElement): void { let value = node.value; let inputMode: InputMode = options.defaultInputMode; if (node.type === 'password' || hasOpenreplayAttribute(node, 'hidden')) { @@ -175,6 +182,13 @@ export default function (app: App, opts: Partial): void { if (id === undefined) { return; } + // TODO: support multiple select (?): use selectedOptions; Need send target? + if (node instanceof HTMLSelectElement) { + sendInputValue(id, node) + app.attachEventListener(node, "change", () => { + sendInputValue(id, node) + }) + } if (isTextEditable(node)) { inputValues.set(id, node.value); sendInputValue(id, node); diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index 90cf17908..196a89653 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -1,4 +1,8 @@ -import { normSpaces, hasOpenreplayAttribute, getLabelAttribute } from "../utils.js"; +import { + normSpaces, + hasOpenreplayAttribute, + getLabelAttribute, +} from "../utils.js"; import App from "../app/index.js"; import { MouseMove, MouseClick } from "../../messages/index.js"; import { getInputLabel } from "./input.js"; @@ -24,6 +28,18 @@ function _getSelector(target: Element): string { return selector } +function isClickable(element: Element): boolean { + const tag = element.tagName.toUpperCase() + return tag === 'BUTTON' || + tag === 'A' || + tag === 'LI' || + tag === 'SELECT' || + (element as HTMLElement).onclick != null || + element.getAttribute('role') === 'button' + //|| element.className.includes("btn") + // MBTODO: intersect addEventListener +} + //TODO: fix (typescript doesn't allow work when the guard is inside the function) function getTarget(target: EventTarget | null): Element | null { if (target instanceof Element) { @@ -56,13 +72,7 @@ function _getTarget(target: Element): Element | null { if (tag === 'INPUT') { return element; } - if ( - tag === 'BUTTON' || - tag === 'A' || - tag === 'LI' || - tag === 'SELECT' || - (element as HTMLElement).onclick != null || - element.getAttribute('role') === 'button' || + if (isClickable(element) || getLabelAttribute(element) !== null ) { return element; @@ -83,19 +93,16 @@ export default function (app: App): void { if (dl !== null) { return dl; } - const tag = target.tagName.toUpperCase(); - if (tag === 'INPUT') { - return getInputLabel(target as HTMLInputElement) + if (target instanceof HTMLInputElement) { + return getInputLabel(target) } - if (tag === 'BUTTON' || - tag === 'A' || - tag === 'LI' || - tag === 'SELECT' || - (target as HTMLElement).onclick != null || - target.getAttribute('role') === 'button' - ) { - const label: string = app.sanitizer.getInnerTextSecure(target as HTMLElement); - return normSpaces(label).slice(0, 100); + if (isClickable(target)) { + let label = '' + if (target instanceof HTMLElement) { + label = app.sanitizer.getInnerTextSecure(target) + } + label = label || target.id || target.className + return normSpaces(label).slice(0, 100) } return ''; } @@ -126,7 +133,7 @@ export default function (app: App): void { } app.attachEventListener( - document.documentElement, + document.documentElement, 'mouseover', (e: MouseEvent): void => { const target = getTarget(e.target); diff --git a/tracker/tracker/src/main/modules/scroll.ts b/tracker/tracker/src/main/modules/scroll.ts index 0f54ba8f9..f9c80e6d9 100644 --- a/tracker/tracker/src/main/modules/scroll.ts +++ b/tracker/tracker/src/main/modules/scroll.ts @@ -20,7 +20,7 @@ export default function (app: App): void { ), ); - const sendSetNodeScroll = app.safe((s, node): void => { + const sendSetNodeScroll = app.safe((s: [number, number], node: Node): void => { const id = app.nodes.getID(node); if (id !== undefined) { app.send(new SetNodeScroll(id, s[0], s[1])); @@ -34,6 +34,12 @@ export default function (app: App): void { nodeScroll.clear(); }); + app.nodes.attachNodeCallback(node => { + if (node instanceof Element && node.scrollLeft + node.scrollTop > 0) { + nodeScroll.set(node, [node.scrollLeft, node.scrollTop]); + } + }) + app.attachEventListener(window, 'scroll', (e: Event): void => { const target = e.target; if (target === document) { @@ -41,9 +47,7 @@ export default function (app: App): void { return; } if (target instanceof Element) { - { - nodeScroll.set(target, [target.scrollLeft, target.scrollTop]); - } + nodeScroll.set(target, [target.scrollLeft, target.scrollTop]); } }); diff --git a/tracker/tracker/src/webworker/index.ts b/tracker/tracker/src/webworker/index.ts index cf0d1586a..f598ac0a4 100644 --- a/tracker/tracker/src/webworker/index.ts +++ b/tracker/tracker/src/webworker/index.ts @@ -31,42 +31,18 @@ let sendIntervalID: ReturnType | null = null; const sendQueue: Array = []; let busy = false; let attemptsCount = 0; -let ATTEMPT_TIMEOUT = 8000; +let ATTEMPT_TIMEOUT = 3000; let MAX_ATTEMPTS_COUNT = 10; // TODO?: exploit https://developer.mozilla.org/en-US/docs/Web/API/Navigator/sendBeacon function sendBatch(batch: Uint8Array):void { - const req = new XMLHttpRequest(); + const xhr = new XMLHttpRequest(); // TODO: async=false (3d param) instead of sendQueue array ? - req.open("POST", ingestPoint + "/v1/web/i", false); // TODO opaque request? - req.setRequestHeader("Authorization", "Bearer " + token); - // req.setRequestHeader("Content-Type", ""); - req.onreadystatechange = function() { - if (this.readyState === 4) { - if (this.status == 0) { - return; // happens simultaneously with onerror TODO: clear codeflow - } - if (this.status >= 400) { // TODO: test workflow. After 400+ it calls /start for some reason - busy = false; - reset(); - sendQueue.length = 0; - if (this.status === 401) { // Unauthorised (Token expired) - self.postMessage("restart") - return - } - self.postMessage(null); - return - } - //if (this.response == null) - const nextBatch = sendQueue.shift(); - if (nextBatch) { - sendBatch(nextBatch); - } else { - busy = false; - } - } - }; - req.onerror = function(e) { + xhr.open("POST", ingestPoint + "/v1/web/i", false); + xhr.setRequestHeader("Authorization", "Bearer " + token); + // xhr.setRequestHeader("Content-Type", ""); + + function retry() { if (attemptsCount >= MAX_ATTEMPTS_COUNT) { reset(); self.postMessage(null); @@ -75,8 +51,32 @@ function sendBatch(batch: Uint8Array):void { attemptsCount++; setTimeout(() => sendBatch(batch), ATTEMPT_TIMEOUT); } - // TODO: handle offline exception - req.send(batch.buffer); + xhr.onreadystatechange = function() { + if (this.readyState === 4) { + if (this.status == 0) { + return; // happens simultaneously with onerror TODO: clear codeflow + } + if (this.status === 401) { // Unauthorised (Token expired) + busy = false + self.postMessage("restart") + return + } else if (this.status >= 400) { // TODO: test workflow. After 400+ it calls /start for some reason + retry() + return + } + // Success + attemptsCount = 0 + const nextBatch = sendQueue.shift(); + if (nextBatch) { + sendBatch(nextBatch); + } else { + busy = false; + } + } + }; + xhr.onerror = retry // TODO: when in Offline mode it doesn't handle the error + // TODO: handle offline exception (?) + xhr.send(batch.buffer); } function send(): void { @@ -101,6 +101,7 @@ function reset() { clearInterval(sendIntervalID); sendIntervalID = null; } + sendQueue.length = 0; writer.reset(); } From ef6b78b1c47dd35282ff02f85267694a8ef83f58 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 11 Apr 2022 12:45:25 +0200 Subject: [PATCH 079/125] feat(api): enhanced auth feat(api): check for project existence before any response --- api/chalicelib/core/authorizers.py | 6 +++--- api/chalicelib/core/projects.py | 3 ++- api/chalicelib/core/users.py | 1 - ee/api/.gitignore | 1 + ee/api/chalicelib/core/projects.py | 3 ++- ee/api/chalicelib/core/users.py | 1 - ee/api/routers/base.py | 14 -------------- 7 files changed, 8 insertions(+), 21 deletions(-) delete mode 100644 ee/api/routers/base.py diff --git a/api/chalicelib/core/authorizers.py b/api/chalicelib/core/authorizers.py index 899fd046f..5756e82ab 100644 --- a/api/chalicelib/core/authorizers.py +++ b/api/chalicelib/core/authorizers.py @@ -13,9 +13,9 @@ def jwt_authorizer(token): try: payload = jwt.decode( token[1], - "", + config("jwt_secret"), algorithms=config("jwt_algorithm"), - audience=[ f"front:default-foss"] + audience=[f"plugin:{helper.get_stage_name()}", f"front:{helper.get_stage_name()}"] ) except jwt.ExpiredSignatureError: print("! JWT Expired signature") @@ -42,7 +42,7 @@ def generate_jwt(id, tenant_id, iat, aud): payload={ "userId": id, "tenantId": tenant_id, - "exp": iat // 1000 + config("jwt_exp_delta_seconds",cast=int) + TimeUTC.get_utc_offset() // 1000, + "exp": iat // 1000 + config("jwt_exp_delta_seconds", cast=int) + TimeUTC.get_utc_offset() // 1000, "iss": config("jwt_issuer"), "iat": iat // 1000, "aud": aud diff --git a/api/chalicelib/core/projects.py b/api/chalicelib/core/projects.py index 3559f645a..e4ac36ad8 100644 --- a/api/chalicelib/core/projects.py +++ b/api/chalicelib/core/projects.py @@ -244,7 +244,8 @@ def get_project_key(project_id): where project_id =%(project_id)s AND deleted_at ISNULL;""", {"project_id": project_id}) ) - return cur.fetchone()["project_key"] + project = cur.fetchone() + return project["project_key"] if project is not None else None def get_capture_status(project_id): diff --git a/api/chalicelib/core/users.py b/api/chalicelib/core/users.py index b4ac0f869..ceada34f8 100644 --- a/api/chalicelib/core/users.py +++ b/api/chalicelib/core/users.py @@ -571,7 +571,6 @@ def auth_exists(user_id, tenant_id, jwt_iat, jwt_aud): ) -@dev.timed def authenticate(email, password, for_change_password=False, for_plugin=False): with pg_client.PostgresClient() as cur: query = cur.mogrify( diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 488fab072..0c649b68e 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -242,6 +242,7 @@ Pipfile /auth/auth_apikey.py /auth/auth_jwt.py /build.sh +/routers/base.py /routers/core.py /routers/crons/core_crons.py /routers/subs/dashboard.py diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index 3072f55a0..0f2b62cc9 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -257,7 +257,8 @@ def get_project_key(project_id): where project_id =%(project_id)s AND deleted_at ISNULL;""", {"project_id": project_id}) ) - return cur.fetchone()["project_key"] + project = cur.fetchone() + return project["project_key"] if project is not None else None def get_capture_status(project_id): diff --git a/ee/api/chalicelib/core/users.py b/ee/api/chalicelib/core/users.py index 9ca77c1ea..b70f6a269 100644 --- a/ee/api/chalicelib/core/users.py +++ b/ee/api/chalicelib/core/users.py @@ -632,7 +632,6 @@ def change_jwt_iat(user_id): return cur.fetchone().get("jwt_iat") -@dev.timed def authenticate(email, password, for_change_password=False, for_plugin=False): with pg_client.PostgresClient() as cur: query = cur.mogrify( diff --git a/ee/api/routers/base.py b/ee/api/routers/base.py deleted file mode 100644 index 5c665b2d1..000000000 --- a/ee/api/routers/base.py +++ /dev/null @@ -1,14 +0,0 @@ -from fastapi import APIRouter, Depends - -from auth.auth_apikey import APIKeyAuth -from auth.auth_jwt import JWTAuth -from auth.auth_project import ProjectAuthorizer -from or_dependencies import ORRoute - - -def get_routers() -> (APIRouter, APIRouter, APIRouter): - public_app = APIRouter(route_class=ORRoute) - app = APIRouter(dependencies=[Depends(JWTAuth()), Depends(ProjectAuthorizer("projectId"))], route_class=ORRoute) - app_apikey = APIRouter(dependencies=[Depends(APIKeyAuth()), Depends(ProjectAuthorizer("projectKey"))], - route_class=ORRoute) - return public_app, app, app_apikey From d4dda5c8222639b037ccaf772665ac1fc8b3e5ff Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 11 Apr 2022 12:46:28 +0200 Subject: [PATCH 080/125] feat(api): check for project existence before any response --- api/routers/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/routers/base.py b/api/routers/base.py index ff7fe165f..5c665b2d1 100644 --- a/api/routers/base.py +++ b/api/routers/base.py @@ -2,11 +2,13 @@ from fastapi import APIRouter, Depends from auth.auth_apikey import APIKeyAuth from auth.auth_jwt import JWTAuth +from auth.auth_project import ProjectAuthorizer from or_dependencies import ORRoute def get_routers() -> (APIRouter, APIRouter, APIRouter): public_app = APIRouter(route_class=ORRoute) - app = APIRouter(dependencies=[Depends(JWTAuth())], route_class=ORRoute) - app_apikey = APIRouter(dependencies=[Depends(APIKeyAuth())], route_class=ORRoute) + app = APIRouter(dependencies=[Depends(JWTAuth()), Depends(ProjectAuthorizer("projectId"))], route_class=ORRoute) + app_apikey = APIRouter(dependencies=[Depends(APIKeyAuth()), Depends(ProjectAuthorizer("projectKey"))], + route_class=ORRoute) return public_app, app, app_apikey From 5cd5ad656d22db1952a626719c835d784c78a19c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 11 Apr 2022 12:46:45 +0200 Subject: [PATCH 081/125] feat(api): check for project existence before any response --- api/auth/auth_project.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 api/auth/auth_project.py diff --git a/api/auth/auth_project.py b/api/auth/auth_project.py new file mode 100644 index 000000000..98a495bbb --- /dev/null +++ b/api/auth/auth_project.py @@ -0,0 +1,24 @@ +from fastapi import Request +from starlette import status +from starlette.exceptions import HTTPException + +import schemas +from chalicelib.core import projects +from or_dependencies import OR_context + + +class ProjectAuthorizer: + def __init__(self, project_identifier): + self.project_identifier: str = project_identifier + + async def __call__(self, request: Request) -> None: + if len(request.path_params.keys()) == 0 or request.path_params.get(self.project_identifier) is None: + return + current_user: schemas.CurrentContext = await OR_context(request) + project_identifier = request.path_params[self.project_identifier] + if (self.project_identifier == "projectId" \ + and projects.get_project(project_id=project_identifier, tenant_id=current_user.tenant_id) is None) \ + or (self.project_identifier.lower() == "projectKey" \ + and projects.get_internal_project_id(project_key=project_identifier) is None): + print("project not found") + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="project not found.") From 888433d3c73ae1ae803865314a7ccb0bf5a6fcc4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 11 Apr 2022 14:39:27 +0200 Subject: [PATCH 082/125] feat(utilities): heapsnapshot --- utilities/server.js | 18 ++++++++++----- utilities/utils/dump.js | 50 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 utilities/utils/dump.js diff --git a/utilities/server.js b/utilities/server.js index 661ef081c..225db88f4 100644 --- a/utilities/server.js +++ b/utilities/server.js @@ -1,14 +1,15 @@ -var sourcemapsReaderServer = require('./servers/sourcemaps-server'); -var {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); -var express = require('express'); +const dumps = require('./utils/dump'); +const sourcemapsReaderServer = require('./servers/sourcemaps-server'); +const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); +const express = require('express'); const {ExpressPeerServer} = require('peer'); const socket = require("./servers/websocket"); const HOST = '0.0.0.0'; const PORT = 9000; -var app = express(); -var wsapp = express(); +const app = express(); +const wsapp = express(); let debug = process.env.debug === "1" || false; const request_logger = (identity) => { return (req, res, next) => { @@ -29,6 +30,13 @@ app.use('/sourcemaps', sourcemapsReaderServer); app.use('/assist', peerRouter); wsapp.use('/assist', socket.wsRouter); +app.get('/heapdump', dumps.sendHeapSnapshot); +app.get('/heapdump/save', dumps.saveHeapSnapshot); +wsapp.get('/heapdump', dumps.sendHeapSnapshot); +wsapp.get('/heapdump/save', dumps.saveHeapSnapshot); + +console.log(`Heapdump enabled. Send a request to "/heapdump" to download a heapdump,\nor "/heapdump/save" to only generate a heapdump.`); + const server = app.listen(PORT, HOST, () => { console.log(`App listening on http://${HOST}:${PORT}`); console.log('Press Ctrl+C to quit.'); diff --git a/utilities/utils/dump.js b/utilities/utils/dump.js new file mode 100644 index 000000000..c22cffc3d --- /dev/null +++ b/utilities/utils/dump.js @@ -0,0 +1,50 @@ +const fs = require('fs'); +const v8 = require('v8'); + +const location = '/tmp/'; + +async function createHeapSnapshot() { + const fileName = `${Date.now()}.heapsnapshot`; + await fs.promises.writeFile( + location + fileName, + v8.getHeapSnapshot() + ); + return fileName; +} + + +async function sendHeapSnapshot(req, res) { + const fileName = await createHeapSnapshot(); + const file = fs.readFileSync(location + fileName, 'binary'); + const stat = fs.statSync(location + fileName); + + res.setHeader('Content-Length', stat.size); + res.setHeader('Content-Type', 'audio/mpeg'); + res.setHeader('Content-Disposition', 'attachment; filename=' + fileName); + res.write(file, 'binary'); + res.end(); + try { + fs.unlinkSync(location + fileName) + } catch (err) { + console.error("error while deleting heapsnapshot file"); + console.error(err); + } +} + +process.on('USR2', () => { + console.info('USR2 signal received.'); +}); + +async function saveHeapSnapshot(req, res) { + const fileName = await createHeapSnapshot(); + const path = location + fileName; + console.log(`Heapdump saved to ${path}`) + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({path})); +} + +process.on('USR2', () => { + console.info('USR2 signal received.'); +}); +module.exports = {sendHeapSnapshot, saveHeapSnapshot} \ No newline at end of file From 8786a121f6e678662cc9a469ca9751c84b09c5fc Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 11 Apr 2022 14:45:54 +0200 Subject: [PATCH 083/125] feat(utilities): FOSS&EE heapsnapshot --- ee/utilities/.gitignore | 3 ++- ee/utilities/server.js | 16 ++++++++++------ utilities/server.js | 5 +---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ee/utilities/.gitignore b/ee/utilities/.gitignore index fc05191e0..2dd72c808 100644 --- a/ee/utilities/.gitignore +++ b/ee/utilities/.gitignore @@ -10,4 +10,5 @@ build.sh servers/peerjs-server.js servers/sourcemaps-handler.js servers/sourcemaps-server.js -#servers/websocket.js \ No newline at end of file +#servers/websocket.js +/utils/dump.js diff --git a/ee/utilities/server.js b/ee/utilities/server.js index d049faa19..8f5a34d7b 100644 --- a/ee/utilities/server.js +++ b/ee/utilities/server.js @@ -1,8 +1,9 @@ -var sourcemapsReaderServer = require('./servers/sourcemaps-server'); -var {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); -var express = require('express'); +const dumps = require('./utils/dump'); +const sourcemapsReaderServer = require('./servers/sourcemaps-server'); +const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); +const express = require('express'); const {ExpressPeerServer} = require('peer'); -var socket; +let socket; if (process.env.redis === "true") { console.log("Using Redis"); socket = require("./servers/websocket-cluster"); @@ -13,7 +14,7 @@ if (process.env.redis === "true") { const HOST = '0.0.0.0'; const PORT = 9000; -var app = express(); +const app = express(); let debug = process.env.debug === "1" || false; const request_logger = (identity) => { @@ -50,6 +51,8 @@ peerServer.on('disconnect', peerDisconnect); peerServer.on('error', peerError); app.use('/', peerServer); app.enable('trust proxy'); +app.get('/heapdump', dumps.sendHeapSnapshot); +app.get('/heapdump/save', dumps.saveHeapSnapshot); if (process.env.uws !== "true") { var wsapp = express(); @@ -125,4 +128,5 @@ if (process.env.uws !== "true") { // process.exit(1); }); module.exports = {uapp, server}; -} \ No newline at end of file +} +console.log(`Heapdump enabled. Send a request to "/heapdump" to download a heapdump,\nor "/heapdump/save" to only generate a heapdump.`); \ No newline at end of file diff --git a/utilities/server.js b/utilities/server.js index 225db88f4..fcbdb5c62 100644 --- a/utilities/server.js +++ b/utilities/server.js @@ -32,10 +32,6 @@ wsapp.use('/assist', socket.wsRouter); app.get('/heapdump', dumps.sendHeapSnapshot); app.get('/heapdump/save', dumps.saveHeapSnapshot); -wsapp.get('/heapdump', dumps.sendHeapSnapshot); -wsapp.get('/heapdump/save', dumps.saveHeapSnapshot); - -console.log(`Heapdump enabled. Send a request to "/heapdump" to download a heapdump,\nor "/heapdump/save" to only generate a heapdump.`); const server = app.listen(PORT, HOST, () => { console.log(`App listening on http://${HOST}:${PORT}`); @@ -59,3 +55,4 @@ app.enable('trust proxy'); wsapp.enable('trust proxy'); socket.start(wsserver); module.exports = {wsserver, server}; +console.log(`Heapdump enabled. Send a request to "/heapdump" to download a heapdump,\nor "/heapdump/save" to only generate a heapdump.`); \ No newline at end of file From 0deb16529010d128fbec81d8250c76738d3234a3 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 11 Apr 2022 15:45:23 +0200 Subject: [PATCH 084/125] feat(utilities): heapsnapshot use express helper for upload --- utilities/utils/dump.js | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/utilities/utils/dump.js b/utilities/utils/dump.js index c22cffc3d..3c6f2de73 100644 --- a/utilities/utils/dump.js +++ b/utilities/utils/dump.js @@ -15,20 +15,14 @@ async function createHeapSnapshot() { async function sendHeapSnapshot(req, res) { const fileName = await createHeapSnapshot(); - const file = fs.readFileSync(location + fileName, 'binary'); - const stat = fs.statSync(location + fileName); - - res.setHeader('Content-Length', stat.size); - res.setHeader('Content-Type', 'audio/mpeg'); - res.setHeader('Content-Disposition', 'attachment; filename=' + fileName); - res.write(file, 'binary'); - res.end(); - try { - fs.unlinkSync(location + fileName) - } catch (err) { - console.error("error while deleting heapsnapshot file"); - console.error(err); - } + res.download(location + fileName, function (err) { + try { + fs.unlinkSync(location + fileName) + } catch (err) { + console.error("error while deleting heapsnapshot file"); + console.error(err); + } + }); } process.on('USR2', () => { From 29f264dcbf85c3c92908ebeb493c1f9b21e04b79 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 11 Apr 2022 16:28:26 +0200 Subject: [PATCH 085/125] feat(utilities): heapsnapshot multiple steps --- ee/utilities/server.js | 3 +-- utilities/server.js | 3 +-- utilities/utils/dump.js | 53 ++++++++++++++++++++++++++++------------- 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/ee/utilities/server.js b/ee/utilities/server.js index 8f5a34d7b..ac39f5dd1 100644 --- a/ee/utilities/server.js +++ b/ee/utilities/server.js @@ -51,8 +51,7 @@ peerServer.on('disconnect', peerDisconnect); peerServer.on('error', peerError); app.use('/', peerServer); app.enable('trust proxy'); -app.get('/heapdump', dumps.sendHeapSnapshot); -app.get('/heapdump/save', dumps.saveHeapSnapshot); +app.use('/heapdump', dumps.router); if (process.env.uws !== "true") { var wsapp = express(); diff --git a/utilities/server.js b/utilities/server.js index fcbdb5c62..0c80170af 100644 --- a/utilities/server.js +++ b/utilities/server.js @@ -30,8 +30,7 @@ app.use('/sourcemaps', sourcemapsReaderServer); app.use('/assist', peerRouter); wsapp.use('/assist', socket.wsRouter); -app.get('/heapdump', dumps.sendHeapSnapshot); -app.get('/heapdump/save', dumps.saveHeapSnapshot); +app.use('/heapdump', dumps.router); const server = app.listen(PORT, HOST, () => { console.log(`App listening on http://${HOST}:${PORT}`); diff --git a/utilities/utils/dump.js b/utilities/utils/dump.js index 3c6f2de73..23708f07c 100644 --- a/utilities/utils/dump.js +++ b/utilities/utils/dump.js @@ -1,20 +1,35 @@ const fs = require('fs'); const v8 = require('v8'); +const express = require('express'); +const router = express.Router(); const location = '/tmp/'; +let creationStatus = null; +let fileName = null; async function createHeapSnapshot() { - const fileName = `${Date.now()}.heapsnapshot`; + if (creationStatus) { + return console.log(`In progress ${fileName}`); + } + if (fileName === null) { + fileName = `${Date.now()}.heapsnapshot`; + } + console.log(`Creating ${fileName}`); await fs.promises.writeFile( location + fileName, v8.getHeapSnapshot() ); - return fileName; + console.log(`Created ${fileName}`); + creationStatus = true; } -async function sendHeapSnapshot(req, res) { - const fileName = await createHeapSnapshot(); +async function downloadHeapSnapshot(req, res) { + if (creationStatus === null) { + return res.end("should call /new first"); + } else if (!creationStatus) { + return res.end("should wait for done status"); + } res.download(location + fileName, function (err) { try { fs.unlinkSync(location + fileName) @@ -25,20 +40,24 @@ async function sendHeapSnapshot(req, res) { }); } -process.on('USR2', () => { - console.info('USR2 signal received.'); -}); - -async function saveHeapSnapshot(req, res) { - const fileName = await createHeapSnapshot(); - const path = location + fileName; - console.log(`Heapdump saved to ${path}`) +function getHeapSnapshotStatus(req, res) { res.statusCode = 200; res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({path})); + res.end(JSON.stringify({path: location + fileName, 'done': creationStatus})); } -process.on('USR2', () => { - console.info('USR2 signal received.'); -}); -module.exports = {sendHeapSnapshot, saveHeapSnapshot} \ No newline at end of file +function createNewHeapSnapshot(req, res) { + creationStatus = false; + fileName = `${Date.now()}.heapsnapshot`; + setTimeout(() => { + createHeapSnapshot() + }, 0); + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({path: location + fileName, 'done': creationStatus})); +} + +router.get('/status', getHeapSnapshotStatus); +router.get(`/new`, createNewHeapSnapshot); +router.get(`/download`, downloadHeapSnapshot); +module.exports = {router} \ No newline at end of file From f2b429a3d898a6fb9444c972882f458e5aacc9be Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 11 Apr 2022 16:36:16 +0200 Subject: [PATCH 086/125] feat(utilities): changed base docker image --- utilities/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/Dockerfile b/utilities/Dockerfile index 9b82358b3..171995974 100644 --- a/utilities/Dockerfile +++ b/utilities/Dockerfile @@ -1,4 +1,4 @@ -FROM node:17-stretch +FROM node:17-alpine WORKDIR /work COPY . . RUN npm install From 58af31c35e8b5a890b4381fd4b224922a9312e1d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 11 Apr 2022 18:17:49 +0200 Subject: [PATCH 087/125] feat(utilities): optimized websocket geo-ip logic feat(utilities): refactored websocket --- ee/utilities/server.js | 2 +- ee/utilities/servers/websocket-cluster.js | 35 +++++++++----------- ee/utilities/servers/websocket.js | 35 +++++++++----------- utilities/Dockerfile | 2 +- utilities/server.js | 2 +- utilities/servers/websocket.js | 35 +++++++++----------- utilities/utils/{dump.js => HeapSnapshot.js} | 0 7 files changed, 48 insertions(+), 63 deletions(-) rename utilities/utils/{dump.js => HeapSnapshot.js} (100%) diff --git a/ee/utilities/server.js b/ee/utilities/server.js index ac39f5dd1..798f4e715 100644 --- a/ee/utilities/server.js +++ b/ee/utilities/server.js @@ -1,4 +1,4 @@ -const dumps = require('./utils/dump'); +const dumps = require('./utils/HeapSnapshot'); const sourcemapsReaderServer = require('./servers/sourcemaps-server'); const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); const express = require('express'); diff --git a/ee/utilities/servers/websocket-cluster.js b/ee/utilities/servers/websocket-cluster.js index 998a457df..8dc3bf94c 100644 --- a/ee/utilities/servers/websocket-cluster.js +++ b/ee/utilities/servers/websocket-cluster.js @@ -255,6 +255,16 @@ async function get_all_agents_ids(io, socket) { return agents; } +let geoip = null; +geoip2Reader.open(process.env.MAXMINDDB_FILE, {}) + .then(reader => { + geoip = reader; + }) + .catch(error => { + console.log("Error while opening the MAXMINDDB_FILE.") + console.error(error); + }); + function extractSessionInfo(socket) { if (socket.handshake.query.sessionInfo !== undefined) { debug && console.log("received headers"); @@ -268,21 +278,11 @@ function extractSessionInfo(socket) { socket.handshake.query.sessionInfo.userDevice = ua.device.model || null; socket.handshake.query.sessionInfo.userDeviceType = ua.device.type || 'desktop'; socket.handshake.query.sessionInfo.userCountry = null; - - const options = { - // you can use options like `cache` or `watchForUpdates` - }; - // console.log("Looking for MMDB file in " + process.env.MAXMINDDB_FILE); - geoip2Reader.open(process.env.MAXMINDDB_FILE, options) - .then(reader => { - debug && console.log("looking for location of "); - debug && console.log(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - let country = reader.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; - }) - .catch(error => { - console.error(error); - }); + if (geoip !== null) { + debug && console.log(`looking for location of ${socket.handshake.headers['x-forwarded-for'] || socket.handshake.address}`); + let country = geoip.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); + socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; + } } } @@ -294,10 +294,6 @@ module.exports = { debug && console.log(`WS started:${socket.id}, Query:${JSON.stringify(socket.handshake.query)}`); socket.peerId = socket.handshake.query.peerId; socket.identity = socket.handshake.query.identity; - const {projectKey, sessionId} = extractPeerId(socket.peerId); - socket.sessionId = sessionId; - socket.projectKey = projectKey; - socket.lastMessageReceivedAt = Date.now(); let {c_sessions, c_agents} = await sessions_agents_count(io, socket); if (socket.identity === IDENTITIES.session) { if (c_sessions > 0) { @@ -361,7 +357,6 @@ module.exports = { }); socket.onAny(async (eventName, ...args) => { - socket.lastMessageReceivedAt = Date.now(); if (socket.identity === IDENTITIES.session) { debug && console.log(`received event:${eventName}, from:${socket.identity}, sending message to room:${socket.peerId}`); socket.to(socket.peerId).emit(eventName, args[0]); diff --git a/ee/utilities/servers/websocket.js b/ee/utilities/servers/websocket.js index 256286351..521655b34 100644 --- a/ee/utilities/servers/websocket.js +++ b/ee/utilities/servers/websocket.js @@ -233,6 +233,16 @@ async function get_all_agents_ids(io, socket) { return agents; } +let geoip = null; +geoip2Reader.open(process.env.MAXMINDDB_FILE, {}) + .then(reader => { + geoip = reader; + }) + .catch(error => { + console.log("Error while opening the MAXMINDDB_FILE.") + console.error(error); + }); + function extractSessionInfo(socket) { if (socket.handshake.query.sessionInfo !== undefined) { debug && console.log("received headers"); @@ -246,21 +256,11 @@ function extractSessionInfo(socket) { socket.handshake.query.sessionInfo.userDevice = ua.device.model || null; socket.handshake.query.sessionInfo.userDeviceType = ua.device.type || 'desktop'; socket.handshake.query.sessionInfo.userCountry = null; - - const options = { - // you can use options like `cache` or `watchForUpdates` - }; - // console.log("Looking for MMDB file in " + process.env.MAXMINDDB_FILE); - geoip2Reader.open(process.env.MAXMINDDB_FILE, options) - .then(reader => { - debug && console.log("looking for location of "); - debug && console.log(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - let country = reader.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; - }) - .catch(error => { - console.error(error); - }); + if (geoip !== null) { + debug && console.log(`looking for location of ${socket.handshake.headers['x-forwarded-for'] || socket.handshake.address}`); + let country = geoip.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); + socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; + } } } @@ -272,10 +272,6 @@ module.exports = { debug && console.log(`WS started:${socket.id}, Query:${JSON.stringify(socket.handshake.query)}`); socket.peerId = socket.handshake.query.peerId; socket.identity = socket.handshake.query.identity; - const {projectKey, sessionId} = extractPeerId(socket.peerId); - socket.sessionId = sessionId; - socket.projectKey = projectKey; - socket.lastMessageReceivedAt = Date.now(); let {c_sessions, c_agents} = await sessions_agents_count(io, socket); if (socket.identity === IDENTITIES.session) { if (c_sessions > 0) { @@ -337,7 +333,6 @@ module.exports = { }); socket.onAny(async (eventName, ...args) => { - socket.lastMessageReceivedAt = Date.now(); if (socket.identity === IDENTITIES.session) { debug && console.log(`received event:${eventName}, from:${socket.identity}, sending message to room:${socket.peerId}`); socket.to(socket.peerId).emit(eventName, args[0]); diff --git a/utilities/Dockerfile b/utilities/Dockerfile index 171995974..9b82358b3 100644 --- a/utilities/Dockerfile +++ b/utilities/Dockerfile @@ -1,4 +1,4 @@ -FROM node:17-alpine +FROM node:17-stretch WORKDIR /work COPY . . RUN npm install diff --git a/utilities/server.js b/utilities/server.js index 0c80170af..5b32f5ec7 100644 --- a/utilities/server.js +++ b/utilities/server.js @@ -1,4 +1,4 @@ -const dumps = require('./utils/dump'); +const dumps = require('./utils/HeapSnapshot'); const sourcemapsReaderServer = require('./servers/sourcemaps-server'); const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); const express = require('express'); diff --git a/utilities/servers/websocket.js b/utilities/servers/websocket.js index 7371f206c..5f7000322 100644 --- a/utilities/servers/websocket.js +++ b/utilities/servers/websocket.js @@ -204,6 +204,16 @@ async function get_all_agents_ids(io, socket) { return agents; } +let geoip = null; +geoip2Reader.open(process.env.MAXMINDDB_FILE, {}) + .then(reader => { + geoip = reader; + }) + .catch(error => { + console.log("Error while opening the MAXMINDDB_FILE.") + console.error(error); + }); + function extractSessionInfo(socket) { if (socket.handshake.query.sessionInfo !== undefined) { debug && console.log("received headers"); @@ -217,21 +227,11 @@ function extractSessionInfo(socket) { socket.handshake.query.sessionInfo.userDevice = ua.device.model || null; socket.handshake.query.sessionInfo.userDeviceType = ua.device.type || 'desktop'; socket.handshake.query.sessionInfo.userCountry = null; - - const options = { - // you can use options like `cache` or `watchForUpdates` - }; - // console.log("Looking for MMDB file in " + process.env.MAXMINDDB_FILE); - geoip2Reader.open(process.env.MAXMINDDB_FILE, options) - .then(reader => { - debug && console.log("looking for location of "); - debug && console.log(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - let country = reader.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; - }) - .catch(error => { - console.error(error); - }); + if (geoip !== null) { + debug && console.log(`looking for location of ${socket.handshake.headers['x-forwarded-for'] || socket.handshake.address}`); + let country = geoip.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); + socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; + } } } @@ -243,10 +243,6 @@ module.exports = { debug && console.log(`WS started:${socket.id}, Query:${JSON.stringify(socket.handshake.query)}`); socket.peerId = socket.handshake.query.peerId; socket.identity = socket.handshake.query.identity; - const {projectKey, sessionId} = extractPeerId(socket.peerId); - socket.sessionId = sessionId; - socket.projectKey = projectKey; - socket.lastMessageReceivedAt = Date.now(); let {c_sessions, c_agents} = await sessions_agents_count(io, socket); if (socket.identity === IDENTITIES.session) { if (c_sessions > 0) { @@ -308,7 +304,6 @@ module.exports = { }); socket.onAny(async (eventName, ...args) => { - socket.lastMessageReceivedAt = Date.now(); if (socket.identity === IDENTITIES.session) { debug && console.log(`received event:${eventName}, from:${socket.identity}, sending message to room:${socket.peerId}`); socket.to(socket.peerId).emit(eventName, args[0]); diff --git a/utilities/utils/dump.js b/utilities/utils/HeapSnapshot.js similarity index 100% rename from utilities/utils/dump.js rename to utilities/utils/HeapSnapshot.js From b7dc03539b8f771d4dff735e71f516407e0166bb Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 12 Apr 2022 18:13:31 +0200 Subject: [PATCH 088/125] feat(utilities): EE cluster-ws fixes --- ee/utilities/servers/websocket-cluster.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ee/utilities/servers/websocket-cluster.js b/ee/utilities/servers/websocket-cluster.js index 8dc3bf94c..e2e6aaa86 100644 --- a/ee/utilities/servers/websocket-cluster.js +++ b/ee/utilities/servers/websocket-cluster.js @@ -87,8 +87,7 @@ const extractProjectKeyFromRequest = function (req) { const getAvailableRooms = async function () { - let rooms = await io.of('/').adapter.allRooms(); - return rooms; + return io.of('/').adapter.allRooms(); } const respond = function (res, data) { @@ -175,7 +174,7 @@ const socketsLive = async function (req, res) { } } } - liveSessions[projectKey] = uniqueSessions(liveSessions[_projectKey]); + liveSessions[projectKey] = uniqueSessions(liveSessions[projectKey]); } } respond(res, liveSessions); @@ -204,7 +203,7 @@ const socketsLiveByProject = async function (req, res) { } } } - liveSessions[projectKey] = uniqueSessions(liveSessions[_projectKey]); + liveSessions[projectKey] = uniqueSessions(liveSessions[projectKey] || []); } } respond(res, liveSessions[_projectKey] || []); From e20b0d4f68b9c8eb14279a1986f0c9a76d8ad0a1 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 12 Apr 2022 19:20:26 +0200 Subject: [PATCH 089/125] feat(api): use BackgroundTasks to flag viewed sessions feat(api): use BackgroundTasks to flag viewed errors --- api/routers/core.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/api/routers/core.py b/api/routers/core.py index 06743c054..813577b88 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -1,7 +1,7 @@ from typing import Union from decouple import config -from fastapi import Depends, Body +from fastapi import Depends, Body, BackgroundTasks import schemas from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assignments, projects, \ @@ -23,7 +23,8 @@ public_app, app, app_apikey = get_routers() @app.get('/{projectId}/sessions/{sessionId}', tags=["sessions"]) @app.get('/{projectId}/sessions2/{sessionId}', tags=["sessions"]) -def get_session2(projectId: int, sessionId: Union[int, str], context: schemas.CurrentContext = Depends(OR_context)): +def get_session2(projectId: int, sessionId: Union[int, str], background_tasks: BackgroundTasks, + context: schemas.CurrentContext = Depends(OR_context)): if isinstance(sessionId, str): return {"errors": ["session not found"]} data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, user_id=context.user_id, @@ -31,7 +32,8 @@ def get_session2(projectId: int, sessionId: Union[int, str], context: schemas.Cu if data is None: return {"errors": ["session not found"]} if data.get("inDB"): - sessions_favorite_viewed.view_session(project_id=projectId, user_id=context.user_id, session_id=sessionId) + background_tasks.add_task(sessions_favorite_viewed.view_session, project_id=projectId, user_id=context.user_id, + session_id=sessionId) return { 'data': data } @@ -833,7 +835,8 @@ def sessions_live(projectId: int, userId: str = None, context: schemas.CurrentCo @app.get('/{projectId}/assist/sessions/{sessionId}', tags=["assist"]) -def get_live_session(projectId: int, sessionId: str, context: schemas.CurrentContext = Depends(OR_context)): +def get_live_session(projectId: int, sessionId: str, background_tasks: BackgroundTasks, + context: schemas.CurrentContext = Depends(OR_context)): data = assist.get_live_session_by_id(project_id=projectId, session_id=sessionId) if data is None: data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, @@ -841,7 +844,8 @@ def get_live_session(projectId: int, sessionId: str, context: schemas.CurrentCon if data is None: return {"errors": ["session not found"]} if data.get("inDB"): - sessions_favorite_viewed.view_session(project_id=projectId, user_id=context.user_id, session_id=sessionId) + background_tasks.add_task(sessions_favorite_viewed.view_session, project_id=projectId, + user_id=context.user_id, session_id=sessionId) return {'data': data} @@ -909,12 +913,14 @@ def errors_stats(projectId: int, startTimestamp: int, endTimestamp: int, @app.get('/{projectId}/errors/{errorId}', tags=['errors']) -def errors_get_details(projectId: int, errorId: str, density24: int = 24, density30: int = 30, +def errors_get_details(projectId: int, errorId: str, background_tasks: BackgroundTasks, density24: int = 24, + density30: int = 30, context: schemas.CurrentContext = Depends(OR_context)): data = errors.get_details(project_id=projectId, user_id=context.user_id, error_id=errorId, **{"density24": density24, "density30": density30}) if data.get("data") is not None: - errors_favorite_viewed.viewed_error(project_id=projectId, user_id=context.user_id, error_id=errorId) + background_tasks.add_task(errors_favorite_viewed.viewed_error, project_id=projectId, user_id=context.user_id, + error_id=errorId) return data From e31dd82b92b90c66fd8e2e6e5f9d205a3151beb1 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 12 Apr 2022 19:44:34 +0200 Subject: [PATCH 090/125] feat(api): optimized get session replay --- api/chalicelib/core/resources.py | 10 +++++----- api/chalicelib/core/sessions.py | 2 +- ee/api/chalicelib/core/resources.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/chalicelib/core/resources.py b/api/chalicelib/core/resources.py index 6a7e395f8..d85e56b6f 100644 --- a/api/chalicelib/core/resources.py +++ b/api/chalicelib/core/resources.py @@ -1,23 +1,23 @@ from chalicelib.utils import helper, pg_client -def get_by_session_id(session_id): +def get_by_session_id(session_id, project_id): with pg_client.PostgresClient() as cur: ch_query = """\ SELECT timestamp AS datetime, url, type, - duration, + resources.duration AS duration, ttfb, header_size, encoded_body_size, decoded_body_size, success, COALESCE(status, CASE WHEN success THEN 200 END) AS status - FROM events.resources - WHERE session_id = %(session_id)s;""" - params = {"session_id": session_id} + FROM events.resources INNER JOIN sessions USING (session_id) + WHERE session_id = %(session_id)s AND project_id= %(project_id)s;""" + params = {"session_id": session_id, "project_id": project_id} cur.execute(cur.mogrify(ch_query, params)) rows = cur.fetchall() return helper.list_to_camel_case(rows) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 83cad2ad7..c0fdf57bd 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -94,7 +94,7 @@ def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_ data['userEvents'] = events.get_customs_by_sessionId2_pg(project_id=project_id, session_id=session_id) data['mobsUrl'] = sessions_mobs.get_web(sessionId=session_id) - data['resources'] = resources.get_by_session_id(session_id=session_id) + data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id) data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['issues'] = issues.get_by_session_id(session_id=session_id) diff --git a/ee/api/chalicelib/core/resources.py b/ee/api/chalicelib/core/resources.py index 332d3709a..4e4f1c4e8 100644 --- a/ee/api/chalicelib/core/resources.py +++ b/ee/api/chalicelib/core/resources.py @@ -3,14 +3,14 @@ from chalicelib.utils import ch_client from chalicelib.utils.TimeUTC import TimeUTC -def get_by_session_id(session_id): +def get_by_session_id(session_id, project_id): with ch_client.ClickHouseClient() as ch: ch_query = """\ SELECT datetime,url,type,duration,ttfb,header_size,encoded_body_size,decoded_body_size,success,coalesce(status,if(success, 200, status)) AS status FROM resources - WHERE session_id = toUInt64(%(session_id)s);""" - params = {"session_id": session_id} + WHERE session_id = toUInt64(%(session_id)s) AND project_id=%(project_id)s;""" + params = {"session_id": session_id, "project_id": project_id} rows = ch.execute(query=ch_query, params=params) results = [] for r in rows: From d285310852537580daa753e1be3cd2b97ce328b6 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 12 Apr 2022 19:47:09 +0200 Subject: [PATCH 091/125] feat(utilities): protected heap-snapshot endpoints --- utilities/utils/HeapSnapshot.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utilities/utils/HeapSnapshot.js b/utilities/utils/HeapSnapshot.js index 23708f07c..dc0b45e9a 100644 --- a/utilities/utils/HeapSnapshot.js +++ b/utilities/utils/HeapSnapshot.js @@ -57,7 +57,7 @@ function createNewHeapSnapshot(req, res) { res.end(JSON.stringify({path: location + fileName, 'done': creationStatus})); } -router.get('/status', getHeapSnapshotStatus); -router.get(`/new`, createNewHeapSnapshot); -router.get(`/download`, downloadHeapSnapshot); +router.get(`/${process.env.S3_KEY}/status`, getHeapSnapshotStatus); +router.get(`/${process.env.S3_KEY}/new`, createNewHeapSnapshot); +router.get(`/${process.env.S3_KEY}/download`, downloadHeapSnapshot); module.exports = {router} \ No newline at end of file From 98109ee9a3c05387c41ce0a354ca30eed86c62b9 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 12:34:40 +0200 Subject: [PATCH 092/125] feat(api): dashboard added chart to avg_dom_content_load_start --- api/chalicelib/core/dashboard.py | 54 ++++++++++++++++++++++------- ee/api/chalicelib/core/dashboard.py | 34 ++++++++++++++++++ 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index 362e4c491..b027beb87 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -1483,7 +1483,7 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] return {"value": avg, "chart": helper.list_to_camel_case(rows), - "unit": schemas.TemplatePredefinedUnits.percentage} + "unit": schemas.TemplatePredefinedUnits.percentage} def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -2426,12 +2426,7 @@ def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now endTimestamp=TimeUTC.now(), density=19, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) - location_constraints = [] - img_constraints = [] request_constraints = [] - - img_constraints_vals = {} - location_constraints_vals = {} request_constraints_vals = {} params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, @@ -2470,16 +2465,16 @@ def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: - rows = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args) - if len(rows) > 0: - results = helper.dict_to_camel_case(rows[0]) + row = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + results["chart"] = __get_page_metrics_avg_dom_content_load_start_chart(cur, project_id, startTimestamp, + endTimestamp, **args) diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff - rows = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args) - if len(rows) > 0: - previous = helper.dict_to_camel_case(rows[0]) - results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + row = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) results["unit"] = schemas.TemplatePredefinedUnits.millisecond return results @@ -2498,6 +2493,39 @@ def __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestam params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return row + + +def __get_page_metrics_avg_dom_content_load_start_chart(cur, project_id, startTimestamp, endTimestamp, density=19, + **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="pages", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s") + pg_sub_query_subset.append("pages.dom_content_loaded_time > 0") + + pg_query = f"""WITH pages AS(SELECT pages.dom_content_loaded_time, pages.timestamp + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT generated_timestamp AS timestamp, + COALESCE(AVG(pages.dom_content_loaded_time),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT pages.dom_content_loaded_time + FROM pages + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS pages ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **__get_constraint_values(args)})) rows = cur.fetchall() return rows diff --git a/ee/api/chalicelib/core/dashboard.py b/ee/api/chalicelib/core/dashboard.py index c36c877da..9e56cc351 100644 --- a/ee/api/chalicelib/core/dashboard.py +++ b/ee/api/chalicelib/core/dashboard.py @@ -1,6 +1,7 @@ import math import random +import schemas from chalicelib.utils import pg_client from chalicelib.utils import args_transformer from chalicelib.utils import helper @@ -2322,9 +2323,12 @@ def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with ch_client.ClickHouseClient() as ch: + results = {} rows = __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args) if len(rows) > 0: results = helper.dict_to_camel_case(rows[0]) + results["chart"] = __get_page_metrics_avg_dom_content_load_start_chart(ch, project_id, startTimestamp, + endTimestamp, **args) diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff @@ -2332,6 +2336,7 @@ def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeU if len(rows) > 0: previous = helper.dict_to_camel_case(rows[0]) results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond return results @@ -2349,6 +2354,35 @@ def __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp return rows +def __get_page_metrics_avg_dom_content_load_start_chart(ch, project_id, startTimestamp, endTimestamp, density=19, + **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) + ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query_chart += meta_condition + + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(pages.dom_content_loaded_event_end,0)) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + GROUP BY timestamp + ORDER BY timestamp;""" + rows = ch.execute(query=ch_query, params={**params, **__get_constraint_values(args)}) + rows = [{"timestamp": i["timestamp"], "value": i["value"]} for i in + __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, + density=density, neutral={"value": 0})] + + for s in rows: + for k in s: + if s[k] is None: + s[k] = 0 + return rows + + def get_page_metrics_avg_first_contentful_pixel(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with ch_client.ClickHouseClient() as ch: From ae9580b34521773dae4d1d932f690a5182bda50f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 12:42:04 +0200 Subject: [PATCH 093/125] feat(api): dashboard user single DB-connexion for charts and data of the same widget --- api/chalicelib/core/dashboard.py | 166 ++++++++++++++-------------- ee/api/chalicelib/core/dashboard.py | 127 +++++++++++---------- 2 files changed, 145 insertions(+), 148 deletions(-) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index b027beb87..1fb526d96 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -2265,7 +2265,7 @@ def get_application_activity_avg_image_load_time(project_id, startTimestamp=Time with pg_client.PostgresClient() as cur: row = __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) results = row - results["chart"] = get_performance_avg_image_load_time(project_id, startTimestamp, endTimestamp, **args) + results["chart"] = get_performance_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff @@ -2276,7 +2276,7 @@ def get_application_activity_avg_image_load_time(project_id, startTimestamp=Time return results -def get_performance_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), +def get_performance_avg_image_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) @@ -2286,34 +2286,33 @@ def get_performance_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(d params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} - with pg_client.PostgresClient() as cur: - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, - chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, - chart=True, data=args, main_table="resources", time_column="timestamp", - duration=False) - pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") - pg_query = f"""WITH resources AS (SELECT resources.duration, resources.timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - AND resources.type = 'img' AND resources.duration>0 - {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} - ) - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(resources.duration),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT resources.duration - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) - GROUP BY timestamp - ORDER BY timestamp;""" - cur.execute(cur.mogrify(pg_query, {**params, **img_constraints_vals, **__get_constraint_values(args)})) - rows = cur.fetchall() - rows = helper.list_to_camel_case(rows) + pg_query = f"""WITH resources AS (SELECT resources.duration, resources.timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + AND resources.type = 'img' AND resources.duration>0 + {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} + ) + SELECT generated_timestamp AS timestamp, + COALESCE(AVG(resources.duration),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT resources.duration + FROM resources + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS resources ON (TRUE) + GROUP BY timestamp + ORDER BY timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **img_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() + rows = helper.list_to_camel_case(rows) return rows @@ -2341,7 +2340,7 @@ def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeU with pg_client.PostgresClient() as cur: row = __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) results = row - results["chart"] = get_performance_avg_page_load_time(project_id, startTimestamp, endTimestamp, **args) + results["chart"] = get_performance_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff @@ -2352,7 +2351,7 @@ def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeU return results -def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), +def get_performance_avg_page_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) @@ -2360,31 +2359,30 @@ def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(de location_constraints_vals = {} params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} - with pg_client.PostgresClient() as cur: - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, - chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, - chart=True, data=args, main_table="pages", time_column="timestamp", - duration=False) - pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s") - pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s") - pg_query = f"""WITH pages AS(SELECT pages.load_time, timestamp - FROM events.pages INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} AND pages.load_time>0 AND pages.load_time IS NOT NULL - {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} - ) - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.load_time),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT pages.load_time - FROM pages - WHERE {" AND ".join(pg_sub_query_chart)} - {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} - ) AS pages ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {**params, **location_constraints_vals, **__get_constraint_values(args)})) - rows = cur.fetchall() + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="pages", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s") + pg_query = f"""WITH pages AS(SELECT pages.load_time, timestamp + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} AND pages.load_time>0 AND pages.load_time IS NOT NULL + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + ) + SELECT generated_timestamp AS timestamp, + COALESCE(AVG(pages.load_time),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT pages.load_time + FROM pages + WHERE {" AND ".join(pg_sub_query_chart)} + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + ) AS pages ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **location_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() return rows @@ -2411,7 +2409,7 @@ def get_application_activity_avg_request_load_time(project_id, startTimestamp=Ti with pg_client.PostgresClient() as cur: row = __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) results = row - results["chart"] = get_performance_avg_request_load_time(project_id, startTimestamp, endTimestamp, **args) + results["chart"] = get_performance_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff @@ -2422,7 +2420,7 @@ def get_application_activity_avg_request_load_time(project_id, startTimestamp=Ti return results -def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), +def get_performance_avg_request_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) @@ -2431,33 +2429,33 @@ def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} - with pg_client.PostgresClient() as cur: - pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, - chart=False, data=args) - pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, - chart=True, data=args, main_table="resources", time_column="timestamp", - duration=False) - pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") - pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") - pg_query = f"""WITH resources AS(SELECT resources.duration, resources.timestamp - FROM events.resources INNER JOIN public.sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query_subset)} - AND resources.type = 'fetch' AND resources.duration>0 - {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} - ) - SELECT generated_timestamp AS timestamp, - COALESCE(AVG(resources.duration),0) AS value - FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( - SELECT resources.duration - FROM resources - WHERE {" AND ".join(pg_sub_query_chart)} - ) AS resources ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""" - cur.execute(cur.mogrify(pg_query, {**params, **request_constraints_vals, **__get_constraint_values(args)})) - rows = cur.fetchall() + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") + + pg_query = f"""WITH resources AS(SELECT resources.duration, resources.timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + AND resources.type = 'fetch' AND resources.duration>0 + {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} + ) + SELECT generated_timestamp AS timestamp, + COALESCE(AVG(resources.duration),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT resources.duration + FROM resources + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **request_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() return rows diff --git a/ee/api/chalicelib/core/dashboard.py b/ee/api/chalicelib/core/dashboard.py index 9e56cc351..7e85c3169 100644 --- a/ee/api/chalicelib/core/dashboard.py +++ b/ee/api/chalicelib/core/dashboard.py @@ -2097,7 +2097,7 @@ def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeU with ch_client.ClickHouseClient() as ch: row = __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args) results = helper.dict_to_camel_case(row) - results["chart"] = get_performance_avg_page_load_time(project_id, startTimestamp, endTimestamp, **args) + results["chart"] = get_performance_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args) diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff @@ -2126,7 +2126,7 @@ def __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp return result -def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), +def get_performance_avg_page_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, resources=None, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) @@ -2143,30 +2143,30 @@ def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(de params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} - with ch_client.ClickHouseClient() as ch: - ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, - data=args) - ch_sub_query_chart += meta_condition - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(pages.load_event_end ,0)) AS value - FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} - WHERE {" AND ".join(ch_sub_query_chart)} - {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} - GROUP BY timestamp - ORDER BY timestamp;""" + ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, + data=args) + ch_sub_query_chart += meta_condition - rows = ch.execute(query=ch_query, - params={**params, **location_constraints_vals, **__get_constraint_values(args)}) - pages = [{"timestamp": i["timestamp"], "value": i["value"]} for i in - __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"value": 0})] + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(pages.load_event_end ,0)) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + GROUP BY timestamp + ORDER BY timestamp;""" - for s in pages: - for k in s: - if s[k] is None: - s[k] = 0 + rows = ch.execute(query=ch_query, + params={**params, **location_constraints_vals, **__get_constraint_values(args)}) + pages = [{"timestamp": i["timestamp"], "value": i["value"]} for i in + __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, + density=density, neutral={"value": 0})] + + for s in pages: + for k in s: + if s[k] is None: + s[k] = 0 return pages @@ -2175,7 +2175,7 @@ def get_application_activity_avg_image_load_time(project_id, startTimestamp=Time with ch_client.ClickHouseClient() as ch: row = __get_application_activity_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args) results = helper.dict_to_camel_case(row) - results["chart"] = get_performance_avg_image_load_time(project_id, startTimestamp, endTimestamp, **args) + results["chart"] = get_performance_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args) diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff @@ -2204,7 +2204,7 @@ def __get_application_activity_avg_image_load_time(ch, project_id, startTimestam return result -def get_performance_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), +def get_performance_avg_image_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, resources=None, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) @@ -2223,25 +2223,24 @@ def get_performance_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(d params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(resources.duration,0)) AS value - FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} - WHERE {" AND ".join(ch_sub_query_chart)} - AND resources.type = 'img' - {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, params={**params, **img_constraints_vals, **__get_constraint_values(args)}) - images = [{"timestamp": i["timestamp"], "value": i["value"]} for i in - __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, - density=density, neutral={"value": 0})] + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS value + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + AND resources.type = 'img' + {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} + GROUP BY timestamp + ORDER BY timestamp;""" + rows = ch.execute(query=ch_query, params={**params, **img_constraints_vals, **__get_constraint_values(args)}) + images = [{"timestamp": i["timestamp"], "value": i["value"]} for i in + __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, + density=density, neutral={"value": 0})] - for s in images: - for k in s: - if s[k] is None: - s[k] = 0 + for s in images: + for k in s: + if s[k] is None: + s[k] = 0 return images @@ -2250,7 +2249,7 @@ def get_application_activity_avg_request_load_time(project_id, startTimestamp=Ti with ch_client.ClickHouseClient() as ch: row = __get_application_activity_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args) results = helper.dict_to_camel_case(row) - results["chart"] = get_performance_avg_request_load_time(project_id, startTimestamp, endTimestamp, **args) + results["chart"] = get_performance_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args) diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff @@ -2279,7 +2278,7 @@ def __get_application_activity_avg_request_load_time(ch, project_id, startTimest return result -def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), +def get_performance_avg_request_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, resources=None, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) @@ -2297,26 +2296,26 @@ def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} - with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(resources.duration,0)) AS value - FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} - WHERE {" AND ".join(ch_sub_query_chart)} - AND resources.type = 'fetch' - {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} - GROUP BY timestamp - ORDER BY timestamp;""" - rows = ch.execute(query=ch_query, - params={**params, **request_constraints_vals, **__get_constraint_values(args)}) - requests = [{"timestamp": i["timestamp"], "value": i["value"]} for i in - __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, density=density, - neutral={"value": 0})] - for s in requests: - for k in s: - if s[k] is None: - s[k] = 0 + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS value + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + AND resources.type = 'fetch' + {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} + GROUP BY timestamp + ORDER BY timestamp;""" + rows = ch.execute(query=ch_query, + params={**params, **request_constraints_vals, **__get_constraint_values(args)}) + requests = [{"timestamp": i["timestamp"], "value": i["value"]} for i in + __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, density=density, + neutral={"value": 0})] + + for s in requests: + for k in s: + if s[k] is None: + s[k] = 0 return requests From ae93db4a7e5957f3887106f2b391011b89e6d2e6 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 13:56:54 +0200 Subject: [PATCH 094/125] feat(db): requests structure change --- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 299 +++++++++++++++--- .../db/init_dbs/postgresql/init_schema.sql | 288 ++++++++++++++--- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 15 +- .../db/init_dbs/postgresql/init_schema.sql | 288 ++++++++++++++--- 4 files changed, 744 insertions(+), 146 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 8cafd767d..3867fb5df 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -24,13 +24,17 @@ ALTER TABLE IF EXISTS metrics DROP CONSTRAINT IF EXISTS unique_key; ALTER TABLE IF EXISTS metrics - ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', + ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, - ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, - ADD COLUMN IF NOT EXISTS default_config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, + ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS default_config jsonb NOT NULL DEFAULT '{ + "col": 2, + "row": 2, + "position": 0 + }'::jsonb, ALTER COLUMN project_id DROP NOT NULL, ADD CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), @@ -48,6 +52,11 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets config jsonb NOT NULL DEFAULT '{}'::jsonb ); +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS host text NULL, + ADD COLUMN IF NOT EXISTS base_path text NULL, + ADD COLUMN IF NOT EXISTS query text NULL; + COMMIT; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'barChart'; @@ -58,55 +67,236 @@ ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'map'; ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'predefined'; -INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) -VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), - ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), - ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), - ('Image Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), - ('DOM Content Load Start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), - ('First Meaningful paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), - ('No. of Visited Pages', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), - ('Session Duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), - ('DOM Build Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), - ('Pages Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), - ('Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), - ('First Paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), - ('DOM Content Loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), - ('Time Till First byte', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), - ('Time To Interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), - ('Captured requests', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), - ('Time To Render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), - ('Memory Consumption', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), - ('CPU Load', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), - ('Frame rate', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_fps', 'predefined', 'overview'), +INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, + view_type) +VALUES ('Captured sessions', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 +}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('Request Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('Page Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('Image Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('DOM Content Load Start', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('First Meaningful paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('No. of Visited Pages', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('Session Duration', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('DOM Build Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('Pages Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('First Paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('DOM Content Loaded', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('Time Till First byte', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), + ('Time To Interactive', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('Captured requests', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'count_requests', 'predefined', 'overview'), + ('Time To Render', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('Memory Consumption', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('CPU Load', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('Frame rate', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_fps', 'predefined', 'overview'), - ('Sessions Affected by JS Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), - ('Top Domains with 4xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), - ('Top Domains with 5xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), - ('Errors per Domain', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), - ('Fetch Calls with Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), - ('Errors by Type', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), - ('Errors by Origin', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + ('Sessions Affected by JS Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('Top Domains with 4xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('Top Domains with 5xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('Errors per Domain', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('Fetch Calls with Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'calls_errors', 'predefined', 'table'), + ('Errors by Type', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('Errors by Origin', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), - ('Speed Index by Location', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), - ('Slowest Domains', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), - ('Sessions per Browser', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), - ('Time To Render', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), - ('Sessions Impacted by Slow Pages', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), - ('Memory Consumption', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), - ('CPU Load', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), - ('Frame Rate', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), - ('Crashes', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), - ('Resources Loaded vs Visually Complete', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), - ('DOM Build Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), - ('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), - ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), + ('Speed Index by Location', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'speed_location', 'predefined', 'map'), + ('Slowest Domains', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('Sessions per Browser', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('Time To Render', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('Sessions Impacted by Slow Pages', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('Memory Consumption', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('CPU Load', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('Frame Rate', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'fps', 'predefined', 'areaChart'), + ('Crashes', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('Resources Loaded vs Visually Complete', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('DOM Build Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('Pages Response Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('Pages Response Time Distribution', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), - ('Missing Resources', 'resources', '{"col":4,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), - ('Slowest Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), - ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), - ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), - ('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') + ('Missing Resources', 'resources', '{ + "col": 4, + "row": 2, + "position": 0 + }', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Resources Fetch Time', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_loading_time', 'predefined', 'table'), + ('Resource Loaded vs Response End', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), + ('Breakdown of Loaded Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, @@ -115,4 +305,11 @@ ON CONFLICT (predefined_key) DO UPDATE is_template=excluded.is_template, is_public=excluded.is_public, metric_type=excluded.metric_type, - view_type=excluded.view_type; \ No newline at end of file + view_type=excluded.view_type; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 5abd6f026..b83f2f84d 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -811,7 +811,11 @@ $$ is_predefined boolean NOT NULL DEFAULT FALSE, is_template boolean NOT NULL DEFAULT FALSE, predefined_key text NULL DEFAULT NULL, - default_config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, + default_config jsonb NOT NULL DEFAULT '{ + "col": 2, + "row": 2, + "position": 0 + }'::jsonb, CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), CONSTRAINT unique_key UNIQUE (predefined_key) @@ -1229,6 +1233,9 @@ $$ response_body text NULL, status_code smallint NULL, method http_method NULL, + host text NULL, + base_path text NULL, + query text NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); CREATE INDEX IF NOT EXISTS requests_url_idx ON events_common.requests (url); @@ -1250,6 +1257,12 @@ $$ CREATE INDEX IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; CREATE INDEX IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; CREATE INDEX IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; END IF; @@ -1257,55 +1270,236 @@ $$ $$ LANGUAGE plpgsql; -INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) -VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), - ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), - ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), - ('Image Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), - ('DOM Content Load Start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), - ('First Meaningful paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), - ('No. of Visited Pages', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), - ('Session Duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), - ('DOM Build Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), - ('Pages Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), - ('Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), - ('First Paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), - ('DOM Content Loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), - ('Time Till First byte', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), - ('Time To Interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), - ('Captured requests', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), - ('Time To Render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), - ('Memory Consumption', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), - ('CPU Load', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), - ('Frame rate', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_fps', 'predefined', 'overview'), +INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, + view_type) +VALUES ('Captured sessions', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 +}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('Request Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('Page Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('Image Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('DOM Content Load Start', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('First Meaningful paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('No. of Visited Pages', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('Session Duration', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('DOM Build Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('Pages Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('First Paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('DOM Content Loaded', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('Time Till First byte', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), + ('Time To Interactive', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('Captured requests', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'count_requests', 'predefined', 'overview'), + ('Time To Render', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('Memory Consumption', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('CPU Load', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('Frame rate', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_fps', 'predefined', 'overview'), - ('Sessions Affected by JS Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), - ('Top Domains with 4xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), - ('Top Domains with 5xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), - ('Errors per Domain', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), - ('Fetch Calls with Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), - ('Errors by Type', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), - ('Errors by Origin', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + ('Sessions Affected by JS Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('Top Domains with 4xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('Top Domains with 5xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('Errors per Domain', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('Fetch Calls with Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'calls_errors', 'predefined', 'table'), + ('Errors by Type', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('Errors by Origin', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), - ('Speed Index by Location', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), - ('Slowest Domains', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), - ('Sessions per Browser', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), - ('Time To Render', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), - ('Sessions Impacted by Slow Pages', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), - ('Memory Consumption', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), - ('CPU Load', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), - ('Frame Rate', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), - ('Crashes', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), - ('Resources Loaded vs Visually Complete', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), - ('DOM Build Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), - ('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), - ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), + ('Speed Index by Location', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'speed_location', 'predefined', 'map'), + ('Slowest Domains', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('Sessions per Browser', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('Time To Render', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('Sessions Impacted by Slow Pages', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('Memory Consumption', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('CPU Load', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('Frame Rate', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'fps', 'predefined', 'areaChart'), + ('Crashes', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('Resources Loaded vs Visually Complete', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('DOM Build Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('Pages Response Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('Pages Response Time Distribution', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), - ('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), - ('Slowest Resources', 'resources', '{"col":4,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), - ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), - ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), - ('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') + ('Missing Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{ + "col": 4, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Resources Fetch Time', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_loading_time', 'predefined', 'table'), + ('Resource Loaded vs Response End', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), + ('Breakdown of Loaded Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index c50f6fc42..f639f3973 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -48,6 +48,11 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets config jsonb NOT NULL DEFAULT '{}'::jsonb ); +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS host text NULL, + ADD COLUMN IF NOT EXISTS base_path text NULL, + ADD COLUMN IF NOT EXISTS query text NULL; + COMMIT; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'barChart'; @@ -114,4 +119,12 @@ ON CONFLICT (predefined_key) DO UPDATE is_template=excluded.is_template, is_public=excluded.is_public, metric_type=excluded.metric_type, - view_type=excluded.view_type; \ No newline at end of file + view_type=excluded.view_type; + + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index d78d0754b..1a77e2b7b 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -645,6 +645,9 @@ $$ response_body text NULL, status_code smallint NULL, method http_method NULL, + host text NULL, + base_path text NULL, + query text NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); CREATE INDEX requests_url_idx ON events_common.requests (url); @@ -664,6 +667,12 @@ $$ CREATE INDEX requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; CREATE INDEX requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; CREATE INDEX requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + CREATE INDEX requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; + CREATE INDEX requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; + CREATE INDEX requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; + CREATE INDEX requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; + CREATE INDEX requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; + CREATE INDEX requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; -- --- events.sql --- CREATE SCHEMA IF NOT EXISTS events; @@ -958,7 +967,11 @@ $$ is_predefined boolean NOT NULL DEFAULT FALSE, is_template boolean NOT NULL DEFAULT FALSE, predefined_key text NULL DEFAULT NULL, - default_config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, + default_config jsonb NOT NULL DEFAULT '{ + "col": 2, + "row": 2, + "position": 0 + }'::jsonb, CONSTRAINT null_project_id_for_template_only CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), CONSTRAINT unique_key UNIQUE (predefined_key) @@ -1048,55 +1061,236 @@ $$ $$ LANGUAGE plpgsql; -INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) -VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), - ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), - ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), - ('Image Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), - ('DOM Content Load Start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), - ('First Meaningful paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), - ('No. of Visited Pages', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), - ('Session Duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), - ('DOM Build Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), - ('Pages Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), - ('Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), - ('First Paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), - ('DOM Content Loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), - ('Time Till First byte', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), - ('Time To Interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), - ('Captured requests', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), - ('Time To Render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), - ('Memory Consumption', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), - ('CPU Load', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), - ('Frame rate', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_fps', 'predefined', 'overview'), +INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, + view_type) +VALUES ('Captured sessions', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 +}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('Request Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('Page Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('Image Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('DOM Content Load Start', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('First Meaningful paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('No. of Visited Pages', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('Session Duration', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('DOM Build Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('Pages Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('First Paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('DOM Content Loaded', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('Time Till First byte', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), + ('Time To Interactive', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('Captured requests', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'count_requests', 'predefined', 'overview'), + ('Time To Render', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('Memory Consumption', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('CPU Load', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('Frame rate', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_fps', 'predefined', 'overview'), - ('Sessions Affected by JS Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), - ('Top Domains with 4xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), - ('Top Domains with 5xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), - ('Errors per Domain', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), - ('Fetch Calls with Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), - ('Errors by Type', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), - ('Errors by Origin', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + ('Sessions Affected by JS Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('Top Domains with 4xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('Top Domains with 5xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('Errors per Domain', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('Fetch Calls with Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'calls_errors', 'predefined', 'table'), + ('Errors by Type', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('Errors by Origin', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), - ('Speed Index by Location', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), - ('Slowest Domains', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), - ('Sessions per Browser', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), - ('Time To Render', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), - ('Sessions Impacted by Slow Pages', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), - ('Memory Consumption', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), - ('CPU Load', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), - ('Frame Rate', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), - ('Crashes', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), - ('Resources Loaded vs Visually Complete', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), - ('DOM Build Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), - ('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), - ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), + ('Speed Index by Location', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'speed_location', 'predefined', 'map'), + ('Slowest Domains', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('Sessions per Browser', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('Time To Render', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('Sessions Impacted by Slow Pages', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('Memory Consumption', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('CPU Load', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('Frame Rate', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'fps', 'predefined', 'areaChart'), + ('Crashes', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('Resources Loaded vs Visually Complete', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('DOM Build Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('Pages Response Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('Pages Response Time Distribution', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), - ('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), - ('Slowest Resources', 'resources', '{"col":4,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), - ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), - ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), - ('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') + ('Missing Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{ + "col": 4, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Resources Fetch Time', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_loading_time', 'predefined', 'table'), + ('Resource Loaded vs Response End', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), + ('Breakdown of Loaded Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') ON CONFLICT (predefined_key) DO UPDATE SET name=excluded.name, category=excluded.category, From 3bc9bd8df4fbdfad58aca0ef9a6ffe5a285be052 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 13 Apr 2022 14:26:39 +0200 Subject: [PATCH 095/125] feat(backend): insert url parts for pages and requests --- backend/pkg/db/postgres/messages-web-stats.go | 2 +- backend/pkg/db/postgres/messages-web.go | 44 ++++++++++++------- backend/pkg/url/url.go | 13 +++--- 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/backend/pkg/db/postgres/messages-web-stats.go b/backend/pkg/db/postgres/messages-web-stats.go index 933442b0b..27a6272e2 100644 --- a/backend/pkg/db/postgres/messages-web-stats.go +++ b/backend/pkg/db/postgres/messages-web-stats.go @@ -35,7 +35,7 @@ func (conn *Conn) InsertWebStatsPerformance(sessionID uint64, p *PerformanceTrac } func (conn *Conn) InsertWebStatsResourceEvent(sessionID uint64, e *ResourceEvent) error { - host, _, err := url.GetURLParts(e.URL) + host, _, _, err := url.GetURLParts(e.URL) if err != nil { return err } diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index 0d822cddd..ff7352594 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -55,7 +55,7 @@ func (conn *Conn) InsertWebUserAnonymousID(sessionID uint64, userAnonymousID *Us // TODO: fix column "dom_content_loaded_event_end" of relation "pages" func (conn *Conn) InsertWebPageEvent(sessionID uint64, e *PageEvent) error { - host, path, err := url.GetURLParts(e.URL) + host, path, query, err := url.GetURLParts(e.URL) if err != nil { return err } @@ -64,20 +64,27 @@ func (conn *Conn) InsertWebPageEvent(sessionID uint64, e *PageEvent) error { return err } defer tx.rollback() + // base_path is depricated if err := tx.exec(` INSERT INTO events.pages ( - session_id, message_id, timestamp, referrer, base_referrer, host, path, base_path, + session_id, message_id, timestamp, referrer, base_referrer, host, path, query, dom_content_loaded_time, load_time, response_end, first_paint_time, first_contentful_paint_time, speed_index, visually_complete, time_to_interactive, - response_time, dom_building_time + response_time, dom_building_time, + base_path ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, + $1, $2, $3, + $4, $5, + $6, $7, $8, NULLIF($9, 0), NULLIF($10, 0), NULLIF($11, 0), NULLIF($12, 0), NULLIF($13, 0), NULLIF($14, 0), NULLIF($15, 0), NULLIF($16, 0), - NULLIF($17, 0), NULLIF($18, 0) + NULLIF($17, 0), NULLIF($18, 0), + '', ) `, - sessionID, e.MessageID, e.Timestamp, e.Referrer, url.DiscardURLQuery(e.Referrer), host, path, url.DiscardURLQuery(path), + sessionID, e.MessageID, e.Timestamp, + e.Referrer, url.DiscardURLQuery(e.Referrer), + host, path, query, e.DomContentLoadedEventEnd, e.LoadEventEnd, e.ResponseEnd, e.FirstPaint, e.FirstContentfulPaint, e.SpeedIndex, e.VisuallyComplete, e.TimeToInteractive, calcResponseTime(e), calcDomBuildingTime(e), @@ -109,7 +116,7 @@ func (conn *Conn) InsertWebClickEvent(sessionID uint64, e *ClickEvent) error { INSERT INTO events.clicks (session_id, message_id, timestamp, label, selector, url) (SELECT - $1, $2, $3, NULLIF($4, ''), $5, host || base_path + $1, $2, $3, NULLIF($4, ''), $5, host || path FROM events.pages WHERE session_id = $1 AND timestamp <= $3 ORDER BY timestamp DESC LIMIT 1 ) @@ -211,19 +218,26 @@ func (conn *Conn) InsertWebFetchEvent(sessionID uint64, savePayload bool, e *Fet response = &e.Response } conn.insertAutocompleteValue(sessionID, "REQUEST", url.DiscardURLQuery(e.URL)) + host, path, query, err := url.GetURLParts(e.URL) + if err != nil { + return err + } return conn.batchQueue(sessionID, ` INSERT INTO events_common.requests ( - session_id, timestamp, - seq_index, url, duration, success, - request_body, response_body, status_code, method + session_id, timestamp, seq_index, + url, host, path, query, + request_body, response_body, status_code, method, + duration, success, ) VALUES ( - $1, $2, - $3, $4, $5, $6, - $7, $8, $9::smallint, NULLIF($10, '')::http_method + $1, $2, $3, + $4, $5, $6, $7 + $8, $9, $10::smallint, NULLIF($11, '')::http_method, + $12, $13 ) ON CONFLICT DO NOTHING`, - sessionID, e.Timestamp, - getSqIdx(e.MessageID), e.URL, e.Duration, e.Status < 400, + sessionID, e.Timestamp, getSqIdx(e.MessageID), + e.URL, host, path, query, request, response, e.Status, url.EnsureMethod(e.Method), + e.Duration, e.Status < 400, ) } diff --git a/backend/pkg/url/url.go b/backend/pkg/url/url.go index b9181774d..48cd0ef8d 100644 --- a/backend/pkg/url/url.go +++ b/backend/pkg/url/url.go @@ -1,18 +1,19 @@ package url import ( - "strings" _url "net/url" + "strings" ) func DiscardURLQuery(url string) string { return strings.Split(url, "?")[0] -} +} -func GetURLParts(rawURL string) (string, string, error) { +func GetURLParts(rawURL string) (string, string, string, error) { u, err := _url.Parse(rawURL) if err != nil { - return "", "", err + return "", "", "", err } - return u.Host, u.RequestURI(), nil -} \ No newline at end of file + // u.Scheme ? + return u.Host, u.RawPath, u.RawQuery, nil +} From abde6b62a85d7624bee8dbd70b175cb7a4c4fd37 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 14:44:35 +0200 Subject: [PATCH 096/125] feat(db): pages structure change --- ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql | 8 +++++++- ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql | 4 ++++ scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql | 6 ++++++ scripts/helm/db/init_dbs/postgresql/init_schema.sql | 3 +++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 3867fb5df..ba00ea335 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -57,7 +57,10 @@ ALTER TABLE events_common.requests ADD COLUMN IF NOT EXISTS base_path text NULL, ADD COLUMN IF NOT EXISTS query text NULL; +ALTER TABLE events.pages + ADD COLUMN IF NOT EXISTS query text NULL; COMMIT; + ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'barChart'; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarChart'; @@ -312,4 +315,7 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_commo CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; -CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; \ No newline at end of file +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index b83f2f84d..bc8d65f93 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -939,6 +939,7 @@ $$ host text NOT NULL, path text NOT NULL, base_path text NOT NULL, + query text NULL, referrer text DEFAULT NULL, base_referrer text DEFAULT NULL, dom_building_time integer DEFAULT NULL, @@ -998,6 +999,9 @@ $$ CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); CREATE INDEX IF NOT EXISTS pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; + CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; + CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; + CREATE TABLE IF NOT EXISTS events.clicks ( diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index f639f3973..9e6421b07 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -53,7 +53,10 @@ ALTER TABLE events_common.requests ADD COLUMN IF NOT EXISTS base_path text NULL, ADD COLUMN IF NOT EXISTS query text NULL; +ALTER TABLE events.pages + ADD COLUMN IF NOT EXISTS query text NULL; COMMIT; + ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'barChart'; ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarChart'; @@ -128,3 +131,6 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_comm CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 1a77e2b7b..7f4d864fe 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -685,6 +685,7 @@ $$ host text NOT NULL, path text NOT NULL, base_path text NOT NULL, + query text NULL, referrer text DEFAULT NULL, base_referrer text DEFAULT NULL, dom_building_time integer DEFAULT NULL, @@ -740,6 +741,8 @@ $$ CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; CREATE INDEX pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; + CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; + CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; CREATE TABLE events.clicks From 0fcaf7bce001fc47f1fa60acefa404f4eddb5ed1 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 13 Apr 2022 15:56:24 +0200 Subject: [PATCH 097/125] feat(backend): request autocomplete use path --- backend/pkg/db/postgres/messages-web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index ff7352594..eb4bca364 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -217,8 +217,8 @@ func (conn *Conn) InsertWebFetchEvent(sessionID uint64, savePayload bool, e *Fet request = &e.Request response = &e.Response } - conn.insertAutocompleteValue(sessionID, "REQUEST", url.DiscardURLQuery(e.URL)) host, path, query, err := url.GetURLParts(e.URL) + conn.insertAutocompleteValue(sessionID, "REQUEST", path) if err != nil { return err } From cbec34db3e6b5dada4793c1490528291f63272ba Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 16:23:31 +0200 Subject: [PATCH 098/125] feat(api): dashboard added charts for avg_dom_content_loaded widgets --- api/chalicelib/core/dashboard.py | 30 +++++++++++++++++++++----- ee/api/chalicelib/core/dashboard.py | 33 ++++++++++++++++++++++------- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index 1fb526d96..66ee770d8 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -2672,11 +2672,15 @@ def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, **args): + endTimestamp=TimeUTC.now(), value=None, density=19, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) if value is not None: pg_sub_query.append("pages.path = %(value)s") + pg_sub_query_chart.append("pages.path = %(value)s") with pg_client.PostgresClient() as cur: pg_query = f"""SELECT COALESCE(AVG(pages.dom_content_loaded_time), 0) AS value FROM events.pages @@ -2685,11 +2689,27 @@ def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.no AND pages.timestamp >= %(startTimestamp)s AND pages.timestamp < %(endTimestamp)s AND pages.dom_content_loaded_time > 0;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)})) + params = {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() + + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(AVG(NULLIF(pages.dom_content_loaded_time,0)),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT dom_content_loaded_time + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS pages ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp ASC;""" + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + row["chart"] = helper.list_to_camel_case(rows), row["unit"] = schemas.TemplatePredefinedUnits.millisecond return helper.dict_to_camel_case(row) diff --git a/ee/api/chalicelib/core/dashboard.py b/ee/api/chalicelib/core/dashboard.py index 7e85c3169..dc99ac0d2 100644 --- a/ee/api/chalicelib/core/dashboard.py +++ b/ee/api/chalicelib/core/dashboard.py @@ -2552,23 +2552,40 @@ def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), value=None, **args): - ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + endTimestamp=TimeUTC.now(), value=None, density=19, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density) + ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) meta_condition = __get_meta_constraint(args) + ch_sub_query_chart += meta_condition + + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) ch_sub_query += meta_condition if value is not None: ch_sub_query.append("pages.url_path = %(value)s") + ch_sub_query_chart.append("pages.url_path = %(value)s") with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT COALESCE(AVG(pages.dom_content_loaded_event_time),0) AS value FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.dom_content_loaded_event_time) AND pages.dom_content_loaded_event_time>0;""" - rows = ch.execute(query=ch_query, - params={"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "value": value, **__get_constraint_values(args)}) - return helper.dict_to_camel_case(rows[0]) + params = {"step_size": step_size, "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)} + rows = ch.execute(query=ch_query, params=params) + results = helper.dict_to_camel_case(rows[0]) + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, + COALESCE(AVG(NULLIF(pages.dom_content_loaded_event_time ,0)),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + GROUP BY timestamp + ORDER BY timestamp;;""" + rows = ch.execute(query=ch_query, params=params) + results["chart"] = helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, + density=density, + neutral={"value": 0})) + return results def get_top_metrics_avg_till_first_bit(project_id, startTimestamp=TimeUTC.now(delta_days=-1), From 154caec63b9eabee8231ddd6501cb620a879e5df Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 17:11:23 +0200 Subject: [PATCH 099/125] feat(utilities): utilities as a WS standalone server --- utilities/package-lock.json | 891 ------------------------ utilities/package.json | 3 - utilities/server.js | 49 +- utilities/servers/peerjs-server.js | 76 -- utilities/servers/sourcemaps-handler.js | 104 --- utilities/servers/sourcemaps-server.js | 30 - utilities/servers/websocket.js | 25 +- utilities/utils/HeapSnapshot.js | 9 +- utilities/utils/geoIP.js | 16 + utilities/utils/helper.js | 30 + 10 files changed, 67 insertions(+), 1166 deletions(-) delete mode 100644 utilities/servers/peerjs-server.js delete mode 100644 utilities/servers/sourcemaps-handler.js delete mode 100644 utilities/servers/sourcemaps-server.js create mode 100644 utilities/utils/geoIP.js create mode 100644 utilities/utils/helper.js diff --git a/utilities/package-lock.json b/utilities/package-lock.json index 724f7341a..df0b8c9ef 100644 --- a/utilities/package-lock.json +++ b/utilities/package-lock.json @@ -10,11 +10,8 @@ "license": "MIT", "dependencies": { "@maxmind/geoip2-node": "^3.4.0", - "aws-sdk": "^2.992.0", "express": "^4.17.1", - "peer": "^0.6.1", "socket.io": "^4.4.1", - "source-map": "^0.7.3", "ua-parser-js": "^1.0.2" } }, @@ -37,28 +34,11 @@ "node": ">= 0.6.0" } }, - "node_modules/@types/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, "node_modules/@types/component-emitter": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.11.tgz", "integrity": "sha512-SRXjM+tfsSlA9VuG8hGO2nft2p8zjXCK1VcC6N4NXbBbYbSia9kzCChYQajIjzIqOOOuh5Ock6MmV2oux4jDZQ==" }, - "node_modules/@types/connect": { - "version": "3.4.35", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", - "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/cookie": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz", @@ -69,64 +49,11 @@ "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" }, - "node_modules/@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.18", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.17.28", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", - "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "node_modules/@types/mime": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", - "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" - }, "node_modules/@types/node": { "version": "17.0.21", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" }, - "node_modules/@types/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" - }, - "node_modules/@types/range-parser": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "node_modules/@types/serve-static": { - "version": "1.13.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", - "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", - "dependencies": { - "@types/mime": "^1", - "@types/node": "*" - } - }, - "node_modules/@types/ws": { - "version": "7.4.7", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", - "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -139,28 +66,6 @@ "node": ">= 0.6" } }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -174,44 +79,6 @@ "node": ">=0.8" } }, - "node_modules/aws-sdk": { - "version": "2.1087.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", - "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", - "dependencies": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "uuid": "3.3.2", - "xml2js": "0.4.19" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/base64id": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz", @@ -240,16 +107,6 @@ "node": ">= 0.8" } }, - "node_modules/buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "dependencies": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -286,32 +143,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, "node_modules/component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", @@ -374,14 +205,6 @@ "ms": "2.0.0" } }, - "node_modules/decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -400,11 +223,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -498,14 +316,6 @@ "node": ">= 0.6" } }, - "node_modules/events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/express": { "version": "4.17.3", "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", @@ -571,18 +381,6 @@ "node": ">= 0.8" } }, - "node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -599,14 +397,6 @@ "node": ">= 0.6" } }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, "node_modules/http-errors": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", @@ -633,11 +423,6 @@ "node": ">=0.10.0" } }, - "node_modules/ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -660,27 +445,6 @@ "node": ">= 0.10" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "node_modules/jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", - "engines": { - "node": ">= 0.6.0" - } - }, "node_modules/json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", @@ -700,17 +464,6 @@ "verror": "1.10.0" } }, - "node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", @@ -832,39 +585,6 @@ "node": ">= 0.8" } }, - "node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "engines": { - "node": ">=6" - } - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -873,50 +593,11 @@ "node": ">= 0.8" } }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "engines": { - "node": ">=8" - } - }, "node_modules/path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "node_modules/peer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", - "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", - "dependencies": { - "@types/cors": "^2.8.6", - "@types/express": "^4.17.3", - "@types/ws": "^7.2.3", - "body-parser": "^1.19.0", - "cors": "^2.8.5", - "express": "^4.17.1", - "uuid": "^3.4.0", - "ws": "^7.2.3", - "yargs": "^15.3.1" - }, - "bin": { - "peerjs": "bin/peerjs" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/peer/node_modules/uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -929,11 +610,6 @@ "node": ">= 0.10" } }, - "node_modules/punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" - }, "node_modules/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", @@ -945,15 +621,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", - "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -987,19 +654,6 @@ "node": ">= 0.8" } }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1024,11 +678,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "node_modules/sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" - }, "node_modules/send": { "version": "0.17.2", "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", @@ -1071,11 +720,6 @@ "node": ">= 0.8.0" } }, - "node_modules/set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -1157,14 +801,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", - "engines": { - "node": ">= 8" - } - }, "node_modules/statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -1173,30 +809,6 @@ "node": ">= 0.6" } }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/tiny-lru": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-7.0.6.tgz", @@ -1262,15 +874,6 @@ "node": ">= 0.8" } }, - "node_modules/url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", - "dependencies": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -1279,15 +882,6 @@ "node": ">= 0.4.0" } }, - "node_modules/uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -1308,107 +902,6 @@ "core-util-is": "1.0.2", "extsprintf": "^1.2.0" } - }, - "node_modules/which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" - }, - "node_modules/wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/ws": { - "version": "7.5.7", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", - "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", - "engines": { - "node": ">=8.3.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/xml2js": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~9.0.1" - } - }, - "node_modules/xmlbuilder": { - "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/y18n": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" - }, - "node_modules/yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", - "dependencies": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "dependencies": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/yargs-parser/node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "engines": { - "node": ">=6" - } } }, "dependencies": { @@ -1428,28 +921,11 @@ "resolved": "https://registry.npmjs.org/@socket.io/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz", "integrity": "sha512-dOlCBKnDw4iShaIsH/bxujKTM18+2TOAsYz+KSc11Am38H4q5Xw8Bbz97ZYdrVNM+um3p7w86Bvvmcn9q+5+eQ==" }, - "@types/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", - "requires": { - "@types/connect": "*", - "@types/node": "*" - } - }, "@types/component-emitter": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.11.tgz", "integrity": "sha512-SRXjM+tfsSlA9VuG8hGO2nft2p8zjXCK1VcC6N4NXbBbYbSia9kzCChYQajIjzIqOOOuh5Ock6MmV2oux4jDZQ==" }, - "@types/connect": { - "version": "3.4.35", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", - "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", - "requires": { - "@types/node": "*" - } - }, "@types/cookie": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz", @@ -1460,64 +936,11 @@ "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" }, - "@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", - "requires": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.18", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "@types/express-serve-static-core": { - "version": "4.17.28", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", - "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", - "requires": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "@types/mime": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", - "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" - }, "@types/node": { "version": "17.0.21", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" }, - "@types/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" - }, - "@types/range-parser": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "@types/serve-static": { - "version": "1.13.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", - "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", - "requires": { - "@types/mime": "^1", - "@types/node": "*" - } - }, - "@types/ws": { - "version": "7.4.7", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", - "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", - "requires": { - "@types/node": "*" - } - }, "accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -1527,19 +950,6 @@ "negotiator": "0.6.3" } }, - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -1550,27 +960,6 @@ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, - "aws-sdk": { - "version": "2.1087.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", - "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", - "requires": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "uuid": "3.3.2", - "xml2js": "0.4.19" - } - }, - "base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" - }, "base64id": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz", @@ -1593,16 +982,6 @@ "type-is": "~1.6.18" } }, - "buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, "bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -1624,29 +1003,6 @@ "type-fest": "^1.2.1" } }, - "cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, "component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", @@ -1697,11 +1053,6 @@ "ms": "2.0.0" } }, - "decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" - }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -1717,11 +1068,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -1783,11 +1129,6 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, - "events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" - }, "express": { "version": "4.17.3", "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", @@ -1844,15 +1185,6 @@ "unpipe": "~1.0.0" } }, - "find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1863,11 +1195,6 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, - "get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" - }, "http-errors": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", @@ -1888,11 +1215,6 @@ "safer-buffer": ">= 2.1.2 < 3" } }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -1912,21 +1234,6 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==" - }, "json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", @@ -1943,14 +1250,6 @@ "verror": "1.10.0" } }, - "locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "requires": { - "p-locate": "^4.1.0" - } - }, "lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", @@ -2031,65 +1330,16 @@ "ee-first": "1.1.1" } }, - "p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "requires": { - "p-limit": "^2.2.0" - } - }, - "p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" - }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, - "path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" - }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "peer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", - "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", - "requires": { - "@types/cors": "^2.8.6", - "@types/express": "^4.17.3", - "@types/ws": "^7.2.3", - "body-parser": "^1.19.0", - "cors": "^2.8.5", - "express": "^4.17.1", - "uuid": "^3.4.0", - "ws": "^7.2.3", - "yargs": "^15.3.1" - }, - "dependencies": { - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - } - } - }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -2099,21 +1349,11 @@ "ipaddr.js": "1.9.1" } }, - "punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" - }, "qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==" }, - "querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" - }, "quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -2135,16 +1375,6 @@ "unpipe": "1.0.0" } }, - "require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" - }, - "require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2155,11 +1385,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" - }, "send": { "version": "0.17.2", "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", @@ -2198,11 +1423,6 @@ "send": "0.17.2" } }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, "setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -2266,34 +1486,11 @@ } } }, - "source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==" - }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - }, "tiny-lru": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-7.0.6.tgz", @@ -2328,25 +1525,11 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, - "url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", - "requires": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" - }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -2361,80 +1544,6 @@ "core-util-is": "1.0.2", "extsprintf": "^1.2.0" } - }, - "which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" - }, - "wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, - "ws": { - "version": "7.5.7", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", - "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", - "requires": {} - }, - "xml2js": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", - "requires": { - "sax": ">=0.6.0", - "xmlbuilder": "~9.0.1" - } - }, - "xmlbuilder": { - "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" - }, - "y18n": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" - }, - "yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", - "requires": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - } - }, - "yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "dependencies": { - "camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" - } - } } } } diff --git a/utilities/package.json b/utilities/package.json index 452c3fe00..483f1b4aa 100644 --- a/utilities/package.json +++ b/utilities/package.json @@ -19,11 +19,8 @@ "homepage": "https://github.com/openreplay/openreplay#readme", "dependencies": { "@maxmind/geoip2-node": "^3.4.0", - "aws-sdk": "^2.992.0", "express": "^4.17.1", - "peer": "^0.6.1", "socket.io": "^4.4.1", - "source-map": "^0.7.3", "ua-parser-js": "^1.0.2" } } diff --git a/utilities/server.js b/utilities/server.js index 5b32f5ec7..9fd8530fd 100644 --- a/utilities/server.js +++ b/utilities/server.js @@ -1,57 +1,22 @@ const dumps = require('./utils/HeapSnapshot'); -const sourcemapsReaderServer = require('./servers/sourcemaps-server'); -const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); const express = require('express'); -const {ExpressPeerServer} = require('peer'); const socket = require("./servers/websocket"); +const {request_logger} = require("./utils/helper"); const HOST = '0.0.0.0'; -const PORT = 9000; +const PORT = 9001; -const app = express(); const wsapp = express(); -let debug = process.env.debug === "1" || false; -const request_logger = (identity) => { - return (req, res, next) => { - debug && console.log(identity, new Date().toTimeString(), 'REQUEST', req.method, req.originalUrl); - res.on('finish', function () { - if (this.statusCode !== 200 || debug) { - console.log(new Date().toTimeString(), 'RESPONSE', req.method, req.originalUrl, this.statusCode); - } - }) - - next(); - } -}; -app.use(request_logger("[app]")); wsapp.use(request_logger("[wsapp]")); -app.use('/sourcemaps', sourcemapsReaderServer); -app.use('/assist', peerRouter); -wsapp.use('/assist', socket.wsRouter); +wsapp.use(`/assist/${process.env.S3_KEY}`, socket.wsRouter); +wsapp.use(`/heapdump/${process.env.S3_KEY}`, dumps.router); -app.use('/heapdump', dumps.router); - -const server = app.listen(PORT, HOST, () => { - console.log(`App listening on http://${HOST}:${PORT}`); +const wsserver = wsapp.listen(PORT, HOST, () => { + console.log(`WS App listening on http://${HOST}:${PORT}`); console.log('Press Ctrl+C to quit.'); }); -const wsserver = wsapp.listen(PORT + 1, HOST, () => { - console.log(`WS App listening on http://${HOST}:${PORT + 1}`); - console.log('Press Ctrl+C to quit.'); -}); -const peerServer = ExpressPeerServer(server, { - debug: true, - path: '/', - proxied: true, - allow_discovery: false -}); -peerServer.on('connection', peerConnection); -peerServer.on('disconnect', peerDisconnect); -peerServer.on('error', peerError); -app.use('/', peerServer); -app.enable('trust proxy'); wsapp.enable('trust proxy'); socket.start(wsserver); -module.exports = {wsserver, server}; +module.exports = {wsserver}; console.log(`Heapdump enabled. Send a request to "/heapdump" to download a heapdump,\nor "/heapdump/save" to only generate a heapdump.`); \ No newline at end of file diff --git a/utilities/servers/peerjs-server.js b/utilities/servers/peerjs-server.js deleted file mode 100644 index 434e5f4bc..000000000 --- a/utilities/servers/peerjs-server.js +++ /dev/null @@ -1,76 +0,0 @@ -var express = require('express'); -var peerRouter = express.Router(); - -let PROJECT_KEY_LENGTH = parseInt(process.env.PROJECT_KEY_LENGTH) || 20; -let debug = process.env.debug === "1" || false; -const extractPeerId = (peerId) => { - let splited = peerId.split("-"); - if (splited.length !== 2) { - debug && console.error(`cannot split peerId: ${peerId}`); - return {}; - } - if (PROJECT_KEY_LENGTH > 0 && splited[0].length !== PROJECT_KEY_LENGTH) { - debug && console.error(`wrong project key length for peerId: ${peerId}`); - return {}; - } - return {projectKey: splited[0], sessionId: splited[1]}; -}; -const connectedPeers = {}; - -const peerConnection = (client) => { - debug && console.log(`initiating ${client.id}`); - const {projectKey, sessionId} = extractPeerId(client.id); - if (projectKey === undefined || sessionId === undefined) { - return; - } - connectedPeers[projectKey] = connectedPeers[projectKey] || []; - if (connectedPeers[projectKey].indexOf(sessionId) === -1) { - debug && console.log(`new connexion ${client.id}`); - connectedPeers[projectKey].push(sessionId); - } else { - debug && console.log(`reconnecting peer ${client.id}`); - } - - -}; -const peerDisconnect = (client) => { - debug && console.log(`disconnect ${client.id}`); - const {projectKey, sessionId} = extractPeerId(client.id); - if (projectKey === undefined || sessionId === undefined) { - return; - } - const i = (connectedPeers[projectKey] || []).indexOf(sessionId); - if (i === -1) { - debug && console.log(`session not found ${client.id}`); - } else { - connectedPeers[projectKey].splice(i, 1); - } -} - -const peerError = (error) => { - console.error('error fired'); - console.error(error); -} - - -peerRouter.get(`/${process.env.S3_KEY}/peers`, function (req, res) { - debug && console.log("looking for all available sessions"); - res.statusCode = 200; - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({"data": connectedPeers})); -}); -peerRouter.get(`/${process.env.S3_KEY}/peers/:projectKey`, function (req, res) { - debug && console.log(`looking for available sessions for ${req.params.projectKey}`); - res.statusCode = 200; - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({"data": connectedPeers[req.params.projectKey] || []})); -}); - - -module.exports = { - peerRouter, - peerConnection, - peerDisconnect, - peerError, - extractPeerId -}; \ No newline at end of file diff --git a/utilities/servers/sourcemaps-handler.js b/utilities/servers/sourcemaps-handler.js deleted file mode 100644 index 91917104d..000000000 --- a/utilities/servers/sourcemaps-handler.js +++ /dev/null @@ -1,104 +0,0 @@ -'use strict'; -const sourceMap = require('source-map'); -const AWS = require('aws-sdk'); -const sourceMapVersion = require('../package.json').dependencies["source-map"]; -const URL = require('url'); -const getVersion = version => version.replace(/[\^\$\=\~]/, ""); - -module.exports.sourcemapReader = async event => { - sourceMap.SourceMapConsumer.initialize({ - "lib/mappings.wasm": `https://unpkg.com/source-map@${getVersion(sourceMapVersion)}/lib/mappings.wasm` - }); - let s3; - if (event.S3_HOST) { - s3 = new AWS.S3({ - endpoint: event.S3_HOST, - accessKeyId: event.S3_KEY, - secretAccessKey: event.S3_SECRET, - region: event.region, - s3ForcePathStyle: true, // needed with minio? - signatureVersion: 'v4' - }); - } else if (process.env.S3_HOST) { - s3 = new AWS.S3({ - endpoint: process.env.S3_HOST, - accessKeyId: process.env.S3_KEY, - secretAccessKey: process.env.S3_SECRET, - s3ForcePathStyle: true, // needed with minio? - signatureVersion: 'v4' - }); - } else { - s3 = new AWS.S3({ - 'AccessKeyID': process.env.aws_access_key_id, - 'SecretAccessKey': process.env.aws_secret_access_key, - 'Region': process.env.aws_region - }); - } - - var options = { - Bucket: event.bucket, - Key: event.key - }; - return new Promise(function (resolve, reject) { - s3.getObject(options, (err, data) => { - if (err) { - console.log("Get S3 object failed"); - console.log(err); - return reject(err); - } - let sourcemap = data.Body.toString(); - - return new sourceMap.SourceMapConsumer(sourcemap) - .then(consumer => { - let results = []; - for (let i = 0; i < event.positions.length; i++) { - let original = consumer.originalPositionFor({ - line: event.positions[i].line, - column: event.positions[i].column - }); - let url = URL.parse(""); - let preview = []; - if (original.source) { - preview = consumer.sourceContentFor(original.source, true); - if (preview !== null) { - preview = preview.split("\n") - .map((line, i) => [i + 1, line]); - if (event.padding) { - let start = original.line < event.padding ? 0 : original.line - event.padding; - preview = preview.slice(start, original.line + event.padding); - } - } else { - console.log("source not found, null preview for:"); - console.log(original.source); - preview = [] - } - url = URL.parse(original.source); - } else { - console.log("couldn't find original position of:"); - console.log({ - line: event.positions[i].line, - column: event.positions[i].column - }); - } - let result = { - "absPath": url.href, - "filename": url.pathname, - "lineNo": original.line, - "colNo": original.column, - "function": original.name, - "context": preview - }; - // console.log(result); - results.push(result); - } - consumer = undefined; - // Use this code if you don't use the http event with the LAMBDA-PROXY integration - return resolve(results); - }) - .finally(() => { - sourcemap = undefined; - }) - - }); - }); -}; \ No newline at end of file diff --git a/utilities/servers/sourcemaps-server.js b/utilities/servers/sourcemaps-server.js deleted file mode 100644 index a1dd501a0..000000000 --- a/utilities/servers/sourcemaps-server.js +++ /dev/null @@ -1,30 +0,0 @@ -var express = require('express'); -var handler = require('./sourcemaps-handler'); -var router = express.Router(); - -router.post('/', (req, res) => { - let data = ''; - req.on('data', chunk => { - data += chunk; - }); - req.on('end', function () { - data = JSON.parse(data); - console.log("Starting parser for: " + data.key); - // process.env = {...process.env, ...data.bucket_config}; - handler.sourcemapReader(data) - .then((results) => { - res.statusCode = 200; - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify(results)); - }) - .catch((e) => { - console.error("Something went wrong"); - console.error(e); - res.statusCode(500); - res.end(e); - }); - }) - -}); - -module.exports = router; \ No newline at end of file diff --git a/utilities/servers/websocket.js b/utilities/servers/websocket.js index 5f7000322..2c4d22342 100644 --- a/utilities/servers/websocket.js +++ b/utilities/servers/websocket.js @@ -1,8 +1,8 @@ const _io = require('socket.io'); const express = require('express'); const uaParser = require('ua-parser-js'); -const geoip2Reader = require('@maxmind/geoip2-node').Reader; -const {extractPeerId} = require('./peerjs-server'); +const {extractPeerId} = require('../utils/helper'); +const {geoip} = require('../utils/geoIP'); const wsRouter = express.Router(); const UPDATE_EVENT = "UPDATE_SESSION"; const IDENTITIES = {agent: 'agent', session: 'session'}; @@ -79,7 +79,7 @@ const socketsList = async function (req, res) { } respond(res, liveSessions); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-list`, socketsList); +wsRouter.get(`/sockets-list`, socketsList); const socketsListByProject = async function (req, res) { debug && console.log("[WS]looking for available sessions"); @@ -105,7 +105,7 @@ const socketsListByProject = async function (req, res) { } respond(res, liveSessions[_projectKey] || []); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-list/:projectKey`, socketsListByProject); +wsRouter.get(`/sockets-list/:projectKey`, socketsListByProject); const socketsLive = async function (req, res) { debug && console.log("[WS]looking for all available LIVE sessions"); @@ -113,7 +113,7 @@ const socketsLive = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey !== undefined) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -132,7 +132,7 @@ const socketsLive = async function (req, res) { } respond(res, liveSessions); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-live`, socketsLive); +wsRouter.get(`/sockets-live`, socketsLive); const socketsLiveByProject = async function (req, res) { debug && console.log("[WS]looking for available LIVE sessions"); @@ -141,7 +141,7 @@ const socketsLiveByProject = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey === _projectKey) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -160,7 +160,7 @@ const socketsLiveByProject = async function (req, res) { } respond(res, liveSessions[_projectKey] || []); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-live/:projectKey`, socketsLiveByProject); +wsRouter.get(`/sockets-live/:projectKey`, socketsLiveByProject); const findSessionSocketId = async (io, peerId) => { const connected_sockets = await io.in(peerId).fetchSockets(); @@ -204,15 +204,6 @@ async function get_all_agents_ids(io, socket) { return agents; } -let geoip = null; -geoip2Reader.open(process.env.MAXMINDDB_FILE, {}) - .then(reader => { - geoip = reader; - }) - .catch(error => { - console.log("Error while opening the MAXMINDDB_FILE.") - console.error(error); - }); function extractSessionInfo(socket) { if (socket.handshake.query.sessionInfo !== undefined) { diff --git a/utilities/utils/HeapSnapshot.js b/utilities/utils/HeapSnapshot.js index dc0b45e9a..4d819be45 100644 --- a/utilities/utils/HeapSnapshot.js +++ b/utilities/utils/HeapSnapshot.js @@ -31,6 +31,9 @@ async function downloadHeapSnapshot(req, res) { return res.end("should wait for done status"); } res.download(location + fileName, function (err) { + if (err) { + return console.error("error while uploading HeapSnapshot file"); + } try { fs.unlinkSync(location + fileName) } catch (err) { @@ -57,7 +60,7 @@ function createNewHeapSnapshot(req, res) { res.end(JSON.stringify({path: location + fileName, 'done': creationStatus})); } -router.get(`/${process.env.S3_KEY}/status`, getHeapSnapshotStatus); -router.get(`/${process.env.S3_KEY}/new`, createNewHeapSnapshot); -router.get(`/${process.env.S3_KEY}/download`, downloadHeapSnapshot); +router.get(`/status`, getHeapSnapshotStatus); +router.get(`/new`, createNewHeapSnapshot); +router.get(`/download`, downloadHeapSnapshot); module.exports = {router} \ No newline at end of file diff --git a/utilities/utils/geoIP.js b/utilities/utils/geoIP.js new file mode 100644 index 000000000..5bfcfd0e8 --- /dev/null +++ b/utilities/utils/geoIP.js @@ -0,0 +1,16 @@ +const geoip2Reader = require('@maxmind/geoip2-node').Reader; +let geoip = null; +if (process.env.MAXMINDDB_FILE !== undefined) { + geoip2Reader.open(process.env.MAXMINDDB_FILE, {}) + .then(reader => { + geoip = reader; + }) + .catch(error => { + console.log("Error while opening the MAXMINDDB_FILE.") + console.error(error); + }); +} else { + console.error("!!! please provide a valid value for MAXMINDDB_FILE env var."); +} + +module.exports = {geoip} \ No newline at end of file diff --git a/utilities/utils/helper.js b/utilities/utils/helper.js new file mode 100644 index 000000000..98322c417 --- /dev/null +++ b/utilities/utils/helper.js @@ -0,0 +1,30 @@ +let PROJECT_KEY_LENGTH = parseInt(process.env.PROJECT_KEY_LENGTH) || 20; +let debug = process.env.debug === "1" || false; +const extractPeerId = (peerId) => { + let splited = peerId.split("-"); + if (splited.length !== 2) { + debug && console.error(`cannot split peerId: ${peerId}`); + return {}; + } + if (PROJECT_KEY_LENGTH > 0 && splited[0].length !== PROJECT_KEY_LENGTH) { + debug && console.error(`wrong project key length for peerId: ${peerId}`); + return {}; + } + return {projectKey: splited[0], sessionId: splited[1]}; +}; +const request_logger = (identity) => { + return (req, res, next) => { + debug && console.log(identity, new Date().toTimeString(), 'REQUEST', req.method, req.originalUrl); + res.on('finish', function () { + if (this.statusCode !== 200 || debug) { + console.log(new Date().toTimeString(), 'RESPONSE', req.method, req.originalUrl, this.statusCode); + } + }) + + next(); + } +}; + +module.exports = { + extractPeerId, request_logger +}; \ No newline at end of file From 032f58738b12c80781ce8468466df752de8210ef Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 18:28:24 +0200 Subject: [PATCH 100/125] feat(utilities): utilities EE-WS as a standalone service --- ee/utilities/.gitignore | 3 +- ee/utilities/package-lock.json | 891 ---------------------- ee/utilities/package.json | 3 - ee/utilities/server.js | 72 +- ee/utilities/servers/websocket-cluster.js | 19 +- ee/utilities/servers/websocket.js | 25 +- utilities/server.js | 3 +- utilities/utils/HeapSnapshot.js | 4 +- 8 files changed, 37 insertions(+), 983 deletions(-) diff --git a/ee/utilities/.gitignore b/ee/utilities/.gitignore index 2dd72c808..0aaf625c9 100644 --- a/ee/utilities/.gitignore +++ b/ee/utilities/.gitignore @@ -11,4 +11,5 @@ servers/peerjs-server.js servers/sourcemaps-handler.js servers/sourcemaps-server.js #servers/websocket.js -/utils/dump.js +/utils +/Dockerfile diff --git a/ee/utilities/package-lock.json b/ee/utilities/package-lock.json index e7974f3f2..66fdb346f 100644 --- a/ee/utilities/package-lock.json +++ b/ee/utilities/package-lock.json @@ -11,12 +11,9 @@ "dependencies": { "@maxmind/geoip2-node": "^3.4.0", "@socket.io/redis-adapter": "^7.1.0", - "aws-sdk": "^2.992.0", "express": "^4.17.1", - "peer": "^0.6.1", "redis": "^4.0.3", "socket.io": "^4.4.1", - "source-map": "^0.7.3", "ua-parser-js": "^1.0.2", "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.6.0" } @@ -108,28 +105,11 @@ "node": ">=10.0.0" } }, - "node_modules/@types/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, "node_modules/@types/component-emitter": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.11.tgz", "integrity": "sha512-SRXjM+tfsSlA9VuG8hGO2nft2p8zjXCK1VcC6N4NXbBbYbSia9kzCChYQajIjzIqOOOuh5Ock6MmV2oux4jDZQ==" }, - "node_modules/@types/connect": { - "version": "3.4.35", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", - "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/cookie": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz", @@ -140,64 +120,11 @@ "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" }, - "node_modules/@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.18", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.17.28", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", - "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "node_modules/@types/mime": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", - "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" - }, "node_modules/@types/node": { "version": "17.0.21", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" }, - "node_modules/@types/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" - }, - "node_modules/@types/range-parser": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "node_modules/@types/serve-static": { - "version": "1.13.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", - "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", - "dependencies": { - "@types/mime": "^1", - "@types/node": "*" - } - }, - "node_modules/@types/ws": { - "version": "7.4.7", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", - "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -210,28 +137,6 @@ "node": ">= 0.6" } }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -245,44 +150,6 @@ "node": ">=0.8" } }, - "node_modules/aws-sdk": { - "version": "2.1087.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", - "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", - "dependencies": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "uuid": "3.3.2", - "xml2js": "0.4.19" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/base64id": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz", @@ -324,16 +191,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, - "node_modules/buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "dependencies": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -370,16 +227,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, "node_modules/cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", @@ -388,22 +235,6 @@ "node": ">=0.10.0" } }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, "node_modules/component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", @@ -474,14 +305,6 @@ } } }, - "node_modules/decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -500,11 +323,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -577,14 +395,6 @@ "node": ">= 0.6" } }, - "node_modules/events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/express": { "version": "4.17.3", "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", @@ -676,18 +486,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, - "node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -712,14 +510,6 @@ "node": ">= 4" } }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, "node_modules/http-errors": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", @@ -746,11 +536,6 @@ "node": ">=0.10.0" } }, - "node_modules/ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -773,27 +558,6 @@ "node": ">= 0.10" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "node_modules/jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", - "engines": { - "node": ">= 0.6.0" - } - }, "node_modules/json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", @@ -813,17 +577,6 @@ "verror": "1.10.0" } }, - "node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", @@ -950,39 +703,6 @@ "node": ">= 0.8" } }, - "node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "engines": { - "node": ">=6" - } - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -991,50 +711,11 @@ "node": ">= 0.8" } }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "engines": { - "node": ">=8" - } - }, "node_modules/path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "node_modules/peer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", - "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", - "dependencies": { - "@types/cors": "^2.8.6", - "@types/express": "^4.17.3", - "@types/ws": "^7.2.3", - "body-parser": "^1.19.0", - "cors": "^2.8.5", - "express": "^4.17.1", - "uuid": "^3.4.0", - "ws": "^7.2.3", - "yargs": "^15.3.1" - }, - "bin": { - "peerjs": "bin/peerjs" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/peer/node_modules/uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -1047,11 +728,6 @@ "node": ">= 0.10" } }, - "node_modules/punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" - }, "node_modules/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", @@ -1063,15 +739,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", - "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -1137,19 +804,6 @@ "node": ">=4" } }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1174,11 +828,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "node_modules/sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" - }, "node_modules/send": { "version": "0.17.2", "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", @@ -1234,11 +883,6 @@ "node": ">= 0.8.0" } }, - "node_modules/set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -1278,14 +922,6 @@ "node": ">=10.0.0" } }, - "node_modules/source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", - "engines": { - "node": ">= 8" - } - }, "node_modules/statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -1294,30 +930,6 @@ "node": ">= 0.6" } }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/tiny-lru": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-7.0.6.tgz", @@ -1388,15 +1000,6 @@ "node": ">= 0.8" } }, - "node_modules/url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", - "dependencies": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -1405,15 +1008,6 @@ "node": ">= 0.4.0" } }, - "node_modules/uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, "node_modules/uWebSockets.js": { "version": "20.6.0", "resolved": "git+ssh://git@github.com/uNetworking/uWebSockets.js.git#a58e810e47a23696410f6073c8c905dc38f75da5" @@ -1439,111 +1033,10 @@ "extsprintf": "^1.2.0" } }, - "node_modules/which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" - }, - "node_modules/wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/ws": { - "version": "7.5.7", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", - "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", - "engines": { - "node": ">=8.3.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/xml2js": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~9.0.1" - } - }, - "node_modules/xmlbuilder": { - "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/y18n": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" - }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "node_modules/yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", - "dependencies": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "dependencies": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/yargs-parser/node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "engines": { - "node": ">=6" - } } }, "dependencies": { @@ -1615,28 +1108,11 @@ "uid2": "0.0.3" } }, - "@types/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", - "requires": { - "@types/connect": "*", - "@types/node": "*" - } - }, "@types/component-emitter": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.11.tgz", "integrity": "sha512-SRXjM+tfsSlA9VuG8hGO2nft2p8zjXCK1VcC6N4NXbBbYbSia9kzCChYQajIjzIqOOOuh5Ock6MmV2oux4jDZQ==" }, - "@types/connect": { - "version": "3.4.35", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", - "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", - "requires": { - "@types/node": "*" - } - }, "@types/cookie": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz", @@ -1647,64 +1123,11 @@ "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" }, - "@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", - "requires": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.18", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "@types/express-serve-static-core": { - "version": "4.17.28", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", - "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", - "requires": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "@types/mime": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", - "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" - }, "@types/node": { "version": "17.0.21", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" }, - "@types/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" - }, - "@types/range-parser": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "@types/serve-static": { - "version": "1.13.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", - "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", - "requires": { - "@types/mime": "^1", - "@types/node": "*" - } - }, - "@types/ws": { - "version": "7.4.7", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", - "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", - "requires": { - "@types/node": "*" - } - }, "accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -1714,19 +1137,6 @@ "negotiator": "0.6.3" } }, - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -1737,27 +1147,6 @@ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, - "aws-sdk": { - "version": "2.1087.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", - "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", - "requires": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "uuid": "3.3.2", - "xml2js": "0.4.19" - } - }, - "base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" - }, "base64id": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz", @@ -1795,16 +1184,6 @@ } } }, - "buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, "bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -1826,34 +1205,11 @@ "type-fest": "^1.2.1" } }, - "cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, "cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, "component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", @@ -1904,11 +1260,6 @@ "ms": "2.1.2" } }, - "decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" - }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -1924,11 +1275,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -1977,11 +1323,6 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, - "events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" - }, "express": { "version": "4.17.3", "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", @@ -2068,15 +1409,6 @@ } } }, - "find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -2092,11 +1424,6 @@ "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", "integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==" }, - "get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" - }, "http-errors": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", @@ -2117,11 +1444,6 @@ "safer-buffer": ">= 2.1.2 < 3" } }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -2141,21 +1463,6 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==" - }, "json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", @@ -2172,14 +1479,6 @@ "verror": "1.10.0" } }, - "locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "requires": { - "p-locate": "^4.1.0" - } - }, "lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", @@ -2265,65 +1564,16 @@ "ee-first": "1.1.1" } }, - "p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "requires": { - "p-limit": "^2.2.0" - } - }, - "p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" - }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, - "path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" - }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "peer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", - "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", - "requires": { - "@types/cors": "^2.8.6", - "@types/express": "^4.17.3", - "@types/ws": "^7.2.3", - "body-parser": "^1.19.0", - "cors": "^2.8.5", - "express": "^4.17.1", - "uuid": "^3.4.0", - "ws": "^7.2.3", - "yargs": "^15.3.1" - }, - "dependencies": { - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - } - } - }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -2333,21 +1583,11 @@ "ipaddr.js": "1.9.1" } }, - "punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" - }, "qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==" }, - "querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" - }, "quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -2395,16 +1635,6 @@ "redis-errors": "^1.0.0" } }, - "require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" - }, - "require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2415,11 +1645,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" - }, "send": { "version": "0.17.2", "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", @@ -2473,11 +1698,6 @@ "send": "0.17.2" } }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, "setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -2511,34 +1731,11 @@ "debug": "~4.3.1" } }, - "source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==" - }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - }, "tiny-lru": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-7.0.6.tgz", @@ -2578,25 +1775,11 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, - "url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", - "requires": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" - }, "uWebSockets.js": { "version": "git+ssh://git@github.com/uNetworking/uWebSockets.js.git#a58e810e47a23696410f6073c8c905dc38f75da5", "from": "uWebSockets.js@github:uNetworking/uWebSockets.js#v20.6.0" @@ -2616,84 +1799,10 @@ "extsprintf": "^1.2.0" } }, - "which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" - }, - "wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, - "ws": { - "version": "7.5.7", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", - "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", - "requires": {} - }, - "xml2js": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", - "requires": { - "sax": ">=0.6.0", - "xmlbuilder": "~9.0.1" - } - }, - "xmlbuilder": { - "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" - }, - "y18n": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" - }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", - "requires": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - } - }, - "yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "dependencies": { - "camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" - } - } } } } diff --git a/ee/utilities/package.json b/ee/utilities/package.json index 87e1d1596..ec895470b 100644 --- a/ee/utilities/package.json +++ b/ee/utilities/package.json @@ -20,12 +20,9 @@ "dependencies": { "@maxmind/geoip2-node": "^3.4.0", "@socket.io/redis-adapter": "^7.1.0", - "aws-sdk": "^2.992.0", "express": "^4.17.1", - "peer": "^0.6.1", "redis": "^4.0.3", "socket.io": "^4.4.1", - "source-map": "^0.7.3", "ua-parser-js": "^1.0.2", "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.6.0" } diff --git a/ee/utilities/server.js b/ee/utilities/server.js index 798f4e715..2c70067f1 100644 --- a/ee/utilities/server.js +++ b/ee/utilities/server.js @@ -1,8 +1,6 @@ const dumps = require('./utils/HeapSnapshot'); -const sourcemapsReaderServer = require('./servers/sourcemaps-server'); -const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); +const {request_logger} = require('./utils/helper'); const express = require('express'); -const {ExpressPeerServer} = require('peer'); let socket; if (process.env.redis === "true") { console.log("Using Redis"); @@ -12,63 +10,29 @@ if (process.env.redis === "true") { } const HOST = '0.0.0.0'; -const PORT = 9000; - -const app = express(); +const PORT = 9001; let debug = process.env.debug === "1" || false; -const request_logger = (identity) => { - return (req, res, next) => { - debug && console.log(identity, new Date().toTimeString(), 'REQUEST', req.method, req.originalUrl); - res.on('finish', function () { - if (this.statusCode !== 200 || debug) { - console.log(new Date().toTimeString(), 'RESPONSE', req.method, req.originalUrl, this.statusCode); - } - }) - - next(); - } -}; -app.use(request_logger("[app]")); - - -app.use('/sourcemaps', sourcemapsReaderServer); -app.use('/assist', peerRouter); - -const server = app.listen(PORT, HOST, () => { - console.log(`App listening on http://${HOST}:${PORT}`); - console.log('Press Ctrl+C to quit.'); -}); - -const peerServer = ExpressPeerServer(server, { - debug: true, - path: '/', - proxied: true, - allow_discovery: false -}); -peerServer.on('connection', peerConnection); -peerServer.on('disconnect', peerDisconnect); -peerServer.on('error', peerError); -app.use('/', peerServer); -app.enable('trust proxy'); -app.use('/heapdump', dumps.router); +const PREFIX = process.env.prefix || `/assist` if (process.env.uws !== "true") { - var wsapp = express(); + let wsapp = express(); wsapp.use(request_logger("[wsapp]")); - wsapp.use('/assist', socket.wsRouter); - - const wsserver = wsapp.listen(PORT + 1, HOST, () => { - console.log(`WS App listening on http://${HOST}:${PORT + 1}`); + wsapp.use(request_logger("[app]")); + wsapp.use(`/heapdump/${process.env.S3_KEY}`, dumps.router); + wsapp.use(`${PREFIX}/${process.env.S3_KEY}`, socket.wsRouter); + wsapp.enable('trust proxy'); + const wsserver = wsapp.listen(PORT, HOST, () => { + console.log(`WS App listening on http://${HOST}:${PORT}`); console.log('Press Ctrl+C to quit.'); }); - wsapp.enable('trust proxy'); + socket.start(wsserver); - module.exports = {wsserver, server}; + module.exports = {wsserver}; } else { console.log("Using uWebSocket"); const {App} = require("uWebSockets.js"); - const PREFIX = process.env.prefix || '/assist' + const uapp = new App(); @@ -77,6 +41,7 @@ if (process.env.uws !== "true") { } uapp.get(PREFIX, healthFn); uapp.get(`${PREFIX}/`, healthFn); + uapp.get(`${PREFIX}/${process.env.S3_KEY}`, healthFn); /* Either onAborted or simply finished request */ @@ -112,11 +77,11 @@ if (process.env.uws !== "true") { socket.start(uapp); - uapp.listen(HOST, PORT + 1, (token) => { + uapp.listen(HOST, PORT, (token) => { if (!token) { console.warn("port already in use"); } - console.log(`WS App listening on http://${HOST}:${PORT + 1}`); + console.log(`WS App listening on http://${HOST}:${PORT}`); console.log('Press Ctrl+C to quit.'); }); @@ -126,6 +91,5 @@ if (process.env.uws !== "true") { debug && console.log(err.stack); // process.exit(1); }); - module.exports = {uapp, server}; -} -console.log(`Heapdump enabled. Send a request to "/heapdump" to download a heapdump,\nor "/heapdump/save" to only generate a heapdump.`); \ No newline at end of file + module.exports = {uapp}; +} \ No newline at end of file diff --git a/ee/utilities/servers/websocket-cluster.js b/ee/utilities/servers/websocket-cluster.js index e2e6aaa86..f21755f9b 100644 --- a/ee/utilities/servers/websocket-cluster.js +++ b/ee/utilities/servers/websocket-cluster.js @@ -1,8 +1,8 @@ const _io = require('socket.io'); const express = require('express'); const uaParser = require('ua-parser-js'); -const geoip2Reader = require('@maxmind/geoip2-node').Reader; -const {extractPeerId} = require('./peerjs-server'); +const {extractPeerId} = require('../utils/helper'); +const {geoip} = require('../utils/geoIP'); const {createAdapter} = require("@socket.io/redis-adapter"); const {createClient} = require("redis"); const wsRouter = express.Router(); @@ -159,7 +159,7 @@ const socketsLive = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey !== undefined) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -188,7 +188,7 @@ const socketsLiveByProject = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey === _projectKey) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -254,15 +254,6 @@ async function get_all_agents_ids(io, socket) { return agents; } -let geoip = null; -geoip2Reader.open(process.env.MAXMINDDB_FILE, {}) - .then(reader => { - geoip = reader; - }) - .catch(error => { - console.log("Error while opening the MAXMINDDB_FILE.") - console.error(error); - }); function extractSessionInfo(socket) { if (socket.handshake.query.sessionInfo !== undefined) { @@ -379,7 +370,7 @@ module.exports = { let rooms = await io.of('/').adapter.allRooms(); let validRooms = []; console.log(` ====== Rooms: ${rooms.size} ====== `); - const arr = Array.from(rooms) + // const arr = Array.from(rooms) // const filtered = arr.filter(room => !room[1].has(room[0])) for (let i of rooms) { let {projectKey, sessionId} = extractPeerId(i); diff --git a/ee/utilities/servers/websocket.js b/ee/utilities/servers/websocket.js index 521655b34..a5126a78f 100644 --- a/ee/utilities/servers/websocket.js +++ b/ee/utilities/servers/websocket.js @@ -1,8 +1,8 @@ const _io = require('socket.io'); const express = require('express'); const uaParser = require('ua-parser-js'); -const geoip2Reader = require('@maxmind/geoip2-node').Reader; -const {extractPeerId} = require('./peerjs-server'); +const {extractPeerId} = require('../utils/helper'); +const {geoip} = require('../utils/geoIP'); const wsRouter = express.Router(); const UPDATE_EVENT = "UPDATE_SESSION"; const IDENTITIES = {agent: 'agent', session: 'session'}; @@ -108,7 +108,7 @@ const socketsList = async function (req, res) { } respond(res, liveSessions); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-list`, socketsList); +wsRouter.get(`/sockets-list`, socketsList); const socketsListByProject = async function (req, res) { debug && console.log("[WS]looking for available sessions"); @@ -134,7 +134,7 @@ const socketsListByProject = async function (req, res) { } respond(res, liveSessions[_projectKey] || []); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-list/:projectKey`, socketsListByProject); +wsRouter.get(`/sockets-list/:projectKey`, socketsListByProject); const socketsLive = async function (req, res) { debug && console.log("[WS]looking for all available LIVE sessions"); @@ -142,7 +142,7 @@ const socketsLive = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey !== undefined) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -161,7 +161,7 @@ const socketsLive = async function (req, res) { } respond(res, liveSessions); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-live`, socketsLive); +wsRouter.get(`/sockets-live`, socketsLive); const socketsLiveByProject = async function (req, res) { debug && console.log("[WS]looking for available LIVE sessions"); @@ -170,7 +170,7 @@ const socketsLiveByProject = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey === _projectKey) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -189,7 +189,7 @@ const socketsLiveByProject = async function (req, res) { } respond(res, liveSessions[_projectKey] || []); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-live/:projectKey`, socketsLiveByProject); +wsRouter.get(`/sockets-live/:projectKey`, socketsLiveByProject); const findSessionSocketId = async (io, peerId) => { const connected_sockets = await io.in(peerId).fetchSockets(); @@ -233,15 +233,6 @@ async function get_all_agents_ids(io, socket) { return agents; } -let geoip = null; -geoip2Reader.open(process.env.MAXMINDDB_FILE, {}) - .then(reader => { - geoip = reader; - }) - .catch(error => { - console.log("Error while opening the MAXMINDDB_FILE.") - console.error(error); - }); function extractSessionInfo(socket) { if (socket.handshake.query.sessionInfo !== undefined) { diff --git a/utilities/server.js b/utilities/server.js index 9fd8530fd..b0eadcccd 100644 --- a/utilities/server.js +++ b/utilities/server.js @@ -18,5 +18,4 @@ const wsserver = wsapp.listen(PORT, HOST, () => { }); wsapp.enable('trust proxy'); socket.start(wsserver); -module.exports = {wsserver}; -console.log(`Heapdump enabled. Send a request to "/heapdump" to download a heapdump,\nor "/heapdump/save" to only generate a heapdump.`); \ No newline at end of file +module.exports = {wsserver}; \ No newline at end of file diff --git a/utilities/utils/HeapSnapshot.js b/utilities/utils/HeapSnapshot.js index 4d819be45..818589fcc 100644 --- a/utilities/utils/HeapSnapshot.js +++ b/utilities/utils/HeapSnapshot.js @@ -63,4 +63,6 @@ function createNewHeapSnapshot(req, res) { router.get(`/status`, getHeapSnapshotStatus); router.get(`/new`, createNewHeapSnapshot); router.get(`/download`, downloadHeapSnapshot); -module.exports = {router} \ No newline at end of file +module.exports = {router} + +console.log(`HeapSnapshot enabled. Send a request to "/heapdump/new" to generate a heapdump.`); \ No newline at end of file From d4dbb7c8bc7dde12933c0add27619bd0bb0ccbf3 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 18:55:49 +0200 Subject: [PATCH 101/125] feat(peers): peerjs server as a standalone service --- peers/.gitignore | 6 + peers/Dockerfile | 14 + peers/build.sh | 33 + peers/package-lock.json | 1584 ++++++++++++++++++++++++++++++++ peers/package.json | 24 + peers/server.js | 33 + peers/servers/peerjs-server.js | 65 ++ 7 files changed, 1759 insertions(+) create mode 100644 peers/.gitignore create mode 100644 peers/Dockerfile create mode 100644 peers/build.sh create mode 100644 peers/package-lock.json create mode 100644 peers/package.json create mode 100644 peers/server.js create mode 100644 peers/servers/peerjs-server.js diff --git a/peers/.gitignore b/peers/.gitignore new file mode 100644 index 000000000..a4b05b411 --- /dev/null +++ b/peers/.gitignore @@ -0,0 +1,6 @@ +.idea +node_modules +npm-debug.log +.cache +test.html +/utils/ diff --git a/peers/Dockerfile b/peers/Dockerfile new file mode 100644 index 000000000..d053bf3af --- /dev/null +++ b/peers/Dockerfile @@ -0,0 +1,14 @@ +FROM node:17-stretch +WORKDIR /work +COPY . . +RUN npm install + +# Add Tini +# Startup daemon +ENV TINI_VERSION v0.19.0 +ARG envarg +ENV ENTERPRISE_BUILD ${envarg} +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] +CMD npm start \ No newline at end of file diff --git a/peers/build.sh b/peers/build.sh new file mode 100644 index 000000000..c15921ea8 --- /dev/null +++ b/peers/build.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Script to build api module +# flags to accept: +# Default will be OSS build. + +# Usage: IMAGE_TAG=latest DOCKER_REPO=myDockerHubID bash build.sh + +git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)} +check_prereq() { + which docker || { + echo "Docker not installed, please install docker." + exit=1 + } + [[ exit -eq 1 ]] && exit 1 +} + +function build_api(){ + cp -R ../utilities/utils . + # Copy enterprise code + [[ $1 == "ee" ]] && { + cp -rf ../ee/peers/* ./ + } + docker build -f ./Dockerfile -t ${DOCKER_REPO:-'local'}/peers:${git_sha1} . + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/peers:${git_sha1} + docker tag ${DOCKER_REPO:-'local'}/peers:${git_sha1} ${DOCKER_REPO:-'local'}/peers:latest + docker push ${DOCKER_REPO:-'local'}/peers:latest + } +} + +check_prereq +build_api $1 diff --git a/peers/package-lock.json b/peers/package-lock.json new file mode 100644 index 000000000..5f253d3bc --- /dev/null +++ b/peers/package-lock.json @@ -0,0 +1,1584 @@ +{ + "name": "utilities_server", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "utilities_server", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "express": "^4.17.1", + "peer": "^0.6.1" + } + }, + "node_modules/@types/body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.35", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", + "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/cors": { + "version": "2.8.12", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", + "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" + }, + "node_modules/@types/express": { + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", + "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.18", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.17.28", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", + "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*" + } + }, + "node_modules/@types/mime": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", + "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" + }, + "node_modules/@types/node": { + "version": "17.0.21", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", + "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" + }, + "node_modules/@types/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" + }, + "node_modules/@types/range-parser": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" + }, + "node_modules/@types/serve-static": { + "version": "1.13.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", + "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/ws": { + "version": "7.4.7", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", + "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "node_modules/body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.9.7", + "raw-body": "2.4.3", + "type-is": "~1.6.18" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cliui": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", + "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^6.2.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", + "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.19.2", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.4.2", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.9.7", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", + "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.34", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", + "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", + "dependencies": { + "mime-db": "1.51.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "node_modules/peer": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", + "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", + "dependencies": { + "@types/cors": "^2.8.6", + "@types/express": "^4.17.3", + "@types/ws": "^7.2.3", + "body-parser": "^1.19.0", + "cors": "^2.8.5", + "express": "^4.17.1", + "uuid": "^3.4.0", + "ws": "^7.2.3", + "yargs": "^15.3.1" + }, + "bin": { + "peerjs": "bin/peerjs" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/peer/node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==", + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", + "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "dependencies": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "1.8.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/serve-static": { + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" + }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ws": { + "version": "7.5.7", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", + "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" + }, + "node_modules/yargs": { + "version": "15.4.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", + "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "dependencies": { + "cliui": "^6.0.0", + "decamelize": "^1.2.0", + "find-up": "^4.1.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^4.2.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^18.1.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs-parser/node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "engines": { + "node": ">=6" + } + } + }, + "dependencies": { + "@types/body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", + "requires": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "@types/connect": { + "version": "3.4.35", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", + "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", + "requires": { + "@types/node": "*" + } + }, + "@types/cors": { + "version": "2.8.12", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", + "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" + }, + "@types/express": { + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", + "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "requires": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.18", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "@types/express-serve-static-core": { + "version": "4.17.28", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", + "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", + "requires": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*" + } + }, + "@types/mime": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", + "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" + }, + "@types/node": { + "version": "17.0.21", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", + "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" + }, + "@types/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" + }, + "@types/range-parser": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" + }, + "@types/serve-static": { + "version": "1.13.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", + "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", + "requires": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "@types/ws": { + "version": "7.4.7", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", + "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", + "requires": { + "@types/node": "*" + } + }, + "accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "requires": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + } + }, + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", + "requires": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.9.7", + "raw-body": "2.4.3", + "type-is": "~1.6.18" + } + }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "cliui": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", + "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^6.2.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "requires": { + "safe-buffer": "5.2.1" + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "requires": { + "object-assign": "^4", + "vary": "^1" + } + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" + }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + }, + "express": { + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", + "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", + "requires": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.19.2", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.4.2", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.9.7", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + } + }, + "finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, + "http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "requires": { + "p-locate": "^4.1.0" + } + }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + }, + "mime-db": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", + "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==" + }, + "mime-types": { + "version": "2.1.34", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", + "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", + "requires": { + "mime-db": "1.51.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "requires": { + "p-limit": "^2.2.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "peer": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", + "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", + "requires": { + "@types/cors": "^2.8.6", + "@types/express": "^4.17.3", + "@types/ws": "^7.2.3", + "body-parser": "^1.19.0", + "cors": "^2.8.5", + "express": "^4.17.1", + "uuid": "^3.4.0", + "ws": "^7.2.3", + "yargs": "^15.3.1" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } + } + }, + "proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "requires": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + } + }, + "qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==" + }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" + }, + "raw-body": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", + "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", + "requires": { + "bytes": "3.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "1.8.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + }, + "serve-static": { + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.2" + } + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" + }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" + }, + "wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, + "ws": { + "version": "7.5.7", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", + "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", + "requires": {} + }, + "y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" + }, + "yargs": { + "version": "15.4.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", + "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "requires": { + "cliui": "^6.0.0", + "decamelize": "^1.2.0", + "find-up": "^4.1.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^4.2.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^18.1.2" + } + }, + "yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "dependencies": { + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" + } + } + } + } +} diff --git a/peers/package.json b/peers/package.json new file mode 100644 index 000000000..ba29b0899 --- /dev/null +++ b/peers/package.json @@ -0,0 +1,24 @@ +{ + "name": "utilities_server", + "version": "1.0.0", + "description": "assist server to get live sessions & sourcemaps reader to get stack trace", + "main": "peerjs-server.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node server.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/openreplay/openreplay.git" + }, + "author": "KRAIEM Taha Yassine ", + "license": "MIT", + "bugs": { + "url": "https://github.com/openreplay/openreplay/issues" + }, + "homepage": "https://github.com/openreplay/openreplay#readme", + "dependencies": { + "express": "^4.17.1", + "peer": "^0.6.1" + } +} diff --git a/peers/server.js b/peers/server.js new file mode 100644 index 000000000..39f46d4f1 --- /dev/null +++ b/peers/server.js @@ -0,0 +1,33 @@ +const dumps = require('./utils/HeapSnapshot'); +const {request_logger} = require('./utils/helper'); +const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); +const express = require('express'); +const {ExpressPeerServer} = require('peer'); + +const HOST = '0.0.0.0'; +const PORT = 9000; + +const app = express(); + +app.use(request_logger("[app]")); + +app.use(`/${process.env.S3_KEY}/assist`, peerRouter); +app.use(`/${process.env.S3_KEY}/heapdump`, dumps.router); + +const server = app.listen(PORT, HOST, () => { + console.log(`App listening on http://${HOST}:${PORT}`); + console.log('Press Ctrl+C to quit.'); +}); + +const peerServer = ExpressPeerServer(server, { + debug: true, + path: '/', + proxied: true, + allow_discovery: false +}); +peerServer.on('connection', peerConnection); +peerServer.on('disconnect', peerDisconnect); +peerServer.on('error', peerError); +app.use('/', peerServer); +app.enable('trust proxy'); +module.exports = {server}; \ No newline at end of file diff --git a/peers/servers/peerjs-server.js b/peers/servers/peerjs-server.js new file mode 100644 index 000000000..a99ec1665 --- /dev/null +++ b/peers/servers/peerjs-server.js @@ -0,0 +1,65 @@ +const express = require('express'); +const peerRouter = express.Router(); +const {extractPeerId} = require('../utils/helper'); + +let debug = process.env.debug === "1" || false; + +const connectedPeers = {}; + +const peerConnection = (client) => { + debug && console.log(`initiating ${client.id}`); + const {projectKey, sessionId} = extractPeerId(client.id); + if (projectKey === undefined || sessionId === undefined) { + return; + } + connectedPeers[projectKey] = connectedPeers[projectKey] || []; + if (connectedPeers[projectKey].indexOf(sessionId) === -1) { + debug && console.log(`new connexion ${client.id}`); + connectedPeers[projectKey].push(sessionId); + } else { + debug && console.log(`reconnecting peer ${client.id}`); + } + + +}; +const peerDisconnect = (client) => { + debug && console.log(`disconnect ${client.id}`); + const {projectKey, sessionId} = extractPeerId(client.id); + if (projectKey === undefined || sessionId === undefined) { + return; + } + const i = (connectedPeers[projectKey] || []).indexOf(sessionId); + if (i === -1) { + debug && console.log(`session not found ${client.id}`); + } else { + connectedPeers[projectKey].splice(i, 1); + } +} + +const peerError = (error) => { + console.error('error fired'); + console.error(error); +} + + +peerRouter.get(`/peers`, function (req, res) { + debug && console.log("looking for all available sessions"); + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({"data": connectedPeers})); +}); +peerRouter.get(`/peers/:projectKey`, function (req, res) { + debug && console.log(`looking for available sessions for ${req.params.projectKey}`); + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({"data": connectedPeers[req.params.projectKey] || []})); +}); + + +module.exports = { + peerRouter, + peerConnection, + peerDisconnect, + peerError, + extractPeerId +}; \ No newline at end of file From 364b9c40f26729ac8f38d5f940468b8e3103fb22 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 19:06:19 +0200 Subject: [PATCH 102/125] feat(sourcemaps-reader): extracted sourcemaps reader --- ee/utilities/package.json | 4 +- peers/package.json | 4 +- sourcemap-reader/.gitignore | 6 + sourcemap-reader/package-lock.json | 1113 +++++++++++++++++ sourcemap-reader/package.json | 25 + sourcemap-reader/server.js | 19 + .../servers/sourcemaps-handler.js | 104 ++ sourcemap-reader/servers/sourcemaps-server.js | 30 + utilities/package.json | 4 +- 9 files changed, 1303 insertions(+), 6 deletions(-) create mode 100644 sourcemap-reader/.gitignore create mode 100644 sourcemap-reader/package-lock.json create mode 100644 sourcemap-reader/package.json create mode 100644 sourcemap-reader/server.js create mode 100644 sourcemap-reader/servers/sourcemaps-handler.js create mode 100644 sourcemap-reader/servers/sourcemaps-server.js diff --git a/ee/utilities/package.json b/ee/utilities/package.json index ec895470b..99c2666da 100644 --- a/ee/utilities/package.json +++ b/ee/utilities/package.json @@ -1,5 +1,5 @@ { - "name": "utilities_server", + "name": "utilities-server", "version": "1.0.0", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", @@ -12,7 +12,7 @@ "url": "git+https://github.com/openreplay/openreplay.git" }, "author": "KRAIEM Taha Yassine ", - "license": "MIT", + "license": "Elastic License 2.0 (ELv2)", "bugs": { "url": "https://github.com/openreplay/openreplay/issues" }, diff --git a/peers/package.json b/peers/package.json index ba29b0899..8a87b3b6a 100644 --- a/peers/package.json +++ b/peers/package.json @@ -1,5 +1,5 @@ { - "name": "utilities_server", + "name": "peers-server", "version": "1.0.0", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", @@ -12,7 +12,7 @@ "url": "git+https://github.com/openreplay/openreplay.git" }, "author": "KRAIEM Taha Yassine ", - "license": "MIT", + "license": "Elastic License 2.0 (ELv2)", "bugs": { "url": "https://github.com/openreplay/openreplay/issues" }, diff --git a/sourcemap-reader/.gitignore b/sourcemap-reader/.gitignore new file mode 100644 index 000000000..a4b05b411 --- /dev/null +++ b/sourcemap-reader/.gitignore @@ -0,0 +1,6 @@ +.idea +node_modules +npm-debug.log +.cache +test.html +/utils/ diff --git a/sourcemap-reader/package-lock.json b/sourcemap-reader/package-lock.json new file mode 100644 index 000000000..e2871acb2 --- /dev/null +++ b/sourcemap-reader/package-lock.json @@ -0,0 +1,1113 @@ +{ + "name": "sourcemaps-reader", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "sourcemaps-reader", + "version": "1.0.0", + "license": "Elastic License 2.0 (ELv2)", + "dependencies": { + "aws-sdk": "^2.992.0", + "express": "^4.17.1", + "source-map": "^0.7.3" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "node_modules/aws-sdk": { + "version": "2.1087.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", + "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.16.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.9.7", + "raw-body": "2.4.3", + "type-is": "~1.6.18" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/express": { + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", + "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.19.2", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.4.2", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.9.7", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/jmespath": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", + "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", + "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.34", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", + "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", + "dependencies": { + "mime-db": "1.51.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "node_modules/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==", + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", + "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "node_modules/send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "dependencies": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "1.8.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/serve-static": { + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "node_modules/source-map": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", + "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "node_modules/xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", + "engines": { + "node": ">=4.0" + } + } + }, + "dependencies": { + "accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "requires": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + } + }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "aws-sdk": { + "version": "2.1087.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", + "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.16.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + } + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", + "requires": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.9.7", + "raw-body": "2.4.3", + "type-is": "~1.6.18" + } + }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "requires": { + "safe-buffer": "5.2.1" + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "express": { + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", + "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", + "requires": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.19.2", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.4.2", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.9.7", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + } + }, + "finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + } + }, + "forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + }, + "http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "jmespath": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", + "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==" + }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + }, + "mime-db": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", + "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==" + }, + "mime-types": { + "version": "2.1.34", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", + "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", + "requires": { + "mime-db": "1.51.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" + }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "requires": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + } + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" + }, + "raw-body": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", + "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", + "requires": { + "bytes": "3.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "1.8.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + }, + "serve-static": { + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.2" + } + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "source-map": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", + "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==" + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" + }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } +} diff --git a/sourcemap-reader/package.json b/sourcemap-reader/package.json new file mode 100644 index 000000000..8ed3189bb --- /dev/null +++ b/sourcemap-reader/package.json @@ -0,0 +1,25 @@ +{ + "name": "sourcemaps-reader", + "version": "1.0.0", + "description": "assist server to get live sessions & sourcemaps reader to get stack trace", + "main": "peerjs-server.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node server.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/openreplay/openreplay.git" + }, + "author": "KRAIEM Taha Yassine ", + "license": "Elastic License 2.0 (ELv2)", + "bugs": { + "url": "https://github.com/openreplay/openreplay/issues" + }, + "homepage": "https://github.com/openreplay/openreplay#readme", + "dependencies": { + "aws-sdk": "^2.992.0", + "express": "^4.17.1", + "source-map": "^0.7.3" + } +} diff --git a/sourcemap-reader/server.js b/sourcemap-reader/server.js new file mode 100644 index 000000000..073cb4cfc --- /dev/null +++ b/sourcemap-reader/server.js @@ -0,0 +1,19 @@ +const dumps = require('./utils/HeapSnapshot'); +const sourcemapsReaderServer = require('./servers/sourcemaps-server'); +const express = require('express'); +const {request_logger} = require("./utils/helper"); + +const HOST = '0.0.0.0'; +const PORT = 9000; + +const app = express(); +app.use(request_logger("[wsapp]")); + +app.use('/sourcemaps', sourcemapsReaderServer); +app.use('/heapdump', dumps.router); + +const server = app.listen(PORT, HOST, () => { + console.log(`WS App listening on http://${HOST}:${PORT}`); + console.log('Press Ctrl+C to quit.'); +}); +module.exports = {server}; \ No newline at end of file diff --git a/sourcemap-reader/servers/sourcemaps-handler.js b/sourcemap-reader/servers/sourcemaps-handler.js new file mode 100644 index 000000000..91917104d --- /dev/null +++ b/sourcemap-reader/servers/sourcemaps-handler.js @@ -0,0 +1,104 @@ +'use strict'; +const sourceMap = require('source-map'); +const AWS = require('aws-sdk'); +const sourceMapVersion = require('../package.json').dependencies["source-map"]; +const URL = require('url'); +const getVersion = version => version.replace(/[\^\$\=\~]/, ""); + +module.exports.sourcemapReader = async event => { + sourceMap.SourceMapConsumer.initialize({ + "lib/mappings.wasm": `https://unpkg.com/source-map@${getVersion(sourceMapVersion)}/lib/mappings.wasm` + }); + let s3; + if (event.S3_HOST) { + s3 = new AWS.S3({ + endpoint: event.S3_HOST, + accessKeyId: event.S3_KEY, + secretAccessKey: event.S3_SECRET, + region: event.region, + s3ForcePathStyle: true, // needed with minio? + signatureVersion: 'v4' + }); + } else if (process.env.S3_HOST) { + s3 = new AWS.S3({ + endpoint: process.env.S3_HOST, + accessKeyId: process.env.S3_KEY, + secretAccessKey: process.env.S3_SECRET, + s3ForcePathStyle: true, // needed with minio? + signatureVersion: 'v4' + }); + } else { + s3 = new AWS.S3({ + 'AccessKeyID': process.env.aws_access_key_id, + 'SecretAccessKey': process.env.aws_secret_access_key, + 'Region': process.env.aws_region + }); + } + + var options = { + Bucket: event.bucket, + Key: event.key + }; + return new Promise(function (resolve, reject) { + s3.getObject(options, (err, data) => { + if (err) { + console.log("Get S3 object failed"); + console.log(err); + return reject(err); + } + let sourcemap = data.Body.toString(); + + return new sourceMap.SourceMapConsumer(sourcemap) + .then(consumer => { + let results = []; + for (let i = 0; i < event.positions.length; i++) { + let original = consumer.originalPositionFor({ + line: event.positions[i].line, + column: event.positions[i].column + }); + let url = URL.parse(""); + let preview = []; + if (original.source) { + preview = consumer.sourceContentFor(original.source, true); + if (preview !== null) { + preview = preview.split("\n") + .map((line, i) => [i + 1, line]); + if (event.padding) { + let start = original.line < event.padding ? 0 : original.line - event.padding; + preview = preview.slice(start, original.line + event.padding); + } + } else { + console.log("source not found, null preview for:"); + console.log(original.source); + preview = [] + } + url = URL.parse(original.source); + } else { + console.log("couldn't find original position of:"); + console.log({ + line: event.positions[i].line, + column: event.positions[i].column + }); + } + let result = { + "absPath": url.href, + "filename": url.pathname, + "lineNo": original.line, + "colNo": original.column, + "function": original.name, + "context": preview + }; + // console.log(result); + results.push(result); + } + consumer = undefined; + // Use this code if you don't use the http event with the LAMBDA-PROXY integration + return resolve(results); + }) + .finally(() => { + sourcemap = undefined; + }) + + }); + }); +}; \ No newline at end of file diff --git a/sourcemap-reader/servers/sourcemaps-server.js b/sourcemap-reader/servers/sourcemaps-server.js new file mode 100644 index 000000000..a1dd501a0 --- /dev/null +++ b/sourcemap-reader/servers/sourcemaps-server.js @@ -0,0 +1,30 @@ +var express = require('express'); +var handler = require('./sourcemaps-handler'); +var router = express.Router(); + +router.post('/', (req, res) => { + let data = ''; + req.on('data', chunk => { + data += chunk; + }); + req.on('end', function () { + data = JSON.parse(data); + console.log("Starting parser for: " + data.key); + // process.env = {...process.env, ...data.bucket_config}; + handler.sourcemapReader(data) + .then((results) => { + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(results)); + }) + .catch((e) => { + console.error("Something went wrong"); + console.error(e); + res.statusCode(500); + res.end(e); + }); + }) + +}); + +module.exports = router; \ No newline at end of file diff --git a/utilities/package.json b/utilities/package.json index 483f1b4aa..73b92d0f2 100644 --- a/utilities/package.json +++ b/utilities/package.json @@ -1,5 +1,5 @@ { - "name": "utilities_server", + "name": "utilities-server", "version": "1.0.0", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", @@ -12,7 +12,7 @@ "url": "git+https://github.com/openreplay/openreplay.git" }, "author": "KRAIEM Taha Yassine ", - "license": "MIT", + "license": "Elastic License 2.0 (ELv2)", "bugs": { "url": "https://github.com/openreplay/openreplay/issues" }, From 7308ba2c1868db32b8f3002ae4306901590cbcf9 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 19:16:24 +0200 Subject: [PATCH 103/125] feat(api): merged sourcemaps reader with API build --- api/.env.default | 2 +- api/Dockerfile | 8 ++++++++ api/build.sh | 2 ++ api/entrypoint.sh | 3 +++ ee/api/.env.default | 2 +- ee/api/.gitignore | 1 + ee/api/Dockerfile | 8 ++++++++ ee/api/entrypoint.sh | 2 -- 8 files changed, 24 insertions(+), 4 deletions(-) delete mode 100755 ee/api/entrypoint.sh diff --git a/api/.env.default b/api/.env.default index 0d4f8130c..6caeb96d8 100644 --- a/api/.env.default +++ b/api/.env.default @@ -44,6 +44,6 @@ sentryURL= sessions_bucket=mobs sessions_region=us-east-1 sourcemaps_bucket=sourcemaps -sourcemaps_reader=http://utilities-openreplay.app.svc.cluster.local:9000/sourcemaps +sourcemaps_reader=http://127.0.0.1:9000/ stage=default-foss version_number=1.4.0 \ No newline at end of file diff --git a/api/Dockerfile b/api/Dockerfile index 4526c32bd..0673ab2b5 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -6,6 +6,14 @@ COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env ENV APP_NAME chalice +# Installing Nodejs +RUN apt update && apt install -y curl && \ + curl -fsSL https://deb.nodesource.com/setup_12.x | bash - && \ + apt install -y nodejs && \ + apt remove --purge -y curl && \ + rm -rf /var/lib/apt/lists/* && \ + cd sourcemap-reader && \ + npm install # Add Tini # Startup daemon diff --git a/api/build.sh b/api/build.sh index 29b8911ca..cec7525f5 100644 --- a/api/build.sh +++ b/api/build.sh @@ -18,6 +18,8 @@ check_prereq() { } function build_api(){ + cp -R ../utilities/utils ../sourcemap-reader/. + cp -R ../sourcemap-reader . tag="" # Copy enterprise code [[ $1 == "ee" ]] && { diff --git a/api/entrypoint.sh b/api/entrypoint.sh index a092737be..fe5912f0f 100755 --- a/api/entrypoint.sh +++ b/api/entrypoint.sh @@ -1,2 +1,5 @@ #!/bin/bash +cd sourcemap-reader +nohup npm start &> /tmp/sourcemap-reader.log & +cd .. uvicorn app:app --host 0.0.0.0 --reload diff --git a/ee/api/.env.default b/ee/api/.env.default index 778a8f32c..b4b1ad291 100644 --- a/ee/api/.env.default +++ b/ee/api/.env.default @@ -53,6 +53,6 @@ sentryURL= sessions_bucket=mobs sessions_region=us-east-1 sourcemaps_bucket=sourcemaps -sourcemaps_reader=http://utilities-openreplay.app.svc.cluster.local:9000/sourcemaps +sourcemaps_reader=http://127.0.0.1:9000/ stage=default-ee version_number=1.0.0 diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 0c649b68e..f8ff0f789 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -261,3 +261,4 @@ Pipfile /routers/subs/metrics.py /routers/subs/v1_api.py /chalicelib/core/dashboards2.py +entrypoint.sh \ No newline at end of file diff --git a/ee/api/Dockerfile b/ee/api/Dockerfile index ee88ee22c..aee6aecb2 100644 --- a/ee/api/Dockerfile +++ b/ee/api/Dockerfile @@ -7,6 +7,14 @@ COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env ENV APP_NAME chalice +# Installing Nodejs +RUN apt update && apt install -y curl && \ + curl -fsSL https://deb.nodesource.com/setup_12.x | bash - && \ + apt install -y nodejs && \ + apt remove --purge -y curl && \ + rm -rf /var/lib/apt/lists/* && \ + cd sourcemap-reader && \ + npm install # Add Tini # Startup daemon diff --git a/ee/api/entrypoint.sh b/ee/api/entrypoint.sh deleted file mode 100755 index a092737be..000000000 --- a/ee/api/entrypoint.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -uvicorn app:app --host 0.0.0.0 --reload From 8429118521181418819d8e3f4983188abef3eaac Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 19:22:55 +0200 Subject: [PATCH 104/125] feat(api): dashboard limit errors_per_domains widget to 5 --- api/routers/subs/dashboard.py | 76 ++++++++++++++--------------- ee/api/chalicelib/core/dashboard.py | 2 +- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/api/routers/subs/dashboard.py b/api/routers/subs/dashboard.py index e2d4ba268..4e61c3a4e 100644 --- a/api/routers/subs/dashboard.py +++ b/api/routers/subs/dashboard.py @@ -352,46 +352,46 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body @app.get('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body(...)): results = [ - {"key": schemas.TemplatePredefinedKeys.count_sessions, - "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_image_load_time, - "data": dashboard.get_application_activity_avg_image_load_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_page_load_time, - "data": dashboard.get_application_activity_avg_page_load_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_request_load_time, - "data": dashboard.get_application_activity_avg_request_load_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_dom_content_load_start, - "data": dashboard.get_page_metrics_avg_dom_content_load_start(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_first_contentful_pixel, - "data": dashboard.get_page_metrics_avg_first_contentful_pixel(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_visited_pages, - "data": dashboard.get_user_activity_avg_visited_pages(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_session_duration, - "data": dashboard.get_user_activity_avg_session_duration(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_pages_dom_buildtime, - "data": dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_pages_response_time, - "data": dashboard.get_pages_response_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_response_time, - "data": dashboard.get_top_metrics_avg_response_time(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_first_paint, - "data": dashboard.get_top_metrics_avg_first_paint(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.count_sessions, + # "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_image_load_time, + # "data": dashboard.get_application_activity_avg_image_load_time(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_page_load_time, + # "data": dashboard.get_application_activity_avg_page_load_time(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_request_load_time, + # "data": dashboard.get_application_activity_avg_request_load_time(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_dom_content_load_start, + # "data": dashboard.get_page_metrics_avg_dom_content_load_start(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_first_contentful_pixel, + # "data": dashboard.get_page_metrics_avg_first_contentful_pixel(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_visited_pages, + # "data": dashboard.get_user_activity_avg_visited_pages(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_session_duration, + # "data": dashboard.get_user_activity_avg_session_duration(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_pages_dom_buildtime, + # "data": dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_pages_response_time, + # "data": dashboard.get_pages_response_time(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_response_time, + # "data": dashboard.get_top_metrics_avg_response_time(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_first_paint, + # "data": dashboard.get_top_metrics_avg_first_paint(project_id=projectId, **data.dict())}, {"key": schemas.TemplatePredefinedKeys.avg_dom_content_loaded, "data": dashboard.get_top_metrics_avg_dom_content_loaded(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_till_first_bit, - "data": dashboard.get_top_metrics_avg_till_first_bit(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_time_to_interactive, - "data": dashboard.get_top_metrics_avg_time_to_interactive(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.count_requests, - "data": dashboard.get_top_metrics_count_requests(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_time_to_render, - "data": dashboard.get_time_to_render(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_used_js_heap_size, - "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_cpu, - "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, - {"key": schemas.TemplatePredefinedKeys.avg_fps, - "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} + # {"key": schemas.TemplatePredefinedKeys.avg_till_first_bit, + # "data": dashboard.get_top_metrics_avg_till_first_bit(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_time_to_interactive, + # "data": dashboard.get_top_metrics_avg_time_to_interactive(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.count_requests, + # "data": dashboard.get_top_metrics_count_requests(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_time_to_render, + # "data": dashboard.get_time_to_render(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_used_js_heap_size, + # "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_cpu, + # "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, + # {"key": schemas.TemplatePredefinedKeys.avg_fps, + # "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} ] results = sorted(results, key=lambda r: r["key"]) return {"data": results} diff --git a/ee/api/chalicelib/core/dashboard.py b/ee/api/chalicelib/core/dashboard.py index dc99ac0d2..404aef795 100644 --- a/ee/api/chalicelib/core/dashboard.py +++ b/ee/api/chalicelib/core/dashboard.py @@ -1690,7 +1690,7 @@ def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1) WHERE {" AND ".join(ch_sub_query)} GROUP BY resources.url_host ORDER BY errors_count DESC - LIMIT 10;""" + LIMIT 5;""" rows = ch.execute(query=ch_query, params={"project_id": project_id, "startTimestamp": startTimestamp, From 34ece1f38eeb612cc1515acfe3fcccbe6e9576a6 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 13 Apr 2022 19:24:15 +0200 Subject: [PATCH 105/125] feat(api): dashboard limit errors_per_domains widget to 5 --- api/chalicelib/core/dashboard.py | 2 +- api/routers/subs/dashboard.py | 76 ++++++++++++++++---------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index 66ee770d8..719fabb3a 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -1784,7 +1784,7 @@ def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1) WHERE {" AND ".join(pg_sub_query)} GROUP BY resources.url_host ORDER BY errors_count DESC - LIMIT 10;""" + LIMIT 5;""" cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)})) diff --git a/api/routers/subs/dashboard.py b/api/routers/subs/dashboard.py index 4e61c3a4e..e2d4ba268 100644 --- a/api/routers/subs/dashboard.py +++ b/api/routers/subs/dashboard.py @@ -352,46 +352,46 @@ def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body @app.get('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body(...)): results = [ - # {"key": schemas.TemplatePredefinedKeys.count_sessions, - # "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_image_load_time, - # "data": dashboard.get_application_activity_avg_image_load_time(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_page_load_time, - # "data": dashboard.get_application_activity_avg_page_load_time(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_request_load_time, - # "data": dashboard.get_application_activity_avg_request_load_time(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_dom_content_load_start, - # "data": dashboard.get_page_metrics_avg_dom_content_load_start(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_first_contentful_pixel, - # "data": dashboard.get_page_metrics_avg_first_contentful_pixel(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_visited_pages, - # "data": dashboard.get_user_activity_avg_visited_pages(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_session_duration, - # "data": dashboard.get_user_activity_avg_session_duration(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_pages_dom_buildtime, - # "data": dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_pages_response_time, - # "data": dashboard.get_pages_response_time(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_response_time, - # "data": dashboard.get_top_metrics_avg_response_time(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_first_paint, - # "data": dashboard.get_top_metrics_avg_first_paint(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.count_sessions, + "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_image_load_time, + "data": dashboard.get_application_activity_avg_image_load_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_page_load_time, + "data": dashboard.get_application_activity_avg_page_load_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_request_load_time, + "data": dashboard.get_application_activity_avg_request_load_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_dom_content_load_start, + "data": dashboard.get_page_metrics_avg_dom_content_load_start(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_first_contentful_pixel, + "data": dashboard.get_page_metrics_avg_first_contentful_pixel(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_visited_pages, + "data": dashboard.get_user_activity_avg_visited_pages(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_session_duration, + "data": dashboard.get_user_activity_avg_session_duration(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_pages_dom_buildtime, + "data": dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_pages_response_time, + "data": dashboard.get_pages_response_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_response_time, + "data": dashboard.get_top_metrics_avg_response_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_first_paint, + "data": dashboard.get_top_metrics_avg_first_paint(project_id=projectId, **data.dict())}, {"key": schemas.TemplatePredefinedKeys.avg_dom_content_loaded, "data": dashboard.get_top_metrics_avg_dom_content_loaded(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_till_first_bit, - # "data": dashboard.get_top_metrics_avg_till_first_bit(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_time_to_interactive, - # "data": dashboard.get_top_metrics_avg_time_to_interactive(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.count_requests, - # "data": dashboard.get_top_metrics_count_requests(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_time_to_render, - # "data": dashboard.get_time_to_render(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_used_js_heap_size, - # "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_cpu, - # "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, - # {"key": schemas.TemplatePredefinedKeys.avg_fps, - # "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} + {"key": schemas.TemplatePredefinedKeys.avg_till_first_bit, + "data": dashboard.get_top_metrics_avg_till_first_bit(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_time_to_interactive, + "data": dashboard.get_top_metrics_avg_time_to_interactive(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.count_requests, + "data": dashboard.get_top_metrics_count_requests(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_time_to_render, + "data": dashboard.get_time_to_render(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_used_js_heap_size, + "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_cpu, + "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_fps, + "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} ] results = sorted(results, key=lambda r: r["key"]) return {"data": results} From 2bb97b3ddff004b127c7d5dba6a4c8abb747dc4c Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 13 Apr 2022 21:53:13 +0200 Subject: [PATCH 106/125] feat(backend): pprof on http worker --- backend/services/http/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/services/http/main.go b/backend/services/http/main.go index f0ae22d32..d11d0d323 100644 --- a/backend/services/http/main.go +++ b/backend/services/http/main.go @@ -21,6 +21,8 @@ import ( "openreplay/backend/pkg/url/assets" "openreplay/backend/services/http/geoip" "openreplay/backend/services/http/uaparser" + + _ "net/http/pprof" ) var rewriter *assets.Rewriter From fb60c5d085e3c6bf7f66c5ff007f31a3658692cf Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 13 Apr 2022 23:02:12 +0200 Subject: [PATCH 107/125] fix(backend):pprof server start --- backend/pkg/pprof/pprof.go | 13 +++++++++++++ backend/services/http/main.go | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 backend/pkg/pprof/pprof.go diff --git a/backend/pkg/pprof/pprof.go b/backend/pkg/pprof/pprof.go new file mode 100644 index 000000000..a05080178 --- /dev/null +++ b/backend/pkg/pprof/pprof.go @@ -0,0 +1,13 @@ +package pprof + +import ( + "log" + "net/http" + _ "net/http/pprof" +) + +func StartProfilingServer() { + go func() { + log.Println(http.ListenAndServe("localhost:6060", nil)) + }() +} diff --git a/backend/services/http/main.go b/backend/services/http/main.go index d11d0d323..1f3bc93b3 100644 --- a/backend/services/http/main.go +++ b/backend/services/http/main.go @@ -22,7 +22,7 @@ import ( "openreplay/backend/services/http/geoip" "openreplay/backend/services/http/uaparser" - _ "net/http/pprof" + "openreplay/backend/pkg/pprof" ) var rewriter *assets.Rewriter @@ -45,6 +45,7 @@ var BEACON_SIZE_LIMIT int64 func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) + pprof.StartProfilingServer() producer = queue.NewProducer() defer producer.Close(15000) From 74833dd671c6618ea3a4d3a2a39c03b2dada7c3a Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Thu, 14 Apr 2022 10:06:40 +0000 Subject: [PATCH 108/125] Kubernetes native ingress (#417) * chore(ingress): changing to nginx-ingress controller https://github.com/openreplay/openreplay/issues/343 Signed-off-by: rjshrjndrn * chore(ingress): Migrating nginx ingress to kube native Signed-off-by: rjshrjndrn * chore(ingress): depricating old ingress * chore(ingress): frontend path TODO: have to migrate frontend from minio to nginx micro container Signed-off-by: rjshrjndrn * fix(ingress): assist port Signed-off-by: rjshrjndrn * chore(helm): nginx-ingress certissuer * chore(helm): update default certificate Signed-off-by: rjshrjndrn * chore(helm): removed old nginx-ingress * chore(helm): make ingress annotaion name same as the ingress class Signed-off-by: rjshrjndrn * chore(initsh): installing certmanager of SSL Signed-off-by: rjshrjndrn --- scripts/helmcharts/certmanager.sh | 34 + scripts/helmcharts/init.sh | 1 - scripts/helmcharts/openreplay/Chart.yaml | 5 + .../charts/assets/templates/ingress.yaml | 49 +- .../openreplay/charts/assets/values.yaml | 15 +- .../charts/chalice/templates/ingress.yaml | 49 +- .../openreplay/charts/chalice/values.yaml | 15 +- .../charts/http/templates/ingress.yaml | 108 +- .../openreplay/charts/http/values.yaml | 21 +- .../.helmignore | 1 - .../charts/ingress-nginx/CHANGELOG.md | 375 +++++++ .../charts/ingress-nginx/Chart.yaml | 25 + .../openreplay/charts/ingress-nginx/OWNERS | 10 + .../openreplay/charts/ingress-nginx/README.md | 486 +++++++++ .../charts/ingress-nginx/README.md.gotmpl | 235 +++++ .../controller-custom-ingressclass-flags.yaml | 7 + .../ci/daemonset-customconfig-values.yaml | 14 + .../ci/daemonset-customnodeport-values.yaml | 22 + .../ci/daemonset-extra-modules.yaml | 10 + .../ci/daemonset-headers-values.yaml | 14 + .../ci/daemonset-internal-lb-values.yaml | 14 + .../ci/daemonset-nodeport-values.yaml | 10 + .../ci/daemonset-podannotations-values.yaml | 17 + ...set-tcp-udp-configMapNamespace-values.yaml | 20 + .../ci/daemonset-tcp-udp-values.yaml | 16 + .../ci/daemonset-tcp-values.yaml | 14 + .../ci/deamonset-default-values.yaml | 10 + .../ci/deamonset-metrics-values.yaml | 12 + .../ci/deamonset-psp-values.yaml | 13 + .../ci/deamonset-webhook-and-psp-values.yaml | 13 + .../ci/deamonset-webhook-values.yaml | 10 + ...eployment-autoscaling-behavior-values.yaml | 14 + .../ci/deployment-autoscaling-values.yaml | 11 + .../ci/deployment-customconfig-values.yaml | 12 + .../ci/deployment-customnodeport-values.yaml | 20 + .../ci/deployment-default-values.yaml | 8 + .../ci/deployment-extra-modules.yaml | 10 + .../ci/deployment-headers-values.yaml | 13 + .../ci/deployment-internal-lb-values.yaml | 13 + .../ci/deployment-metrics-values.yaml | 11 + .../ci/deployment-nodeport-values.yaml | 9 + .../ci/deployment-podannotations-values.yaml | 16 + .../ci/deployment-psp-values.yaml | 10 + ...ent-tcp-udp-configMapNamespace-values.yaml | 19 + .../ci/deployment-tcp-udp-values.yaml | 15 + .../ci/deployment-tcp-values.yaml | 11 + .../ci/deployment-webhook-and-psp-values.yaml | 12 + .../deployment-webhook-resources-values.yaml | 23 + .../ci/deployment-webhook-values.yaml | 9 + .../charts/ingress-nginx/templates/NOTES.txt | 80 ++ .../ingress-nginx/templates/_helpers.tpl | 156 +++ .../ingress-nginx/templates/_params.tpl | 62 ++ .../job-patch/clusterrole.yaml | 34 + .../job-patch/clusterrolebinding.yaml | 23 + .../job-patch/job-createSecret.yaml | 76 ++ .../job-patch/job-patchWebhook.yaml | 78 ++ .../admission-webhooks/job-patch/psp.yaml | 39 + .../admission-webhooks/job-patch/role.yaml | 24 + .../job-patch/rolebinding.yaml | 24 + .../job-patch/serviceaccount.yaml | 16 + .../validating-webhook.yaml | 48 + .../ingress-nginx/templates/clusterrole.yaml | 87 ++ .../templates/clusterrolebinding.yaml | 19 + .../controller-configmap-addheaders.yaml | 14 + .../controller-configmap-proxyheaders.yaml | 19 + .../templates/controller-configmap-tcp.yaml | 17 + .../templates/controller-configmap-udp.yaml | 17 + .../templates/controller-configmap.yaml | 29 + .../templates/controller-daemonset.yaml | 227 +++++ .../templates/controller-deployment.yaml | 225 +++++ .../templates/controller-hpa.yaml | 52 + .../templates/controller-ingressclass.yaml | 21 + .../templates/controller-keda.yaml | 42 + .../controller-poddisruptionbudget.yaml | 19 + .../templates/controller-prometheusrules.yaml | 21 + .../templates/controller-psp.yaml | 89 ++ .../templates/controller-role.yaml | 93 ++ .../templates/controller-rolebinding.yaml | 21 + .../controller-service-internal.yaml | 79 ++ .../templates/controller-service-metrics.yaml | 45 + .../templates/controller-service-webhook.yaml | 40 + .../templates/controller-service.yaml | 101 ++ .../templates/controller-serviceaccount.yaml | 18 + .../templates/controller-servicemonitor.yaml | 48 + .../templates/default-backend-deployment.yaml | 118 +++ .../templates/default-backend-hpa.yaml | 33 + .../default-backend-poddisruptionbudget.yaml | 21 + .../templates/default-backend-psp.yaml | 36 + .../templates/default-backend-role.yaml | 22 + .../default-backend-rolebinding.yaml | 21 + .../templates/default-backend-service.yaml | 41 + .../default-backend-serviceaccount.yaml | 14 + .../templates/dh-param-secret.yaml | 10 + .../charts/ingress-nginx/values.yaml | 919 ++++++++++++++++++ .../charts/nginx-ingress/Chart.yaml | 24 - .../charts/nginx-ingress/files/site.crt | 1 - .../charts/nginx-ingress/files/site.key | 1 - .../charts/nginx-ingress/templates/NOTES.txt | 22 - .../nginx-ingress/templates/_helpers.tpl | 62 -- .../nginx-ingress/templates/configMap.yaml | 190 ---- .../nginx-ingress/templates/deployment.yaml | 80 -- .../charts/nginx-ingress/templates/hpa.yaml | 28 - .../nginx-ingress/templates/ingress.yaml | 61 -- .../nginx-ingress/templates/secrets.yaml | 9 - .../nginx-ingress/templates/service.yaml | 41 - .../templates/serviceaccount.yaml | 12 - .../templates/tests/test-connection.yaml | 15 - .../charts/nginx-ingress/values.yaml | 109 --- .../charts/utilities/templates/ingress.yaml | 62 +- .../openreplay/charts/utilities/values.yaml | 17 +- scripts/helmcharts/vars.yaml | 21 + 111 files changed, 5017 insertions(+), 857 deletions(-) create mode 100644 scripts/helmcharts/certmanager.sh rename scripts/helmcharts/openreplay/charts/{nginx-ingress => ingress-nginx}/.helmignore (97%) create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/CHANGELOG.md create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/Chart.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/OWNERS create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/README.md create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/README.md.gotmpl create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/controller-custom-ingressclass-flags.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customconfig-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customnodeport-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-extra-modules.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-headers-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-internal-lb-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-nodeport-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-podannotations-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-configMapNamespace-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-default-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-metrics-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-psp-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-and-psp-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-behavior-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customconfig-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customnodeport-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-default-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-extra-modules.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-headers-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-internal-lb-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-metrics-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-nodeport-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-podannotations-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-psp-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-configMapNamespace-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-and-psp-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-resources-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-values.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/NOTES.txt create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_helpers.tpl create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_params.tpl create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/psp.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrole.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrolebinding.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-proxyheaders.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-tcp.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-udp.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-daemonset.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-deployment.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-hpa.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-ingressclass.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-keda.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-poddisruptionbudget.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-prometheusrules.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-psp.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-role.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-rolebinding.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-internal.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-metrics.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-webhook.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-serviceaccount.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-servicemonitor.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-deployment.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-hpa.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-poddisruptionbudget.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-psp.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-role.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-rolebinding.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-service.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/templates/dh-param-secret.yaml create mode 100644 scripts/helmcharts/openreplay/charts/ingress-nginx/values.yaml delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml delete mode 120000 scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt delete mode 120000 scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml delete mode 100644 scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml diff --git a/scripts/helmcharts/certmanager.sh b/scripts/helmcharts/certmanager.sh new file mode 100644 index 000000000..23f138084 --- /dev/null +++ b/scripts/helmcharts/certmanager.sh @@ -0,0 +1,34 @@ +# --- helper functions for logs --- +info() +{ + echo '[INFO] ' "$@" +} +warn() +{ + echo '[WARN] ' "$@" >&2 +} +fatal() +{ + echo '[ERROR] ' "$@" >&2 + exit 1 +} + +# Reading email address for ssl certificate +[[ -z $EMAIL_ADDRESS ]] && { + read -p "Enter your email address for letsencrypt certificate: " EMAIL_ADDRESS + echo +} +if [[ "$EMAIL_ADDRESS" =~ ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}$ ]] +then + info "Email address $EMAIL_ADDRESS is valid." +else + fatal "Email address $EMAIL_ADDRESS is invalid." +fi + +sed -i "s/email: \"\"/email: \"${EMAIL_ADDRESS}\"/g" clusterIssuer.yaml +info "Installing cert-manager for auto letsencrypt certificate" +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.0/cert-manager.crds.yaml +helm repo add jetstack https://charts.jetstack.io +helm repo update +helm upgrade --install cert-manager --namespace cert-manager --version v1.8.0 jetstack/cert-manager --create-namespace +kubectl apply -f clusterIssuer.yaml diff --git a/scripts/helmcharts/init.sh b/scripts/helmcharts/init.sh index 281e8586b..c772629fc 100644 --- a/scripts/helmcharts/init.sh +++ b/scripts/helmcharts/init.sh @@ -81,7 +81,6 @@ sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"$(randomPass)\"/g" v sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"$(randomPass)\"/g" vars.yaml sed -i "s/domainName: \"\"/domainName: \"${DOMAIN_NAME}\"/g" vars.yaml - ## Installing OpenReplay info "Installing databases" helm upgrade --install databases ./databases -n db --create-namespace --wait -f ./vars.yaml --atomic diff --git a/scripts/helmcharts/openreplay/Chart.yaml b/scripts/helmcharts/openreplay/Chart.yaml index be16d6344..827f8566e 100644 --- a/scripts/helmcharts/openreplay/Chart.yaml +++ b/scripts/helmcharts/openreplay/Chart.yaml @@ -23,3 +23,8 @@ version: 0.1.0 # It is recommended to use it with quotes. # Ref: https://github.com/helm/helm/issues/7858#issuecomment-608114589 AppVersion: "v1.5.4" + +dependencies: + - name: ingress-nginx + version: "4.x.x" + repository: "https://kubernetes.github.io/ingress-nginx" diff --git a/scripts/helmcharts/openreplay/charts/assets/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/ingress.yaml index 62c421a9a..3c6ae561b 100644 --- a/scripts/helmcharts/openreplay/charts/assets/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/templates/ingress.yaml @@ -1,61 +1,34 @@ -{{- if .Values.ingress.enabled -}} +{{- if .Values.ingress.enabled }} {{- $fullName := include "assets.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} kind: Ingress metadata: name: {{ $fullName }} labels: {{- include "assets.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 + {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} + ingressClassName: "{{ tpl .Values.ingress.className . }}" tls: - {{- range .Values.ingress.tls }} - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} + - host: {{ .Values.global.domainName }} http: paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} + - pathType: Prefix backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: name: {{ $fullName }} port: number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} + path: /assets/(.*) {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/assets/values.yaml b/scripts/helmcharts/openreplay/charts/assets/values.yaml index 2069e0fe8..709f502d1 100644 --- a/scripts/helmcharts/openreplay/charts/assets/values.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/values.yaml @@ -41,20 +41,13 @@ service: port: 9000 ingress: - enabled: false - className: "" + enabled: true + className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}" annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local + tls: + secretName: openreplay-ssl resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/ingress.yaml index 8f680e949..d870e7673 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/ingress.yaml @@ -1,61 +1,34 @@ -{{- if .Values.ingress.enabled -}} +{{- if .Values.ingress.enabled }} {{- $fullName := include "chalice.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} kind: Ingress metadata: name: {{ $fullName }} labels: {{- include "chalice.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 + {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} + ingressClassName: "{{ tpl .Values.ingress.className . }}" tls: - {{- range .Values.ingress.tls }} - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} + - host: {{ .Values.global.domainName }} http: paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} + - pathType: Prefix backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: name: {{ $fullName }} port: number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} + path: /api/(.*) {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index 3290b9872..a95ed4732 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -41,20 +41,13 @@ service: port: 8000 ingress: - enabled: false - className: "" + enabled: true + className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}" annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local + tls: + secretName: openreplay-ssl resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml index 51a20e2a6..8888c7ae9 100644 --- a/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml @@ -1,61 +1,93 @@ -{{- if .Values.ingress.enabled -}} +{{- if .Values.ingress.enabled }} {{- $fullName := include "http.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} kind: Ingress metadata: name: {{ $fullName }} labels: {{- include "http.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/upstream-hash-by: $http_x_forwarded_for + {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} + ingressClassName: "{{ tpl .Values.ingress.className . }}" tls: - {{- range .Values.ingress.tls }} - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} + - host: {{ .Values.global.domainName }} http: paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} + - pathType: Prefix backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: name: {{ $fullName }} port: number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} + path: /ingest/(.*) {{- end }} + +## TODO: +## Frontend service from minio will be migrated to nginx atomic container. +## This ingress is just a workaround. +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: minio + namespace: db +spec: + ingressClassName: "{{ tpl .Values.ingress.className . }}" + rules: + - host: {{ .Values.global.domainName }} + http: + paths: + - pathType: Prefix + backend: + service: + name: minio + port: + number: 9000 + path: /(minio|mobs|sessions-assets|frontend|static|sourcemaps|ios-images)/ + tls: + - hosts: + - {{ .Values.global.domainName }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: minio-frontend + namespace: db + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /frontend/$1 + nginx.ingress.kubernetes.io/configuration-snippet: | + index /index.html; + rewrite ^((?!.(js|css|png|svg|jpg|woff|woff2)).)*$ /frontend/index.html break; + proxy_intercept_errors on; # see http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_intercept_errors + error_page 404 =200 /index.html; +spec: + ingressClassName: "{{ tpl .Values.ingress.className . }}" + rules: + - host: {{ .Values.global.domainName }} + http: + paths: + - pathType: Prefix + backend: + service: + name: minio + port: + number: 9000 + path: /(.*) + tls: + - hosts: + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} diff --git a/scripts/helmcharts/openreplay/charts/http/values.yaml b/scripts/helmcharts/openreplay/charts/http/values.yaml index 0eb280a12..1f52ed5cf 100644 --- a/scripts/helmcharts/openreplay/charts/http/values.yaml +++ b/scripts/helmcharts/openreplay/charts/http/values.yaml @@ -41,20 +41,17 @@ service: port: 80 ingress: - enabled: false - className: "" - annotations: {} + enabled: true + className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}" + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "120" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local + tls: + secretName: openreplay-ssl resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/.helmignore b/scripts/helmcharts/openreplay/charts/ingress-nginx/.helmignore similarity index 97% rename from scripts/helmcharts/openreplay/charts/nginx-ingress/.helmignore rename to scripts/helmcharts/openreplay/charts/ingress-nginx/.helmignore index 0e8a0eb36..50af03172 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/.helmignore +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/.helmignore @@ -14,7 +14,6 @@ *.swp *.bak *.tmp -*.orig *~ # Various IDEs .project diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/CHANGELOG.md b/scripts/helmcharts/openreplay/charts/ingress-nginx/CHANGELOG.md new file mode 100644 index 000000000..f3f44c336 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/CHANGELOG.md @@ -0,0 +1,375 @@ +# Changelog + +This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). + +### 4.0.18 +"[8291](https://github.com/kubernetes/ingress-nginx/pull/8291) remove git tag env from cloud build" +"[8286](https://github.com/kubernetes/ingress-nginx/pull/8286) Fix OpenTelemetry sidecar image build" +"[8277](https://github.com/kubernetes/ingress-nginx/pull/8277) Add OpenSSF Best practices badge" +"[8273](https://github.com/kubernetes/ingress-nginx/pull/8273) Issue#8241" +"[8267](https://github.com/kubernetes/ingress-nginx/pull/8267) Add fsGroup value to admission-webhooks/job-patch charts" +"[8262](https://github.com/kubernetes/ingress-nginx/pull/8262) Updated confusing error" +"[8256](https://github.com/kubernetes/ingress-nginx/pull/8256) fix: deny locations with invalid auth-url annotation" +"[8253](https://github.com/kubernetes/ingress-nginx/pull/8253) Add a certificate info metric" +"[8236](https://github.com/kubernetes/ingress-nginx/pull/8236) webhook: remove useless code." +"[8227](https://github.com/kubernetes/ingress-nginx/pull/8227) Update libraries in webhook image" +"[8225](https://github.com/kubernetes/ingress-nginx/pull/8225) fix inconsistent-label-cardinality for prometheus metrics: nginx_ingress_controller_requests" +"[8221](https://github.com/kubernetes/ingress-nginx/pull/8221) Do not validate ingresses with unknown ingress class in admission webhook endpoint" +"[8210](https://github.com/kubernetes/ingress-nginx/pull/8210) Bump github.com/prometheus/client_golang from 1.11.0 to 1.12.1" +"[8209](https://github.com/kubernetes/ingress-nginx/pull/8209) Bump google.golang.org/grpc from 1.43.0 to 1.44.0" +"[8204](https://github.com/kubernetes/ingress-nginx/pull/8204) Add Artifact Hub lint" +"[8203](https://github.com/kubernetes/ingress-nginx/pull/8203) Fix Indentation of example and link to cert-manager tutorial" +"[8201](https://github.com/kubernetes/ingress-nginx/pull/8201) feat(metrics): add path and method labels to requests countera" +"[8199](https://github.com/kubernetes/ingress-nginx/pull/8199) use functional options to reduce number of methods creating an EchoDeployment" +"[8196](https://github.com/kubernetes/ingress-nginx/pull/8196) docs: fix inconsistent controller annotation" +"[8191](https://github.com/kubernetes/ingress-nginx/pull/8191) Using Go install for misspell" +"[8186](https://github.com/kubernetes/ingress-nginx/pull/8186) prometheus+grafana using servicemonitor" +"[8185](https://github.com/kubernetes/ingress-nginx/pull/8185) Append elements on match, instead of removing for cors-annotations" +"[8179](https://github.com/kubernetes/ingress-nginx/pull/8179) Bump github.com/opencontainers/runc from 1.0.3 to 1.1.0" +"[8173](https://github.com/kubernetes/ingress-nginx/pull/8173) Adding annotations to the controller service account" +"[8163](https://github.com/kubernetes/ingress-nginx/pull/8163) Update the $req_id placeholder description" +"[8162](https://github.com/kubernetes/ingress-nginx/pull/8162) Versioned static manifests" +"[8159](https://github.com/kubernetes/ingress-nginx/pull/8159) Adding some geoip variables and default values" +"[8155](https://github.com/kubernetes/ingress-nginx/pull/8155) #7271 feat: avoid-pdb-creation-when-default-backend-disabled-and-replicas-gt-1" +"[8151](https://github.com/kubernetes/ingress-nginx/pull/8151) Automatically generate helm docs" +"[8143](https://github.com/kubernetes/ingress-nginx/pull/8143) Allow to configure delay before controller exits" +"[8136](https://github.com/kubernetes/ingress-nginx/pull/8136) add ingressClass option to helm chart - back compatibility with ingress.class annotations" +"[8126](https://github.com/kubernetes/ingress-nginx/pull/8126) Example for JWT" + + +### 4.0.15 + +- [8120] https://github.com/kubernetes/ingress-nginx/pull/8120 Update go in runner and release v1.1.1 +- [8119] https://github.com/kubernetes/ingress-nginx/pull/8119 Update to go v1.17.6 +- [8118] https://github.com/kubernetes/ingress-nginx/pull/8118 Remove deprecated libraries, update other libs +- [8117] https://github.com/kubernetes/ingress-nginx/pull/8117 Fix codegen errors +- [8115] https://github.com/kubernetes/ingress-nginx/pull/8115 chart/ghaction: set the correct permission to have access to push a release +- [8098] https://github.com/kubernetes/ingress-nginx/pull/8098 generating SHA for CA only certs in backend_ssl.go + comparision of P… +- [8088] https://github.com/kubernetes/ingress-nginx/pull/8088 Fix Edit this page link to use main branch +- [8072] https://github.com/kubernetes/ingress-nginx/pull/8072 Expose GeoIP2 Continent code as variable +- [8061] https://github.com/kubernetes/ingress-nginx/pull/8061 docs(charts): using helm-docs for chart +- [8058] https://github.com/kubernetes/ingress-nginx/pull/8058 Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 +- [8054] https://github.com/kubernetes/ingress-nginx/pull/8054 Bump google.golang.org/grpc from 1.41.0 to 1.43.0 +- [8051] https://github.com/kubernetes/ingress-nginx/pull/8051 align bug report with feature request regarding kind documentation +- [8046] https://github.com/kubernetes/ingress-nginx/pull/8046 Report expired certificates (#8045) +- [8044] https://github.com/kubernetes/ingress-nginx/pull/8044 remove G109 check till gosec resolves issues +- [8042] https://github.com/kubernetes/ingress-nginx/pull/8042 docs_multiple_instances_one_cluster_ticket_7543 +- [8041] https://github.com/kubernetes/ingress-nginx/pull/8041 docs: fix typo'd executible name +- [8035] https://github.com/kubernetes/ingress-nginx/pull/8035 Comment busy owners +- [8029] https://github.com/kubernetes/ingress-nginx/pull/8029 Add stream-snippet as a ConfigMap and Annotation option +- [8023] https://github.com/kubernetes/ingress-nginx/pull/8023 fix nginx compilation flags +- [8021] https://github.com/kubernetes/ingress-nginx/pull/8021 Disable default modsecurity_rules_file if modsecurity-snippet is specified +- [8019] https://github.com/kubernetes/ingress-nginx/pull/8019 Revise main documentation page +- [8018] https://github.com/kubernetes/ingress-nginx/pull/8018 Preserve order of plugin invocation +- [8015] https://github.com/kubernetes/ingress-nginx/pull/8015 Add newline indenting to admission webhook annotations +- [8014] https://github.com/kubernetes/ingress-nginx/pull/8014 Add link to example error page manifest in docs +- [8009] https://github.com/kubernetes/ingress-nginx/pull/8009 Fix spelling in documentation and top-level files +- [8008] https://github.com/kubernetes/ingress-nginx/pull/8008 Add relabelings in controller-servicemonitor.yaml +- [8003] https://github.com/kubernetes/ingress-nginx/pull/8003 Minor improvements (formatting, consistency) in install guide +- [8001] https://github.com/kubernetes/ingress-nginx/pull/8001 fix: go-grpc Dockerfile +- [7999] https://github.com/kubernetes/ingress-nginx/pull/7999 images: use k8s-staging-test-infra/gcb-docker-gcloud +- [7996] https://github.com/kubernetes/ingress-nginx/pull/7996 doc: improvement +- [7983] https://github.com/kubernetes/ingress-nginx/pull/7983 Fix a couple of misspellings in the annotations documentation. +- [7979] https://github.com/kubernetes/ingress-nginx/pull/7979 allow set annotations for admission Jobs +- [7977] https://github.com/kubernetes/ingress-nginx/pull/7977 Add ssl_reject_handshake to defaul server +- [7975] https://github.com/kubernetes/ingress-nginx/pull/7975 add legacy version update v0.50.0 to main changelog +- [7972] https://github.com/kubernetes/ingress-nginx/pull/7972 updated service upstream definition + +### 4.0.14 + +- [8061] https://github.com/kubernetes/ingress-nginx/pull/8061 Using helm-docs to populate values table in README.md + +### 4.0.13 + +- [8008] https://github.com/kubernetes/ingress-nginx/pull/8008 Add relabelings in controller-servicemonitor.yaml + +### 4.0.12 + +- [7978] https://github.com/kubernetes/ingress-nginx/pull/7979 Support custom annotations in admissions Jobs + +### 4.0.11 + +- [7873] https://github.com/kubernetes/ingress-nginx/pull/7873 Makes the [appProtocol](https://kubernetes.io/docs/concepts/services-networking/_print/#application-protocol) field optional. + +### 4.0.10 + +- [7964] https://github.com/kubernetes/ingress-nginx/pull/7964 Update controller version to v1.1.0 + +### 4.0.9 + +- [6992] https://github.com/kubernetes/ingress-nginx/pull/6992 Add ability to specify labels for all resources + +### 4.0.7 + +- [7923] https://github.com/kubernetes/ingress-nginx/pull/7923 Release v1.0.5 of ingress-nginx +- [7806] https://github.com/kubernetes/ingress-nginx/pull/7806 Choice option for internal/external loadbalancer type service + +### 4.0.6 + +- [7804] https://github.com/kubernetes/ingress-nginx/pull/7804 Release v1.0.4 of ingress-nginx +- [7651] https://github.com/kubernetes/ingress-nginx/pull/7651 Support ipFamilyPolicy and ipFamilies fields in Helm Chart +- [7798] https://github.com/kubernetes/ingress-nginx/pull/7798 Exoscale: use HTTP Healthcheck mode +- [7793] https://github.com/kubernetes/ingress-nginx/pull/7793 Update kube-webhook-certgen to v1.1.1 + +### 4.0.5 + +- [7740] https://github.com/kubernetes/ingress-nginx/pull/7740 Release v1.0.3 of ingress-nginx + +### 4.0.3 + +- [7707] https://github.com/kubernetes/ingress-nginx/pull/7707 Release v1.0.2 of ingress-nginx + +### 4.0.2 + +- [7681] https://github.com/kubernetes/ingress-nginx/pull/7681 Release v1.0.1 of ingress-nginx + +### 4.0.1 + +- [7535] https://github.com/kubernetes/ingress-nginx/pull/7535 Release v1.0.0 ingress-nginx + +### 3.34.0 + +- [7256] https://github.com/kubernetes/ingress-nginx/pull/7256 Add namespace field in the namespace scoped resource templates + +### 3.33.0 + +- [7164] https://github.com/kubernetes/ingress-nginx/pull/7164 Update nginx to v1.20.1 + +### 3.32.0 + +- [7117] https://github.com/kubernetes/ingress-nginx/pull/7117 Add annotations for HPA + +### 3.31.0 + +- [7137] https://github.com/kubernetes/ingress-nginx/pull/7137 Add support for custom probes + +### 3.30.0 + +- [#7092](https://github.com/kubernetes/ingress-nginx/pull/7092) Removes the possibility of using localhost in ExternalNames as endpoints + +### 3.29.0 + +- [X] [#6945](https://github.com/kubernetes/ingress-nginx/pull/7020) Add option to specify job label for ServiceMonitor + +### 3.28.0 + +- [ ] [#6900](https://github.com/kubernetes/ingress-nginx/pull/6900) Support existing PSPs + +### 3.27.0 + +- Update ingress-nginx v0.45.0 + +### 3.26.0 + +- [X] [#6979](https://github.com/kubernetes/ingress-nginx/pull/6979) Changed servicePort value for metrics + +### 3.25.0 + +- [X] [#6957](https://github.com/kubernetes/ingress-nginx/pull/6957) Add ability to specify automountServiceAccountToken + +### 3.24.0 + +- [X] [#6908](https://github.com/kubernetes/ingress-nginx/pull/6908) Add volumes to default-backend deployment + +### 3.23.0 + +- Update ingress-nginx v0.44.0 + +### 3.22.0 + +- [X] [#6802](https://github.com/kubernetes/ingress-nginx/pull/6802) Add value for configuring a custom Diffie-Hellman parameters file +- [X] [#6815](https://github.com/kubernetes/ingress-nginx/pull/6815) Allow use of numeric namespaces in helm chart + +### 3.21.0 + +- [X] [#6783](https://github.com/kubernetes/ingress-nginx/pull/6783) Add custom annotations to ScaledObject +- [X] [#6761](https://github.com/kubernetes/ingress-nginx/pull/6761) Adding quotes in the serviceAccount name in Helm values +- [X] [#6767](https://github.com/kubernetes/ingress-nginx/pull/6767) Remove ClusterRole when scope option is enabled +- [X] [#6785](https://github.com/kubernetes/ingress-nginx/pull/6785) Update kube-webhook-certgen image to v1.5.1 + +### 3.20.1 + +- Do not create KEDA in case of DaemonSets. +- Fix KEDA v2 definition + +### 3.20.0 + +- [X] [#6730](https://github.com/kubernetes/ingress-nginx/pull/6730) Do not create HPA for defaultBackend if not enabled. + +### 3.19.0 + +- Update ingress-nginx v0.43.0 + +### 3.18.0 + +- [X] [#6688](https://github.com/kubernetes/ingress-nginx/pull/6688) Allow volume-type emptyDir in controller podsecuritypolicy +- [X] [#6691](https://github.com/kubernetes/ingress-nginx/pull/6691) Improve parsing of helm parameters + +### 3.17.0 + +- Update ingress-nginx v0.42.0 + +### 3.16.1 + +- Fix chart-releaser action + +### 3.16.0 + +- [X] [#6646](https://github.com/kubernetes/ingress-nginx/pull/6646) Added LoadBalancerIP value for internal service + +### 3.15.1 + +- Fix chart-releaser action + +### 3.15.0 + +- [X] [#6586](https://github.com/kubernetes/ingress-nginx/pull/6586) Fix 'maxmindLicenseKey' location in values.yaml + +### 3.14.0 + +- [X] [#6469](https://github.com/kubernetes/ingress-nginx/pull/6469) Allow custom service names for controller and backend + +### 3.13.0 + +- [X] [#6544](https://github.com/kubernetes/ingress-nginx/pull/6544) Fix default backend HPA name variable + +### 3.12.0 + +- [X] [#6514](https://github.com/kubernetes/ingress-nginx/pull/6514) Remove helm2 support and update docs + +### 3.11.1 + +- [X] [#6505](https://github.com/kubernetes/ingress-nginx/pull/6505) Reorder HPA resource list to work with GitOps tooling + +### 3.11.0 + +- Support Keda Autoscaling + +### 3.10.1 + +- Fix regression introduced in 0.41.0 with external authentication + +### 3.10.0 + +- Fix routing regression introduced in 0.41.0 with PathType Exact + +### 3.9.0 + +- [X] [#6423](https://github.com/kubernetes/ingress-nginx/pull/6423) Add Default backend HPA autoscaling + +### 3.8.0 + +- [X] [#6395](https://github.com/kubernetes/ingress-nginx/pull/6395) Update jettech/kube-webhook-certgen image +- [X] [#6377](https://github.com/kubernetes/ingress-nginx/pull/6377) Added loadBalancerSourceRanges for internal lbs +- [X] [#6356](https://github.com/kubernetes/ingress-nginx/pull/6356) Add securitycontext settings on defaultbackend +- [X] [#6401](https://github.com/kubernetes/ingress-nginx/pull/6401) Fix controller service annotations +- [X] [#6403](https://github.com/kubernetes/ingress-nginx/pull/6403) Initial helm chart changelog + +### 3.7.1 + +- [X] [#6326](https://github.com/kubernetes/ingress-nginx/pull/6326) Fix liveness and readiness probe path in daemonset chart + +### 3.7.0 + +- [X] [#6316](https://github.com/kubernetes/ingress-nginx/pull/6316) Numerals in podAnnotations in quotes [#6315](https://github.com/kubernetes/ingress-nginx/issues/6315) + +### 3.6.0 + +- [X] [#6305](https://github.com/kubernetes/ingress-nginx/pull/6305) Add default linux nodeSelector + +### 3.5.1 + +- [X] [#6299](https://github.com/kubernetes/ingress-nginx/pull/6299) Fix helm chart release + +### 3.5.0 + +- [X] [#6260](https://github.com/kubernetes/ingress-nginx/pull/6260) Allow Helm Chart to customize admission webhook's annotations, timeoutSeconds, namespaceSelector, objectSelector and cert files locations + +### 3.4.0 + +- [X] [#6268](https://github.com/kubernetes/ingress-nginx/pull/6268) Update to 0.40.2 in helm chart #6288 + +### 3.3.1 + +- [X] [#6259](https://github.com/kubernetes/ingress-nginx/pull/6259) Release helm chart +- [X] [#6258](https://github.com/kubernetes/ingress-nginx/pull/6258) Fix chart markdown link +- [X] [#6253](https://github.com/kubernetes/ingress-nginx/pull/6253) Release v0.40.0 + +### 3.3.1 + +- [X] [#6233](https://github.com/kubernetes/ingress-nginx/pull/6233) Add admission controller e2e test + +### 3.3.0 + +- [X] [#6203](https://github.com/kubernetes/ingress-nginx/pull/6203) Refactor parsing of key values +- [X] [#6162](https://github.com/kubernetes/ingress-nginx/pull/6162) Add helm chart options to expose metrics service as NodePort +- [X] [#6180](https://github.com/kubernetes/ingress-nginx/pull/6180) Fix helm chart admissionReviewVersions regression +- [X] [#6169](https://github.com/kubernetes/ingress-nginx/pull/6169) Fix Typo in example prometheus rules + +### 3.0.0 + +- [X] [#6167](https://github.com/kubernetes/ingress-nginx/pull/6167) Update chart requirements + +### 2.16.0 + +- [X] [#6154](https://github.com/kubernetes/ingress-nginx/pull/6154) add `topologySpreadConstraint` to controller + +### 2.15.0 + +- [X] [#6087](https://github.com/kubernetes/ingress-nginx/pull/6087) Adding parameter for externalTrafficPolicy in internal controller service spec + +### 2.14.0 + +- [X] [#6104](https://github.com/kubernetes/ingress-nginx/pull/6104) Misc fixes for nginx-ingress chart for better keel and prometheus-operator integration + +### 2.13.0 + +- [X] [#6093](https://github.com/kubernetes/ingress-nginx/pull/6093) Release v0.35.0 + +### 2.13.0 + +- [X] [#6093](https://github.com/kubernetes/ingress-nginx/pull/6093) Release v0.35.0 +- [X] [#6080](https://github.com/kubernetes/ingress-nginx/pull/6080) Switch images to k8s.gcr.io after Vanity Domain Flip + +### 2.12.1 + +- [X] [#6075](https://github.com/kubernetes/ingress-nginx/pull/6075) Sync helm chart affinity examples + +### 2.12.0 + +- [X] [#6039](https://github.com/kubernetes/ingress-nginx/pull/6039) Add configurable serviceMonitor metricRelabelling and targetLabels +- [X] [#6044](https://github.com/kubernetes/ingress-nginx/pull/6044) Fix YAML linting + +### 2.11.3 + +- [X] [#6038](https://github.com/kubernetes/ingress-nginx/pull/6038) Bump chart version PATCH + +### 2.11.2 + +- [X] [#5951](https://github.com/kubernetes/ingress-nginx/pull/5951) Bump chart patch version + +### 2.11.1 + +- [X] [#5900](https://github.com/kubernetes/ingress-nginx/pull/5900) Release helm chart for v0.34.1 + +### 2.11.0 + +- [X] [#5879](https://github.com/kubernetes/ingress-nginx/pull/5879) Update helm chart for v0.34.0 +- [X] [#5671](https://github.com/kubernetes/ingress-nginx/pull/5671) Make liveness probe more fault tolerant than readiness probe + +### 2.10.0 + +- [X] [#5843](https://github.com/kubernetes/ingress-nginx/pull/5843) Update jettech/kube-webhook-certgen image + +### 2.9.1 + +- [X] [#5823](https://github.com/kubernetes/ingress-nginx/pull/5823) Add quoting to sysctls because numeric values need to be presented as strings (#5823) + +### 2.9.0 + +- [X] [#5795](https://github.com/kubernetes/ingress-nginx/pull/5795) Use fully qualified images to avoid cri-o issues + + +### TODO + +Keep building the changelog using *git log charts* checking the tag diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/Chart.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/Chart.yaml new file mode 100644 index 000000000..6445231d5 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/Chart.yaml @@ -0,0 +1,25 @@ +annotations: + artifacthub.io/changes: | + - "#8253 Add a certificate info metric" + - "#8225 fix inconsistent-label-cardinality for prometheus metrics: nginx_ingress_controller_requests" + - "#8162 Versioned static manifests" + - "#8159 Adding some geoip variables and default values" + - "#8221 Do not validate ingresses with unknown ingress class in admission webhook endpoint" + artifacthub.io/prerelease: "false" +apiVersion: v2 +appVersion: 1.1.2 +description: Ingress controller for Kubernetes using NGINX as a reverse proxy and + load balancer +home: https://github.com/kubernetes/ingress-nginx +icon: https://upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Nginx_logo.svg/500px-Nginx_logo.svg.png +keywords: +- ingress +- nginx +kubeVersion: '>=1.19.0-0' +maintainers: +- name: ChiefAlexander +name: ingress-nginx +sources: +- https://github.com/kubernetes/ingress-nginx +type: application +version: 4.0.18 diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/OWNERS b/scripts/helmcharts/openreplay/charts/ingress-nginx/OWNERS new file mode 100644 index 000000000..6b7e049ca --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/OWNERS @@ -0,0 +1,10 @@ +# See the OWNERS docs: https://github.com/kubernetes/community/blob/master/contributors/guide/owners.md + +approvers: +- ingress-nginx-helm-maintainers + +reviewers: +- ingress-nginx-helm-reviewers + +labels: +- area/helm diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md b/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md new file mode 100644 index 000000000..ad641f726 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md @@ -0,0 +1,486 @@ +# ingress-nginx + +[ingress-nginx](https://github.com/kubernetes/ingress-nginx) Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer + +![Version: 4.0.18](https://img.shields.io/badge/Version-4.0.18-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.2](https://img.shields.io/badge/AppVersion-1.1.2-informational?style=flat-square) + +To use, add `ingressClassName: nginx` spec field or the `kubernetes.io/ingress.class: nginx` annotation to your Ingress resources. + +This chart bootstraps an ingress-nginx deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Chart version 3.x.x: Kubernetes v1.16+ +- Chart version 4.x.x and above: Kubernetes v1.19+ + +## Get Repo Info + +```console +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +``` + +## Install Chart + +**Important:** only helm3 is supported + +```console +helm install [RELEASE_NAME] ingress-nginx/ingress-nginx +``` + +The command deploys ingress-nginx on the Kubernetes cluster in the default configuration. + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading With Zero Downtime in Production + +By default the ingress-nginx controller has service interruptions whenever it's pods are restarted or redeployed. In order to fix that, see the excellent blog post by Lindsay Landry from Codecademy: [Kubernetes: Nginx and Zero Downtime in Production](https://medium.com/codecademy-engineering/kubernetes-nginx-and-zero-downtime-in-production-2c910c6a5ed8). + +### Migrating from stable/nginx-ingress + +There are two main ways to migrate a release from `stable/nginx-ingress` to `ingress-nginx/ingress-nginx` chart: + +1. For Nginx Ingress controllers used for non-critical services, the easiest method is to [uninstall](#uninstall-chart) the old release and [install](#install-chart) the new one +1. For critical services in production that require zero-downtime, you will want to: + 1. [Install](#install-chart) a second Ingress controller + 1. Redirect your DNS traffic from the old controller to the new controller + 1. Log traffic from both controllers during this changeover + 1. [Uninstall](#uninstall-chart) the old controller once traffic has fully drained from it + 1. For details on all of these steps see [Upgrading With Zero Downtime in Production](#upgrading-with-zero-downtime-in-production) + +Note that there are some different and upgraded configurations between the two charts, described by Rimas Mocevicius from JFrog in the "Upgrading to ingress-nginx Helm chart" section of [Migrating from Helm chart nginx-ingress to ingress-nginx](https://rimusz.net/migrating-to-ingress-nginx). As the `ingress-nginx/ingress-nginx` chart continues to update, you will want to check current differences by running [helm configuration](#configuration) commands on both charts. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values ingress-nginx/ingress-nginx +``` + +### PodDisruptionBudget + +Note that the PodDisruptionBudget resource will only be defined if the replicaCount is greater than one, +else it would make it impossible to evacuate a node. See [gh issue #7127](https://github.com/helm/charts/issues/7127) for more info. + +### Prometheus Metrics + +The Nginx ingress controller can export Prometheus metrics, by setting `controller.metrics.enabled` to `true`. + +You can add Prometheus annotations to the metrics service using `controller.metrics.service.annotations`. +Alternatively, if you use the Prometheus Operator, you can enable ServiceMonitor creation using `controller.metrics.serviceMonitor.enabled`. And set `controller.metrics.serviceMonitor.additionalLabels.release="prometheus"`. "release=prometheus" should match the label configured in the prometheus servicemonitor ( see `kubectl get servicemonitor prometheus-kube-prom-prometheus -oyaml -n prometheus`) + +### ingress-nginx nginx\_status page/stats server + +Previous versions of this chart had a `controller.stats.*` configuration block, which is now obsolete due to the following changes in nginx ingress controller: + +- In [0.16.1](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0161), the vts (virtual host traffic status) dashboard was removed +- In [0.23.0](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230), the status page at port 18080 is now a unix socket webserver only available at localhost. + You can use `curl --unix-socket /tmp/nginx-status-server.sock http://localhost/nginx_status` inside the controller container to access it locally, or use the snippet from [nginx-ingress changelog](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230) to re-enable the http server + +### ExternalDNS Service Configuration + +Add an [ExternalDNS](https://github.com/kubernetes-incubator/external-dns) annotation to the LoadBalancer service: + +```yaml +controller: + service: + annotations: + external-dns.alpha.kubernetes.io/hostname: kubernetes-example.com. +``` + +### AWS L7 ELB with SSL Termination + +Annotate the controller as shown in the [nginx-ingress l7 patch](https://github.com/kubernetes/ingress-nginx/blob/main/deploy/aws/l7/service-l7.yaml): + +```yaml +controller: + service: + targetPorts: + http: http + https: http + annotations: + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:XX-XXXX-X:XXXXXXXXX:certificate/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http" + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https" + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '3600' +``` + +### AWS route53-mapper + +To configure the LoadBalancer service with the [route53-mapper addon](https://github.com/kubernetes/kops/tree/master/addons/route53-mapper), add the `domainName` annotation and `dns` label: + +```yaml +controller: + service: + labels: + dns: "route53" + annotations: + domainName: "kubernetes-example.com" +``` + +### Additional Internal Load Balancer + +This setup is useful when you need both external and internal load balancers but don't want to have multiple ingress controllers and multiple ingress objects per application. + +By default, the ingress object will point to the external load balancer address, but if correctly configured, you can make use of the internal one if the URL you are looking up resolves to the internal load balancer's URL. + +You'll need to set both the following values: + +`controller.service.internal.enabled` +`controller.service.internal.annotations` + +If one of them is missing the internal load balancer will not be deployed. Example you may have `controller.service.internal.enabled=true` but no annotations set, in this case no action will be taken. + +`controller.service.internal.annotations` varies with the cloud service you're using. + +Example for AWS: + +```yaml +controller: + service: + internal: + enabled: true + annotations: + # Create internal ELB + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +Example for GCE: + +```yaml +controller: + service: + internal: + enabled: true + annotations: + # Create internal LB. More informations: https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing + # For GKE versions 1.17 and later + networking.gke.io/load-balancer-type: "Internal" + # For earlier versions + # cloud.google.com/load-balancer-type: "Internal" + + # Any other annotation can be declared here. +``` + +Example for Azure: + +```yaml +controller: + service: + annotations: + # Create internal LB + service.beta.kubernetes.io/azure-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +Example for Oracle Cloud Infrastructure: + +```yaml +controller: + service: + annotations: + # Create internal LB + service.beta.kubernetes.io/oci-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +An use case for this scenario is having a split-view DNS setup where the public zone CNAME records point to the external balancer URL while the private zone CNAME records point to the internal balancer URL. This way, you only need one ingress kubernetes object. + +Optionally you can set `controller.service.loadBalancerIP` if you need a static IP for the resulting `LoadBalancer`. + +### Ingress Admission Webhooks + +With nginx-ingress-controller version 0.25+, the nginx ingress controller pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent bad ingress from being added to the cluster. +**This feature is enabled by default since 0.31.0.** + +With nginx-ingress-controller in 0.25.* work only with kubernetes 1.14+, 0.26 fix [this issue](https://github.com/kubernetes/ingress-nginx/pull/4521) + +### Helm Error When Upgrading: spec.clusterIP: Invalid value: "" + +If you are upgrading this chart from a version between 0.31.0 and 1.2.2 then you may get an error like this: + +```console +Error: UPGRADE FAILED: Service "?????-controller" is invalid: spec.clusterIP: Invalid value: "": field is immutable +``` + +Detail of how and why are in [this issue](https://github.com/helm/charts/pull/13646) but to resolve this you can set `xxxx.service.omitClusterIP` to `true` where `xxxx` is the service referenced in the error. + +As of version `1.26.0` of this chart, by simply not providing any clusterIP value, `invalid: spec.clusterIP: Invalid value: "": field is immutable` will no longer occur since `clusterIP: ""` will not be rendered. + +## Requirements + +Kubernetes: `>=1.19.0-0` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| commonLabels | object | `{}` | | +| controller.addHeaders | object | `{}` | Will add custom headers before sending response traffic to the client according to: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#add-headers | +| controller.admissionWebhooks.annotations | object | `{}` | | +| controller.admissionWebhooks.certificate | string | `"/usr/local/certificates/cert"` | | +| controller.admissionWebhooks.createSecretJob.resources | object | `{}` | | +| controller.admissionWebhooks.enabled | bool | `true` | | +| controller.admissionWebhooks.existingPsp | string | `""` | Use an existing PSP instead of creating one | +| controller.admissionWebhooks.failurePolicy | string | `"Fail"` | | +| controller.admissionWebhooks.key | string | `"/usr/local/certificates/key"` | | +| controller.admissionWebhooks.labels | object | `{}` | Labels to be added to admission webhooks | +| controller.admissionWebhooks.namespaceSelector | object | `{}` | | +| controller.admissionWebhooks.objectSelector | object | `{}` | | +| controller.admissionWebhooks.patch.enabled | bool | `true` | | +| controller.admissionWebhooks.patch.fsGroup | int | `2000` | | +| controller.admissionWebhooks.patch.image.digest | string | `"sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660"` | | +| controller.admissionWebhooks.patch.image.image | string | `"ingress-nginx/kube-webhook-certgen"` | | +| controller.admissionWebhooks.patch.image.pullPolicy | string | `"IfNotPresent"` | | +| controller.admissionWebhooks.patch.image.registry | string | `"k8s.gcr.io"` | | +| controller.admissionWebhooks.patch.image.tag | string | `"v1.1.1"` | | +| controller.admissionWebhooks.patch.labels | object | `{}` | Labels to be added to patch job resources | +| controller.admissionWebhooks.patch.nodeSelector."kubernetes.io/os" | string | `"linux"` | | +| controller.admissionWebhooks.patch.podAnnotations | object | `{}` | | +| controller.admissionWebhooks.patch.priorityClassName | string | `""` | Provide a priority class name to the webhook patching job | +| controller.admissionWebhooks.patch.runAsUser | int | `2000` | | +| controller.admissionWebhooks.patch.tolerations | list | `[]` | | +| controller.admissionWebhooks.patchWebhookJob.resources | object | `{}` | | +| controller.admissionWebhooks.port | int | `8443` | | +| controller.admissionWebhooks.service.annotations | object | `{}` | | +| controller.admissionWebhooks.service.externalIPs | list | `[]` | | +| controller.admissionWebhooks.service.loadBalancerSourceRanges | list | `[]` | | +| controller.admissionWebhooks.service.servicePort | int | `443` | | +| controller.admissionWebhooks.service.type | string | `"ClusterIP"` | | +| controller.affinity | object | `{}` | Affinity and anti-affinity rules for server scheduling to nodes | +| controller.allowSnippetAnnotations | bool | `true` | This configuration defines if Ingress Controller should allow users to set their own *-snippet annotations, otherwise this is forbidden / dropped when users add those annotations. Global snippets in ConfigMap are still respected | +| controller.annotations | object | `{}` | Annotations to be added to the controller Deployment or DaemonSet | +| controller.autoscaling.behavior | object | `{}` | | +| controller.autoscaling.enabled | bool | `false` | | +| controller.autoscaling.maxReplicas | int | `11` | | +| controller.autoscaling.minReplicas | int | `1` | | +| controller.autoscaling.targetCPUUtilizationPercentage | int | `50` | | +| controller.autoscaling.targetMemoryUtilizationPercentage | int | `50` | | +| controller.autoscalingTemplate | list | `[]` | | +| controller.config | object | `{}` | Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/ | +| controller.configAnnotations | object | `{}` | Annotations to be added to the controller config configuration configmap. | +| controller.configMapNamespace | string | `""` | Allows customization of the configmap / nginx-configmap namespace; defaults to $(POD_NAMESPACE) | +| controller.containerName | string | `"controller"` | Configures the controller container name | +| controller.containerPort | object | `{"http":80,"https":443}` | Configures the ports that the nginx-controller listens on | +| controller.customTemplate.configMapKey | string | `""` | | +| controller.customTemplate.configMapName | string | `""` | | +| controller.dnsConfig | object | `{}` | Optionally customize the pod dnsConfig. | +| controller.dnsPolicy | string | `"ClusterFirst"` | Optionally change this to ClusterFirstWithHostNet in case you have 'hostNetwork: true'. By default, while using host network, name resolution uses the host's DNS. If you wish nginx-controller to keep resolving names inside the k8s network, use ClusterFirstWithHostNet. | +| controller.electionID | string | `"ingress-controller-leader"` | Election ID to use for status update | +| controller.enableMimalloc | bool | `true` | Enable mimalloc as a drop-in replacement for malloc. | +| controller.existingPsp | string | `""` | Use an existing PSP instead of creating one | +| controller.extraArgs | object | `{}` | Additional command line arguments to pass to nginx-ingress-controller E.g. to specify the default SSL certificate you can use | +| controller.extraContainers | list | `[]` | Additional containers to be added to the controller pod. See https://github.com/lemonldap-ng-controller/lemonldap-ng-controller as example. | +| controller.extraEnvs | list | `[]` | Additional environment variables to set | +| controller.extraInitContainers | list | `[]` | Containers, which are run before the app containers are started. | +| controller.extraModules | list | `[]` | | +| controller.extraVolumeMounts | list | `[]` | Additional volumeMounts to the controller main container. | +| controller.extraVolumes | list | `[]` | Additional volumes to the controller pod. | +| controller.healthCheckHost | string | `""` | Address to bind the health check endpoint. It is better to set this option to the internal node address if the ingress nginx controller is running in the `hostNetwork: true` mode. | +| controller.healthCheckPath | string | `"/healthz"` | Path of the health check endpoint. All requests received on the port defined by the healthz-port parameter are forwarded internally to this path. | +| controller.hostNetwork | bool | `false` | Required for use with CNI based kubernetes installations (such as ones set up by kubeadm), since CNI and hostport don't mix yet. Can be deprecated once https://github.com/kubernetes/kubernetes/issues/23920 is merged | +| controller.hostPort.enabled | bool | `false` | Enable 'hostPort' or not | +| controller.hostPort.ports.http | int | `80` | 'hostPort' http port | +| controller.hostPort.ports.https | int | `443` | 'hostPort' https port | +| controller.hostname | object | `{}` | Optionally customize the pod hostname. | +| controller.image.allowPrivilegeEscalation | bool | `true` | | +| controller.image.digest | string | `"sha256:28b11ce69e57843de44e3db6413e98d09de0f6688e33d4bd384002a44f78405c"` | | +| controller.image.image | string | `"ingress-nginx/controller"` | | +| controller.image.pullPolicy | string | `"IfNotPresent"` | | +| controller.image.registry | string | `"k8s.gcr.io"` | | +| controller.image.runAsUser | int | `101` | | +| controller.image.tag | string | `"v1.1.2"` | | +| controller.ingressClass | string | `"nginx"` | For backwards compatibility with ingress.class annotation, use ingressClass. Algorithm is as follows, first ingressClassName is considered, if not present, controller looks for ingress.class annotation | +| controller.ingressClassByName | bool | `false` | Process IngressClass per name (additionally as per spec.controller). | +| controller.ingressClassResource.controllerValue | string | `"k8s.io/ingress-nginx"` | Controller-value of the controller that is processing this ingressClass | +| controller.ingressClassResource.default | bool | `false` | Is this the default ingressClass for the cluster | +| controller.ingressClassResource.enabled | bool | `true` | Is this ingressClass enabled or not | +| controller.ingressClassResource.name | string | `"nginx"` | Name of the ingressClass | +| controller.ingressClassResource.parameters | object | `{}` | Parameters is a link to a custom resource containing additional configuration for the controller. This is optional if the controller does not require extra parameters. | +| controller.keda.apiVersion | string | `"keda.sh/v1alpha1"` | | +| controller.keda.behavior | object | `{}` | | +| controller.keda.cooldownPeriod | int | `300` | | +| controller.keda.enabled | bool | `false` | | +| controller.keda.maxReplicas | int | `11` | | +| controller.keda.minReplicas | int | `1` | | +| controller.keda.pollingInterval | int | `30` | | +| controller.keda.restoreToOriginalReplicaCount | bool | `false` | | +| controller.keda.scaledObject.annotations | object | `{}` | | +| controller.keda.triggers | list | `[]` | | +| controller.kind | string | `"Deployment"` | Use a `DaemonSet` or `Deployment` | +| controller.labels | object | `{}` | Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels | +| controller.lifecycle | object | `{"preStop":{"exec":{"command":["/wait-shutdown"]}}}` | Improve connection draining when ingress controller pod is deleted using a lifecycle hook: With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds to 300, allowing the draining of connections up to five minutes. If the active connections end before that, the pod will terminate gracefully at that time. To effectively take advantage of this feature, the Configmap feature worker-shutdown-timeout new value is 240s instead of 10s. | +| controller.livenessProbe.failureThreshold | int | `5` | | +| controller.livenessProbe.httpGet.path | string | `"/healthz"` | | +| controller.livenessProbe.httpGet.port | int | `10254` | | +| controller.livenessProbe.httpGet.scheme | string | `"HTTP"` | | +| controller.livenessProbe.initialDelaySeconds | int | `10` | | +| controller.livenessProbe.periodSeconds | int | `10` | | +| controller.livenessProbe.successThreshold | int | `1` | | +| controller.livenessProbe.timeoutSeconds | int | `1` | | +| controller.maxmindLicenseKey | string | `""` | Maxmind license key to download GeoLite2 Databases. | +| controller.metrics.enabled | bool | `false` | | +| controller.metrics.port | int | `10254` | | +| controller.metrics.prometheusRule.additionalLabels | object | `{}` | | +| controller.metrics.prometheusRule.enabled | bool | `false` | | +| controller.metrics.prometheusRule.rules | list | `[]` | | +| controller.metrics.service.annotations | object | `{}` | | +| controller.metrics.service.externalIPs | list | `[]` | List of IP addresses at which the stats-exporter service is available | +| controller.metrics.service.loadBalancerSourceRanges | list | `[]` | | +| controller.metrics.service.servicePort | int | `10254` | | +| controller.metrics.service.type | string | `"ClusterIP"` | | +| controller.metrics.serviceMonitor.additionalLabels | object | `{}` | | +| controller.metrics.serviceMonitor.enabled | bool | `false` | | +| controller.metrics.serviceMonitor.metricRelabelings | list | `[]` | | +| controller.metrics.serviceMonitor.namespace | string | `""` | | +| controller.metrics.serviceMonitor.namespaceSelector | object | `{}` | | +| controller.metrics.serviceMonitor.relabelings | list | `[]` | | +| controller.metrics.serviceMonitor.scrapeInterval | string | `"30s"` | | +| controller.metrics.serviceMonitor.targetLabels | list | `[]` | | +| controller.minAvailable | int | `1` | | +| controller.minReadySeconds | int | `0` | `minReadySeconds` to avoid killing pods before we are ready | +| controller.name | string | `"controller"` | | +| controller.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for controller pod assignment | +| controller.podAnnotations | object | `{}` | Annotations to be added to controller pods | +| controller.podLabels | object | `{}` | Labels to add to the pod container metadata | +| controller.podSecurityContext | object | `{}` | Security Context policies for controller pods | +| controller.priorityClassName | string | `""` | | +| controller.proxySetHeaders | object | `{}` | Will add custom headers before sending traffic to backends according to https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/custom-headers | +| controller.publishService | object | `{"enabled":true,"pathOverride":""}` | Allows customization of the source of the IP address or FQDN to report in the ingress status field. By default, it reads the information provided by the service. If disable, the status field reports the IP address of the node or nodes where an ingress controller pod is running. | +| controller.publishService.enabled | bool | `true` | Enable 'publishService' or not | +| controller.publishService.pathOverride | string | `""` | Allows overriding of the publish service to bind to Must be / | +| controller.readinessProbe.failureThreshold | int | `3` | | +| controller.readinessProbe.httpGet.path | string | `"/healthz"` | | +| controller.readinessProbe.httpGet.port | int | `10254` | | +| controller.readinessProbe.httpGet.scheme | string | `"HTTP"` | | +| controller.readinessProbe.initialDelaySeconds | int | `10` | | +| controller.readinessProbe.periodSeconds | int | `10` | | +| controller.readinessProbe.successThreshold | int | `1` | | +| controller.readinessProbe.timeoutSeconds | int | `1` | | +| controller.replicaCount | int | `1` | | +| controller.reportNodeInternalIp | bool | `false` | Bare-metal considerations via the host network https://kubernetes.github.io/ingress-nginx/deploy/baremetal/#via-the-host-network Ingress status was blank because there is no Service exposing the NGINX Ingress controller in a configuration using the host network, the default --publish-service flag used in standard cloud setups does not apply | +| controller.resources.requests.cpu | string | `"100m"` | | +| controller.resources.requests.memory | string | `"90Mi"` | | +| controller.scope.enabled | bool | `false` | Enable 'scope' or not | +| controller.scope.namespace | string | `""` | Namespace to limit the controller to; defaults to $(POD_NAMESPACE) | +| controller.scope.namespaceSelector | string | `""` | When scope.enabled == false, instead of watching all namespaces, we watching namespaces whose labels only match with namespaceSelector. Format like foo=bar. Defaults to empty, means watching all namespaces. | +| controller.service.annotations | object | `{}` | | +| controller.service.appProtocol | bool | `true` | If enabled is adding an appProtocol option for Kubernetes service. An appProtocol field replacing annotations that were using for setting a backend protocol. Here is an example for AWS: service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http It allows choosing the protocol for each backend specified in the Kubernetes service. See the following GitHub issue for more details about the purpose: https://github.com/kubernetes/kubernetes/issues/40244 Will be ignored for Kubernetes versions older than 1.20 | +| controller.service.enableHttp | bool | `true` | | +| controller.service.enableHttps | bool | `true` | | +| controller.service.enabled | bool | `true` | | +| controller.service.external.enabled | bool | `true` | | +| controller.service.externalIPs | list | `[]` | List of IP addresses at which the controller services are available | +| controller.service.internal.annotations | object | `{}` | Annotations are mandatory for the load balancer to come up. Varies with the cloud service. | +| controller.service.internal.enabled | bool | `false` | Enables an additional internal load balancer (besides the external one). | +| controller.service.internal.loadBalancerSourceRanges | list | `[]` | Restrict access For LoadBalancer service. Defaults to 0.0.0.0/0. | +| controller.service.ipFamilies | list | `["IPv4"]` | List of IP families (e.g. IPv4, IPv6) assigned to the service. This field is usually assigned automatically based on cluster configuration and the ipFamilyPolicy field. | +| controller.service.ipFamilyPolicy | string | `"SingleStack"` | Represents the dual-stack-ness requested or required by this Service. Possible values are SingleStack, PreferDualStack or RequireDualStack. The ipFamilies and clusterIPs fields depend on the value of this field. | +| controller.service.labels | object | `{}` | | +| controller.service.loadBalancerSourceRanges | list | `[]` | | +| controller.service.nodePorts.http | string | `""` | | +| controller.service.nodePorts.https | string | `""` | | +| controller.service.nodePorts.tcp | object | `{}` | | +| controller.service.nodePorts.udp | object | `{}` | | +| controller.service.ports.http | int | `80` | | +| controller.service.ports.https | int | `443` | | +| controller.service.targetPorts.http | string | `"http"` | | +| controller.service.targetPorts.https | string | `"https"` | | +| controller.service.type | string | `"LoadBalancer"` | | +| controller.sysctls | object | `{}` | See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for notes on enabling and using sysctls | +| controller.tcp.annotations | object | `{}` | Annotations to be added to the tcp config configmap | +| controller.tcp.configMapNamespace | string | `""` | Allows customization of the tcp-services-configmap; defaults to $(POD_NAMESPACE) | +| controller.terminationGracePeriodSeconds | int | `300` | `terminationGracePeriodSeconds` to avoid killing pods before we are ready | +| controller.tolerations | list | `[]` | Node tolerations for server scheduling to nodes with taints | +| controller.topologySpreadConstraints | list | `[]` | Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. | +| controller.udp.annotations | object | `{}` | Annotations to be added to the udp config configmap | +| controller.udp.configMapNamespace | string | `""` | Allows customization of the udp-services-configmap; defaults to $(POD_NAMESPACE) | +| controller.updateStrategy | object | `{}` | The update strategy to apply to the Deployment or DaemonSet | +| controller.watchIngressWithoutClass | bool | `false` | Process Ingress objects without ingressClass annotation/ingressClassName field Overrides value for --watch-ingress-without-class flag of the controller binary Defaults to false | +| defaultBackend.affinity | object | `{}` | | +| defaultBackend.autoscaling.annotations | object | `{}` | | +| defaultBackend.autoscaling.enabled | bool | `false` | | +| defaultBackend.autoscaling.maxReplicas | int | `2` | | +| defaultBackend.autoscaling.minReplicas | int | `1` | | +| defaultBackend.autoscaling.targetCPUUtilizationPercentage | int | `50` | | +| defaultBackend.autoscaling.targetMemoryUtilizationPercentage | int | `50` | | +| defaultBackend.containerSecurityContext | object | `{}` | Security Context policies for controller main container. See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for notes on enabling and using sysctls | +| defaultBackend.enabled | bool | `false` | | +| defaultBackend.existingPsp | string | `""` | Use an existing PSP instead of creating one | +| defaultBackend.extraArgs | object | `{}` | | +| defaultBackend.extraEnvs | list | `[]` | Additional environment variables to set for defaultBackend pods | +| defaultBackend.extraVolumeMounts | list | `[]` | | +| defaultBackend.extraVolumes | list | `[]` | | +| defaultBackend.image.allowPrivilegeEscalation | bool | `false` | | +| defaultBackend.image.image | string | `"defaultbackend-amd64"` | | +| defaultBackend.image.pullPolicy | string | `"IfNotPresent"` | | +| defaultBackend.image.readOnlyRootFilesystem | bool | `true` | | +| defaultBackend.image.registry | string | `"k8s.gcr.io"` | | +| defaultBackend.image.runAsNonRoot | bool | `true` | | +| defaultBackend.image.runAsUser | int | `65534` | | +| defaultBackend.image.tag | string | `"1.5"` | | +| defaultBackend.labels | object | `{}` | Labels to be added to the default backend resources | +| defaultBackend.livenessProbe.failureThreshold | int | `3` | | +| defaultBackend.livenessProbe.initialDelaySeconds | int | `30` | | +| defaultBackend.livenessProbe.periodSeconds | int | `10` | | +| defaultBackend.livenessProbe.successThreshold | int | `1` | | +| defaultBackend.livenessProbe.timeoutSeconds | int | `5` | | +| defaultBackend.minAvailable | int | `1` | | +| defaultBackend.name | string | `"defaultbackend"` | | +| defaultBackend.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for default backend pod assignment | +| defaultBackend.podAnnotations | object | `{}` | Annotations to be added to default backend pods | +| defaultBackend.podLabels | object | `{}` | Labels to add to the pod container metadata | +| defaultBackend.podSecurityContext | object | `{}` | Security Context policies for controller pods See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for notes on enabling and using sysctls | +| defaultBackend.port | int | `8080` | | +| defaultBackend.priorityClassName | string | `""` | | +| defaultBackend.readinessProbe.failureThreshold | int | `6` | | +| defaultBackend.readinessProbe.initialDelaySeconds | int | `0` | | +| defaultBackend.readinessProbe.periodSeconds | int | `5` | | +| defaultBackend.readinessProbe.successThreshold | int | `1` | | +| defaultBackend.readinessProbe.timeoutSeconds | int | `5` | | +| defaultBackend.replicaCount | int | `1` | | +| defaultBackend.resources | object | `{}` | | +| defaultBackend.service.annotations | object | `{}` | | +| defaultBackend.service.externalIPs | list | `[]` | List of IP addresses at which the default backend service is available | +| defaultBackend.service.loadBalancerSourceRanges | list | `[]` | | +| defaultBackend.service.servicePort | int | `80` | | +| defaultBackend.service.type | string | `"ClusterIP"` | | +| defaultBackend.serviceAccount.automountServiceAccountToken | bool | `true` | | +| defaultBackend.serviceAccount.create | bool | `true` | | +| defaultBackend.serviceAccount.name | string | `""` | | +| defaultBackend.tolerations | list | `[]` | Node tolerations for server scheduling to nodes with taints | +| dhParam | string | `nil` | A base64-encoded Diffie-Hellman parameter. This can be generated with: `openssl dhparam 4096 2> /dev/null | base64` | +| imagePullSecrets | list | `[]` | Optional array of imagePullSecrets containing private registry credentials | +| podSecurityPolicy.enabled | bool | `false` | | +| rbac.create | bool | `true` | | +| rbac.scope | bool | `false` | | +| revisionHistoryLimit | int | `10` | Rollback limit | +| serviceAccount.annotations | object | `{}` | Annotations for the controller service account | +| serviceAccount.automountServiceAccountToken | bool | `true` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| tcp | object | `{}` | TCP service key:value pairs | +| udp | object | `{}` | UDP service key:value pairs | + diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md.gotmpl b/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md.gotmpl new file mode 100644 index 000000000..5cd9e59e1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md.gotmpl @@ -0,0 +1,235 @@ +{{ template "chart.header" . }} +[ingress-nginx](https://github.com/kubernetes/ingress-nginx) Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +To use, add `ingressClassName: nginx` spec field or the `kubernetes.io/ingress.class: nginx` annotation to your Ingress resources. + +This chart bootstraps an ingress-nginx deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Chart version 3.x.x: Kubernetes v1.16+ +- Chart version 4.x.x and above: Kubernetes v1.19+ + +## Get Repo Info + +```console +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +``` + +## Install Chart + +**Important:** only helm3 is supported + +```console +helm install [RELEASE_NAME] ingress-nginx/ingress-nginx +``` + +The command deploys ingress-nginx on the Kubernetes cluster in the default configuration. + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading With Zero Downtime in Production + +By default the ingress-nginx controller has service interruptions whenever it's pods are restarted or redeployed. In order to fix that, see the excellent blog post by Lindsay Landry from Codecademy: [Kubernetes: Nginx and Zero Downtime in Production](https://medium.com/codecademy-engineering/kubernetes-nginx-and-zero-downtime-in-production-2c910c6a5ed8). + +### Migrating from stable/nginx-ingress + +There are two main ways to migrate a release from `stable/nginx-ingress` to `ingress-nginx/ingress-nginx` chart: + +1. For Nginx Ingress controllers used for non-critical services, the easiest method is to [uninstall](#uninstall-chart) the old release and [install](#install-chart) the new one +1. For critical services in production that require zero-downtime, you will want to: + 1. [Install](#install-chart) a second Ingress controller + 1. Redirect your DNS traffic from the old controller to the new controller + 1. Log traffic from both controllers during this changeover + 1. [Uninstall](#uninstall-chart) the old controller once traffic has fully drained from it + 1. For details on all of these steps see [Upgrading With Zero Downtime in Production](#upgrading-with-zero-downtime-in-production) + +Note that there are some different and upgraded configurations between the two charts, described by Rimas Mocevicius from JFrog in the "Upgrading to ingress-nginx Helm chart" section of [Migrating from Helm chart nginx-ingress to ingress-nginx](https://rimusz.net/migrating-to-ingress-nginx). As the `ingress-nginx/ingress-nginx` chart continues to update, you will want to check current differences by running [helm configuration](#configuration) commands on both charts. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values ingress-nginx/ingress-nginx +``` + +### PodDisruptionBudget + +Note that the PodDisruptionBudget resource will only be defined if the replicaCount is greater than one, +else it would make it impossible to evacuate a node. See [gh issue #7127](https://github.com/helm/charts/issues/7127) for more info. + +### Prometheus Metrics + +The Nginx ingress controller can export Prometheus metrics, by setting `controller.metrics.enabled` to `true`. + +You can add Prometheus annotations to the metrics service using `controller.metrics.service.annotations`. +Alternatively, if you use the Prometheus Operator, you can enable ServiceMonitor creation using `controller.metrics.serviceMonitor.enabled`. And set `controller.metrics.serviceMonitor.additionalLabels.release="prometheus"`. "release=prometheus" should match the label configured in the prometheus servicemonitor ( see `kubectl get servicemonitor prometheus-kube-prom-prometheus -oyaml -n prometheus`) + +### ingress-nginx nginx\_status page/stats server + +Previous versions of this chart had a `controller.stats.*` configuration block, which is now obsolete due to the following changes in nginx ingress controller: + +- In [0.16.1](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0161), the vts (virtual host traffic status) dashboard was removed +- In [0.23.0](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230), the status page at port 18080 is now a unix socket webserver only available at localhost. + You can use `curl --unix-socket /tmp/nginx-status-server.sock http://localhost/nginx_status` inside the controller container to access it locally, or use the snippet from [nginx-ingress changelog](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230) to re-enable the http server + +### ExternalDNS Service Configuration + +Add an [ExternalDNS](https://github.com/kubernetes-incubator/external-dns) annotation to the LoadBalancer service: + +```yaml +controller: + service: + annotations: + external-dns.alpha.kubernetes.io/hostname: kubernetes-example.com. +``` + +### AWS L7 ELB with SSL Termination + +Annotate the controller as shown in the [nginx-ingress l7 patch](https://github.com/kubernetes/ingress-nginx/blob/main/deploy/aws/l7/service-l7.yaml): + +```yaml +controller: + service: + targetPorts: + http: http + https: http + annotations: + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:XX-XXXX-X:XXXXXXXXX:certificate/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http" + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https" + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '3600' +``` + +### AWS route53-mapper + +To configure the LoadBalancer service with the [route53-mapper addon](https://github.com/kubernetes/kops/tree/master/addons/route53-mapper), add the `domainName` annotation and `dns` label: + +```yaml +controller: + service: + labels: + dns: "route53" + annotations: + domainName: "kubernetes-example.com" +``` + +### Additional Internal Load Balancer + +This setup is useful when you need both external and internal load balancers but don't want to have multiple ingress controllers and multiple ingress objects per application. + +By default, the ingress object will point to the external load balancer address, but if correctly configured, you can make use of the internal one if the URL you are looking up resolves to the internal load balancer's URL. + +You'll need to set both the following values: + +`controller.service.internal.enabled` +`controller.service.internal.annotations` + +If one of them is missing the internal load balancer will not be deployed. Example you may have `controller.service.internal.enabled=true` but no annotations set, in this case no action will be taken. + +`controller.service.internal.annotations` varies with the cloud service you're using. + +Example for AWS: + +```yaml +controller: + service: + internal: + enabled: true + annotations: + # Create internal ELB + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +Example for GCE: + +```yaml +controller: + service: + internal: + enabled: true + annotations: + # Create internal LB. More informations: https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing + # For GKE versions 1.17 and later + networking.gke.io/load-balancer-type: "Internal" + # For earlier versions + # cloud.google.com/load-balancer-type: "Internal" + + # Any other annotation can be declared here. +``` + +Example for Azure: + +```yaml +controller: + service: + annotations: + # Create internal LB + service.beta.kubernetes.io/azure-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +Example for Oracle Cloud Infrastructure: + +```yaml +controller: + service: + annotations: + # Create internal LB + service.beta.kubernetes.io/oci-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +An use case for this scenario is having a split-view DNS setup where the public zone CNAME records point to the external balancer URL while the private zone CNAME records point to the internal balancer URL. This way, you only need one ingress kubernetes object. + +Optionally you can set `controller.service.loadBalancerIP` if you need a static IP for the resulting `LoadBalancer`. + +### Ingress Admission Webhooks + +With nginx-ingress-controller version 0.25+, the nginx ingress controller pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent bad ingress from being added to the cluster. +**This feature is enabled by default since 0.31.0.** + +With nginx-ingress-controller in 0.25.* work only with kubernetes 1.14+, 0.26 fix [this issue](https://github.com/kubernetes/ingress-nginx/pull/4521) + +### Helm Error When Upgrading: spec.clusterIP: Invalid value: "" + +If you are upgrading this chart from a version between 0.31.0 and 1.2.2 then you may get an error like this: + +```console +Error: UPGRADE FAILED: Service "?????-controller" is invalid: spec.clusterIP: Invalid value: "": field is immutable +``` + +Detail of how and why are in [this issue](https://github.com/helm/charts/pull/13646) but to resolve this you can set `xxxx.service.omitClusterIP` to `true` where `xxxx` is the service referenced in the error. + +As of version `1.26.0` of this chart, by simply not providing any clusterIP value, `invalid: spec.clusterIP: Invalid value: "": field is immutable` will no longer occur since `clusterIP: ""` will not be rendered. + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} + +{{ template "helm-docs.versionFooter" . }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/controller-custom-ingressclass-flags.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/controller-custom-ingressclass-flags.yaml new file mode 100644 index 000000000..b28a2326e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/controller-custom-ingressclass-flags.yaml @@ -0,0 +1,7 @@ +controller: + watchIngressWithoutClass: true + ingressClassResource: + name: custom-nginx + enabled: true + default: true + controllerValue: "k8s.io/custom-nginx" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customconfig-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customconfig-values.yaml new file mode 100644 index 000000000..4393a5bc0 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customconfig-values.yaml @@ -0,0 +1,14 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + kind: DaemonSet + allowSnippetAnnotations: false + admissionWebhooks: + enabled: false + service: + type: ClusterIP + + config: + use-proxy-protocol: "true" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customnodeport-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customnodeport-values.yaml new file mode 100644 index 000000000..1d94be219 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customnodeport-values.yaml @@ -0,0 +1,22 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + + service: + type: NodePort + nodePorts: + tcp: + 9000: 30090 + udp: + 9001: 30091 + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-extra-modules.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-extra-modules.yaml new file mode 100644 index 000000000..f299dbf1c --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-extra-modules.yaml @@ -0,0 +1,10 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + service: + type: ClusterIP + extraModules: + - name: opentelemetry + image: busybox diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-headers-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-headers-values.yaml new file mode 100644 index 000000000..ab7d47bd4 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-headers-values.yaml @@ -0,0 +1,14 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + addHeaders: + X-Frame-Options: deny + proxySetHeaders: + X-Forwarded-Proto: https + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-internal-lb-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-internal-lb-values.yaml new file mode 100644 index 000000000..0a200a746 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-internal-lb-values.yaml @@ -0,0 +1,14 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + internal: + enabled: true + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: "true" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-nodeport-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-nodeport-values.yaml new file mode 100644 index 000000000..3b7aa2fcd --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-nodeport-values.yaml @@ -0,0 +1,10 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: NodePort diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-podannotations-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-podannotations-values.yaml new file mode 100644 index 000000000..0b55306a1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-podannotations-values.yaml @@ -0,0 +1,17 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + metrics: + enabled: true + service: + type: ClusterIP + podAnnotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scheme: http + prometheus.io/scrape: "true" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-configMapNamespace-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-configMapNamespace-values.yaml new file mode 100644 index 000000000..acd86a77a --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-configMapNamespace-values.yaml @@ -0,0 +1,20 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + tcp: + configMapNamespace: default + udp: + configMapNamespace: default + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-values.yaml new file mode 100644 index 000000000..25ee64d85 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-values.yaml @@ -0,0 +1,16 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-values.yaml new file mode 100644 index 000000000..380c8b4b1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-values.yaml @@ -0,0 +1,14 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-default-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-default-values.yaml new file mode 100644 index 000000000..82fa23e85 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-default-values.yaml @@ -0,0 +1,10 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-metrics-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-metrics-values.yaml new file mode 100644 index 000000000..cb3cb54be --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-metrics-values.yaml @@ -0,0 +1,12 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + metrics: + enabled: true + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-psp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-psp-values.yaml new file mode 100644 index 000000000..8026a6356 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-psp-values.yaml @@ -0,0 +1,13 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-and-psp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-and-psp-values.yaml new file mode 100644 index 000000000..fccdb134c --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-and-psp-values.yaml @@ -0,0 +1,13 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: true + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-values.yaml new file mode 100644 index 000000000..54d364df1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-values.yaml @@ -0,0 +1,10 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: true + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-behavior-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-behavior-values.yaml new file mode 100644 index 000000000..dca3f35f8 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-behavior-values.yaml @@ -0,0 +1,14 @@ +controller: + autoscaling: + enabled: true + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Pods + value: 1 + periodSeconds: 180 + admissionWebhooks: + enabled: false + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-values.yaml new file mode 100644 index 000000000..b8b3ac686 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-values.yaml @@ -0,0 +1,11 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + autoscaling: + enabled: true + admissionWebhooks: + enabled: false + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customconfig-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customconfig-values.yaml new file mode 100644 index 000000000..174941848 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customconfig-values.yaml @@ -0,0 +1,12 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + config: + use-proxy-protocol: "true" + allowSnippetAnnotations: false + admissionWebhooks: + enabled: false + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customnodeport-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customnodeport-values.yaml new file mode 100644 index 000000000..a564eaf93 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customnodeport-values.yaml @@ -0,0 +1,20 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: NodePort + nodePorts: + tcp: + 9000: 30090 + udp: + 9001: 30091 + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-default-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-default-values.yaml new file mode 100644 index 000000000..9f46b4e7e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-default-values.yaml @@ -0,0 +1,8 @@ +# Left blank to test default values +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-extra-modules.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-extra-modules.yaml new file mode 100644 index 000000000..ec5923548 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-extra-modules.yaml @@ -0,0 +1,10 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + service: + type: ClusterIP + extraModules: + - name: opentelemetry + image: busybox diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-headers-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-headers-values.yaml new file mode 100644 index 000000000..17a11ac37 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-headers-values.yaml @@ -0,0 +1,13 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + addHeaders: + X-Frame-Options: deny + proxySetHeaders: + X-Forwarded-Proto: https + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-internal-lb-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-internal-lb-values.yaml new file mode 100644 index 000000000..fd8df8de5 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-internal-lb-values.yaml @@ -0,0 +1,13 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + internal: + enabled: true + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: "true" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-metrics-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-metrics-values.yaml new file mode 100644 index 000000000..9209ad5a6 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-metrics-values.yaml @@ -0,0 +1,11 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + metrics: + enabled: true + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-nodeport-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-nodeport-values.yaml new file mode 100644 index 000000000..cd9b32352 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-nodeport-values.yaml @@ -0,0 +1,9 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: NodePort diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-podannotations-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-podannotations-values.yaml new file mode 100644 index 000000000..b48d93c46 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-podannotations-values.yaml @@ -0,0 +1,16 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + metrics: + enabled: true + service: + type: ClusterIP + podAnnotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scheme: http + prometheus.io/scrape: "true" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-psp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-psp-values.yaml new file mode 100644 index 000000000..2f332a7b2 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-psp-values.yaml @@ -0,0 +1,10 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-configMapNamespace-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-configMapNamespace-values.yaml new file mode 100644 index 000000000..c51a4e91f --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-configMapNamespace-values.yaml @@ -0,0 +1,19 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + tcp: + configMapNamespace: default + udp: + configMapNamespace: default + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-values.yaml new file mode 100644 index 000000000..5b45b69dc --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-values.yaml @@ -0,0 +1,15 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-values.yaml new file mode 100644 index 000000000..ac0b6e60e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-values.yaml @@ -0,0 +1,11 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-and-psp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-and-psp-values.yaml new file mode 100644 index 000000000..6195bb339 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-and-psp-values.yaml @@ -0,0 +1,12 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: true + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-resources-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-resources-values.yaml new file mode 100644 index 000000000..49ebbb02c --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-resources-values.yaml @@ -0,0 +1,23 @@ +controller: + service: + type: ClusterIP + admissionWebhooks: + enabled: true + createSecretJob: + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + patchWebhookJob: + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + patch: + enabled: true diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-values.yaml new file mode 100644 index 000000000..76669a530 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-values.yaml @@ -0,0 +1,9 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: true + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/NOTES.txt new file mode 100644 index 000000000..8985c56c0 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/NOTES.txt @@ -0,0 +1,80 @@ +The ingress-nginx controller has been installed. + +{{- if contains "NodePort" .Values.controller.service.type }} +Get the application URL by running these commands: + +{{- if (not (empty .Values.controller.service.nodePorts.http)) }} + export HTTP_NODE_PORT={{ .Values.controller.service.nodePorts.http }} +{{- else }} + export HTTP_NODE_PORT=$(kubectl --namespace {{ .Release.Namespace }} get services -o jsonpath="{.spec.ports[0].nodePort}" {{ include "ingress-nginx.controller.fullname" . }}) +{{- end }} +{{- if (not (empty .Values.controller.service.nodePorts.https)) }} + export HTTPS_NODE_PORT={{ .Values.controller.service.nodePorts.https }} +{{- else }} + export HTTPS_NODE_PORT=$(kubectl --namespace {{ .Release.Namespace }} get services -o jsonpath="{.spec.ports[1].nodePort}" {{ include "ingress-nginx.controller.fullname" . }}) +{{- end }} + export NODE_IP=$(kubectl --namespace {{ .Release.Namespace }} get nodes -o jsonpath="{.items[0].status.addresses[1].address}") + + echo "Visit http://$NODE_IP:$HTTP_NODE_PORT to access your application via HTTP." + echo "Visit https://$NODE_IP:$HTTPS_NODE_PORT to access your application via HTTPS." +{{- else if contains "LoadBalancer" .Values.controller.service.type }} +It may take a few minutes for the LoadBalancer IP to be available. +You can watch the status by running 'kubectl --namespace {{ .Release.Namespace }} get services -o wide -w {{ include "ingress-nginx.controller.fullname" . }}' +{{- else if contains "ClusterIP" .Values.controller.service.type }} +Get the application URL by running these commands: + export POD_NAME=$(kubectl --namespace {{ .Release.Namespace }} get pods -o jsonpath="{.items[0].metadata.name}" -l "app={{ template "ingress-nginx.name" . }},component={{ .Values.controller.name }},release={{ .Release.Name }}") + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:80 + echo "Visit http://127.0.0.1:8080 to access your application." +{{- end }} + +An example Ingress that makes use of the controller: + +{{- $isV1 := semverCompare ">=1" .Chart.AppVersion}} + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: example + namespace: foo + {{- if eq $isV1 false }} + annotations: + kubernetes.io/ingress.class: {{ .Values.controller.ingressClass }} + {{- end }} + spec: + {{- if $isV1 }} + ingressClassName: {{ .Values.controller.ingressClassResource.name }} + {{- end }} + rules: + - host: www.example.com + http: + paths: + - pathType: Prefix + backend: + service: + name: exampleService + port: + number: 80 + path: / + # This section is only required if TLS is to be enabled for the Ingress + tls: + - hosts: + - www.example.com + secretName: example-tls + +If TLS is enabled for the Ingress, a Secret containing the certificate and key must also be provided: + + apiVersion: v1 + kind: Secret + metadata: + name: example-tls + namespace: foo + data: + tls.crt: + tls.key: + type: kubernetes.io/tls + +{{- if .Values.controller.headers }} +################################################################################# +###### WARNING: `controller.headers` has been deprecated! ##### +###### It has been renamed to `controller.proxySetHeaders`. ##### +################################################################################# +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_helpers.tpl new file mode 100644 index 000000000..a72af5d9d --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_helpers.tpl @@ -0,0 +1,156 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "ingress-nginx.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ingress-nginx.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "ingress-nginx.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + + +{{/* +Container SecurityContext. +*/}} +{{- define "controller.containerSecurityContext" -}} +{{- if .Values.controller.containerSecurityContext -}} +{{- toYaml .Values.controller.containerSecurityContext -}} +{{- else -}} +capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE +runAsUser: {{ .Values.controller.image.runAsUser }} +allowPrivilegeEscalation: {{ .Values.controller.image.allowPrivilegeEscalation }} +{{- end }} +{{- end -}} + +{{/* +Create a default fully qualified controller name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "ingress-nginx.controller.fullname" -}} +{{- printf "%s-%s" (include "ingress-nginx.fullname" .) .Values.controller.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Construct the path for the publish-service. + +By convention this will simply use the / to match the name of the +service generated. + +Users can provide an override for an explicit service they want bound via `.Values.controller.publishService.pathOverride` + +*/}} +{{- define "ingress-nginx.controller.publishServicePath" -}} +{{- $defServiceName := printf "%s/%s" "$(POD_NAMESPACE)" (include "ingress-nginx.controller.fullname" .) -}} +{{- $servicePath := default $defServiceName .Values.controller.publishService.pathOverride }} +{{- print $servicePath | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified default backend name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "ingress-nginx.defaultBackend.fullname" -}} +{{- printf "%s-%s" (include "ingress-nginx.fullname" .) .Values.defaultBackend.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "ingress-nginx.labels" -}} +helm.sh/chart: {{ include "ingress-nginx.chart" . }} +{{ include "ingress-nginx.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/part-of: {{ template "ingress-nginx.name" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "ingress-nginx.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ingress-nginx.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the controller service account to use +*/}} +{{- define "ingress-nginx.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "ingress-nginx.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the backend service account to use - only used when podsecuritypolicy is also enabled +*/}} +{{- define "ingress-nginx.defaultBackend.serviceAccountName" -}} +{{- if .Values.defaultBackend.serviceAccount.create -}} + {{ default (printf "%s-backend" (include "ingress-nginx.fullname" .)) .Values.defaultBackend.serviceAccount.name }} +{{- else -}} + {{ default "default-backend" .Values.defaultBackend.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiGroup for PodSecurityPolicy. +*/}} +{{- define "podSecurityPolicy.apiGroup" -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "policy" -}} +{{- else -}} +{{- print "extensions" -}} +{{- end -}} +{{- end -}} + +{{/* +Check the ingress controller version tag is at most three versions behind the last release +*/}} +{{- define "isControllerTagValid" -}} +{{- if not (semverCompare ">=0.27.0-0" .Values.controller.image.tag) -}} +{{- fail "Controller container image tag should be 0.27.0 or higher" -}} +{{- end -}} +{{- end -}} + +{{/* +IngressClass parameters. +*/}} +{{- define "ingressClass.parameters" -}} + {{- if .Values.controller.ingressClassResource.parameters -}} + parameters: +{{ toYaml .Values.controller.ingressClassResource.parameters | indent 4}} + {{ end }} +{{- end -}} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_params.tpl b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_params.tpl new file mode 100644 index 000000000..305ce0dd2 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_params.tpl @@ -0,0 +1,62 @@ +{{- define "ingress-nginx.params" -}} +- /nginx-ingress-controller +{{- if .Values.defaultBackend.enabled }} +- --default-backend-service=$(POD_NAMESPACE)/{{ include "ingress-nginx.defaultBackend.fullname" . }} +{{- end }} +{{- if and .Values.controller.publishService.enabled .Values.controller.service.enabled }} +{{- if .Values.controller.service.external.enabled }} +- --publish-service={{ template "ingress-nginx.controller.publishServicePath" . }} +{{- else if .Values.controller.service.internal.enabled }} +- --publish-service={{ template "ingress-nginx.controller.publishServicePath" . }}-internal +{{- end }} +{{- end }} +- --election-id={{ .Values.controller.electionID }} +- --controller-class={{ .Values.controller.ingressClassResource.controllerValue }} +{{- if .Values.controller.ingressClass }} +- --ingress-class={{ .Values.controller.ingressClass }} +{{- end }} +- --configmap={{ default "$(POD_NAMESPACE)" .Values.controller.configMapNamespace }}/{{ include "ingress-nginx.controller.fullname" . }} +{{- if .Values.tcp }} +- --tcp-services-configmap={{ default "$(POD_NAMESPACE)" .Values.controller.tcp.configMapNamespace }}/{{ include "ingress-nginx.fullname" . }}-tcp +{{- end }} +{{- if .Values.udp }} +- --udp-services-configmap={{ default "$(POD_NAMESPACE)" .Values.controller.udp.configMapNamespace }}/{{ include "ingress-nginx.fullname" . }}-udp +{{- end }} +{{- if .Values.controller.scope.enabled }} +- --watch-namespace={{ default "$(POD_NAMESPACE)" .Values.controller.scope.namespace }} +{{- end }} +{{- if and (not .Values.controller.scope.enabled) .Values.controller.scope.namespaceSelector }} +- --watch-namespace-selector={{ default "" .Values.controller.scope.namespaceSelector }} +{{- end }} +{{- if and .Values.controller.reportNodeInternalIp .Values.controller.hostNetwork }} +- --report-node-internal-ip-address={{ .Values.controller.reportNodeInternalIp }} +{{- end }} +{{- if .Values.controller.admissionWebhooks.enabled }} +- --validating-webhook=:{{ .Values.controller.admissionWebhooks.port }} +- --validating-webhook-certificate={{ .Values.controller.admissionWebhooks.certificate }} +- --validating-webhook-key={{ .Values.controller.admissionWebhooks.key }} +{{- end }} +{{- if .Values.controller.maxmindLicenseKey }} +- --maxmind-license-key={{ .Values.controller.maxmindLicenseKey }} +{{- end }} +{{- if .Values.controller.healthCheckHost }} +- --healthz-host={{ .Values.controller.healthCheckHost }} +{{- end }} +{{- if not (eq .Values.controller.healthCheckPath "/healthz") }} +- --health-check-path={{ .Values.controller.healthCheckPath }} +{{- end }} +{{- if .Values.controller.ingressClassByName }} +- --ingress-class-by-name=true +{{- end }} +{{- if .Values.controller.watchIngressWithoutClass }} +- --watch-ingress-without-class=true +{{- end }} +{{- range $key, $value := .Values.controller.extraArgs }} +{{- /* Accept keys without values or with false as value */}} +{{- if eq ($value | quote | len) 2 }} +- --{{ $key }} +{{- else }} +- --{{ $key }}={{ $value }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml new file mode 100644 index 000000000..5659a1f10 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update +{{- if .Values.podSecurityPolicy.enabled }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + {{- with .Values.controller.admissionWebhooks.existingPsp }} + - {{ . }} + {{- else }} + - {{ include "ingress-nginx.fullname" . }}-admission + {{- end }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100644 index 000000000..abf17fb9f --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "ingress-nginx.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100644 index 000000000..f20e247f9 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,76 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission-create + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + {{- with .Values.controller.admissionWebhooks.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 +{{- end }} + template: + metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission-create + {{- if .Values.controller.admissionWebhooks.patch.podAnnotations }} + annotations: {{ toYaml .Values.controller.admissionWebhooks.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.controller.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.controller.admissionWebhooks.patch.priorityClassName }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + containers: + - name: create + {{- with .Values.controller.admissionWebhooks.patch.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + {{- end }} + imagePullPolicy: {{ .Values.controller.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{ include "ingress-nginx.controller.fullname" . }}-admission,{{ include "ingress-nginx.controller.fullname" . }}-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name={{ include "ingress-nginx.fullname" . }}-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + allowPrivilegeEscalation: false + {{- if .Values.controller.admissionWebhooks.createSecretJob.resources }} + resources: {{ toYaml .Values.controller.admissionWebhooks.createSecretJob.resources | nindent 12 }} + {{- end }} + restartPolicy: OnFailure + serviceAccountName: {{ include "ingress-nginx.fullname" . }}-admission + {{- if .Values.controller.admissionWebhooks.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.controller.admissionWebhooks.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.controller.admissionWebhooks.patch.tolerations }} + tolerations: {{ toYaml .Values.controller.admissionWebhooks.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.controller.admissionWebhooks.patch.runAsUser }} + fsGroup: {{ .Values.controller.admissionWebhooks.patch.fsGroup }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100644 index 000000000..8583685fa --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,78 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission-patch + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + {{- with .Values.controller.admissionWebhooks.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 +{{- end }} + template: + metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission-patch + {{- if .Values.controller.admissionWebhooks.patch.podAnnotations }} + annotations: {{ toYaml .Values.controller.admissionWebhooks.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.controller.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.controller.admissionWebhooks.patch.priorityClassName }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + containers: + - name: patch + {{- with .Values.controller.admissionWebhooks.patch.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + {{- end }} + imagePullPolicy: {{ .Values.controller.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ include "ingress-nginx.fullname" . }}-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name={{ include "ingress-nginx.fullname" . }}-admission + - --patch-failure-policy={{ .Values.controller.admissionWebhooks.failurePolicy }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + allowPrivilegeEscalation: false + {{- if .Values.controller.admissionWebhooks.patchWebhookJob.resources }} + resources: {{ toYaml .Values.controller.admissionWebhooks.patchWebhookJob.resources | nindent 12 }} + {{- end }} + restartPolicy: OnFailure + serviceAccountName: {{ include "ingress-nginx.fullname" . }}-admission + {{- if .Values.controller.admissionWebhooks.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.controller.admissionWebhooks.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.controller.admissionWebhooks.patch.tolerations }} + tolerations: {{ toYaml .Values.controller.admissionWebhooks.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.controller.admissionWebhooks.patch.runAsUser }} + fsGroup: {{ .Values.controller.admissionWebhooks.patch.fsGroup }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/psp.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 000000000..70edde334 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled .Values.podSecurityPolicy.enabled (empty .Values.controller.admissionWebhooks.existingPsp) -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + allowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + requiredDropCapabilities: + - ALL + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - emptyDir + - projected + - secret + - downwardAPI +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml new file mode 100644 index 000000000..795bac6b9 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml new file mode 100644 index 000000000..698c5c864 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "ingress-nginx.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100644 index 000000000..eae475118 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml new file mode 100644 index 000000000..8caffcb03 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml @@ -0,0 +1,48 @@ +{{- if .Values.controller.admissionWebhooks.enabled -}} +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + {{- if .Values.controller.admissionWebhooks.annotations }} + annotations: {{ toYaml .Values.controller.admissionWebhooks.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }}-admission +webhooks: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: {{ .Values.controller.admissionWebhooks.failurePolicy | default "Fail" }} + sideEffects: None + admissionReviewVersions: + - v1 + clientConfig: + service: + namespace: {{ .Release.Namespace | quote }} + name: {{ include "ingress-nginx.controller.fullname" . }}-admission + path: /networking/v1/ingresses + {{- if .Values.controller.admissionWebhooks.timeoutSeconds }} + timeoutSeconds: {{ .Values.controller.admissionWebhooks.timeoutSeconds }} + {{- end }} + {{- if .Values.controller.admissionWebhooks.namespaceSelector }} + namespaceSelector: {{ toYaml .Values.controller.admissionWebhooks.namespaceSelector | nindent 6 }} + {{- end }} + {{- if .Values.controller.admissionWebhooks.objectSelector }} + objectSelector: {{ toYaml .Values.controller.admissionWebhooks.objectSelector | nindent 6 }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrole.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrole.yaml new file mode 100644 index 000000000..c093f048a --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrole.yaml @@ -0,0 +1,87 @@ +{{- if .Values.rbac.create }} + +{{- if and .Values.rbac.scope (not .Values.controller.scope.enabled) -}} + {{ required "Invalid configuration: 'rbac.scope' should be equal to 'controller.scope.enabled' (true/false)." (index (dict) ".") }} +{{- end }} + +{{- if not .Values.rbac.scope -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }} +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets +{{- if not .Values.controller.scope.enabled }} + - namespaces +{{- end}} + verbs: + - list + - watch +{{- if and .Values.controller.scope.enabled .Values.controller.scope.namespace }} + - apiGroups: + - "" + resources: + - namespaces + resourceNames: + - "{{ .Values.controller.scope.namespace }}" + verbs: + - get +{{- end }} + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +{{- end }} + +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrolebinding.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..acbbd8b10 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create (not .Values.rbac.scope) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "ingress-nginx.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "ingress-nginx.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml new file mode 100644 index 000000000..dfd49a126 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml @@ -0,0 +1,14 @@ +{{- if .Values.controller.addHeaders -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }}-custom-add-headers + namespace: {{ .Release.Namespace }} +data: {{ toYaml .Values.controller.addHeaders | nindent 2 }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-proxyheaders.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-proxyheaders.yaml new file mode 100644 index 000000000..f8d15faf9 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-proxyheaders.yaml @@ -0,0 +1,19 @@ +{{- if or .Values.controller.proxySetHeaders .Values.controller.headers -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }}-custom-proxy-headers + namespace: {{ .Release.Namespace }} +data: +{{- if .Values.controller.proxySetHeaders }} +{{ toYaml .Values.controller.proxySetHeaders | indent 2 }} +{{ else if and .Values.controller.headers (not .Values.controller.proxySetHeaders) }} +{{ toYaml .Values.controller.headers | indent 2 }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-tcp.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-tcp.yaml new file mode 100644 index 000000000..0f6088ea9 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-tcp.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tcp -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.controller.tcp.annotations }} + annotations: {{ toYaml .Values.controller.tcp.annotations | nindent 4 }} +{{- end }} + name: {{ include "ingress-nginx.fullname" . }}-tcp + namespace: {{ .Release.Namespace }} +data: {{ tpl (toYaml .Values.tcp) . | nindent 2 }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-udp.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-udp.yaml new file mode 100644 index 000000000..3772ec514 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-udp.yaml @@ -0,0 +1,17 @@ +{{- if .Values.udp -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.controller.udp.annotations }} + annotations: {{ toYaml .Values.controller.udp.annotations | nindent 4 }} +{{- end }} + name: {{ include "ingress-nginx.fullname" . }}-udp + namespace: {{ .Release.Namespace }} +data: {{ tpl (toYaml .Values.udp) . | nindent 2 }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap.yaml new file mode 100644 index 000000000..f28b26e1e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.controller.configAnnotations }} + annotations: {{ toYaml .Values.controller.configAnnotations | nindent 4 }} +{{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} +data: + allow-snippet-annotations: "{{ .Values.controller.allowSnippetAnnotations }}" +{{- if .Values.controller.addHeaders }} + add-headers: {{ .Release.Namespace }}/{{ include "ingress-nginx.fullname" . }}-custom-add-headers +{{- end }} +{{- if or .Values.controller.proxySetHeaders .Values.controller.headers }} + proxy-set-headers: {{ .Release.Namespace }}/{{ include "ingress-nginx.fullname" . }}-custom-proxy-headers +{{- end }} +{{- if .Values.dhParam }} + ssl-dh-param: {{ printf "%s/%s" .Release.Namespace (include "ingress-nginx.controller.fullname" .) }} +{{- end }} +{{- range $key, $value := .Values.controller.config }} + {{- $key | nindent 2 }}: {{ $value | quote }} +{{- end }} + diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-daemonset.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-daemonset.yaml new file mode 100644 index 000000000..72811fbe4 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-daemonset.yaml @@ -0,0 +1,227 @@ +{{- if or (eq .Values.controller.kind "DaemonSet") (eq .Values.controller.kind "Both") -}} +{{- include "isControllerTagValid" . -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} + {{- if .Values.controller.annotations }} + annotations: {{ toYaml .Values.controller.annotations | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.controller.updateStrategy }} + updateStrategy: {{ toYaml .Values.controller.updateStrategy | nindent 4 }} + {{- end }} + minReadySeconds: {{ .Values.controller.minReadySeconds }} + template: + metadata: + {{- if .Values.controller.podAnnotations }} + annotations: + {{- range $key, $value := .Values.controller.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + labels: + {{- include "ingress-nginx.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.controller.podLabels }} + {{- toYaml .Values.controller.podLabels | nindent 8 }} + {{- end }} + spec: + {{- if .Values.controller.dnsConfig }} + dnsConfig: {{ toYaml .Values.controller.dnsConfig | nindent 8 }} + {{- end }} + {{- if .Values.controller.hostname }} + hostname: {{ toYaml .Values.controller.hostname | nindent 8 }} + {{- end }} + dnsPolicy: {{ .Values.controller.dnsPolicy }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if .Values.controller.priorityClassName }} + priorityClassName: {{ .Values.controller.priorityClassName }} + {{- end }} + {{- if or .Values.controller.podSecurityContext .Values.controller.sysctls }} + securityContext: + {{- end }} + {{- if .Values.controller.podSecurityContext }} + {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + {{- end }} + {{- if .Values.controller.sysctls }} + sysctls: + {{- range $sysctl, $value := .Values.controller.sysctls }} + - name: {{ $sysctl | quote }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + containers: + - name: {{ .Values.controller.containerName }} + {{- with .Values.controller.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + {{- end }} + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} + {{- if .Values.controller.lifecycle }} + lifecycle: {{ toYaml .Values.controller.lifecycle | nindent 12 }} + {{- end }} + args: + {{- include "ingress-nginx.params" . | nindent 12 }} + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: {{ .Values.controller.image.runAsUser }} + allowPrivilegeEscalation: {{ .Values.controller.image.allowPrivilegeEscalation }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.controller.enableMimalloc }} + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + {{- end }} + {{- if .Values.controller.extraEnvs }} + {{- toYaml .Values.controller.extraEnvs | nindent 12 }} + {{- end }} + {{- if .Values.controller.startupProbe }} + startupProbe: {{ toYaml .Values.controller.startupProbe | nindent 12 }} + {{- end }} + livenessProbe: {{ toYaml .Values.controller.livenessProbe | nindent 12 }} + readinessProbe: {{ toYaml .Values.controller.readinessProbe | nindent 12 }} + ports: + {{- range $key, $value := .Values.controller.containerPort }} + - name: {{ $key }} + containerPort: {{ $value }} + protocol: TCP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ index $.Values.controller.hostPort.ports $key | default $value }} + {{- end }} + {{- end }} + {{- if .Values.controller.metrics.enabled }} + - name: metrics + containerPort: {{ .Values.controller.metrics.port }} + protocol: TCP + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook + containerPort: {{ .Values.controller.admissionWebhooks.port }} + protocol: TCP + {{- end }} + {{- range $key, $value := .Values.tcp }} + - name: {{ $key }}-tcp + containerPort: {{ $key }} + protocol: TCP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ $key }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.udp }} + - name: {{ $key }}-udp + containerPort: {{ $key }} + protocol: UDP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ $key }} + {{- end }} + {{- end }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }} + volumeMounts: + {{- if .Values.controller.extraModules }} + - name: modules + mountPath: /modules_mount + {{- end }} + {{- if .Values.controller.customTemplate.configMapName }} + - mountPath: /etc/nginx/template + name: nginx-template-volume + readOnly: true + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + {{- end }} + {{- if .Values.controller.extraVolumeMounts }} + {{- toYaml .Values.controller.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.controller.resources }} + resources: {{ toYaml .Values.controller.resources | nindent 12 }} + {{- end }} + {{- if .Values.controller.extraContainers }} + {{ toYaml .Values.controller.extraContainers | nindent 8 }} + {{- end }} + + + {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }} + initContainers: + {{- if .Values.controller.extraInitContainers }} + {{ toYaml .Values.controller.extraInitContainers | nindent 8 }} + {{- end }} + {{- if .Values.controller.extraModules }} + {{- range .Values.controller.extraModules }} + - name: {{ .Name }} + image: {{ .Image }} + command: ['sh', '-c', '/usr/local/bin/init_module.sh'] + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.controller.hostNetwork }} + hostNetwork: {{ .Values.controller.hostNetwork }} + {{- end }} + {{- if .Values.controller.nodeSelector }} + nodeSelector: {{ toYaml .Values.controller.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.controller.tolerations }} + tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.controller.affinity }} + affinity: {{ toYaml .Values.controller.affinity | nindent 8 }} + {{- end }} + {{- if .Values.controller.topologySpreadConstraints }} + topologySpreadConstraints: {{ toYaml .Values.controller.topologySpreadConstraints | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }} + terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }} + volumes: + {{- if .Values.controller.extraModules }} + - name: modules + emptyDir: {} + {{- end }} + {{- if .Values.controller.customTemplate.configMapName }} + - name: nginx-template-volume + configMap: + name: {{ .Values.controller.customTemplate.configMapName }} + items: + - key: {{ .Values.controller.customTemplate.configMapKey }} + path: nginx.tmpl + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook-cert + secret: + secretName: {{ include "ingress-nginx.fullname" . }}-admission + {{- end }} + {{- if .Values.controller.extraVolumes }} + {{ toYaml .Values.controller.extraVolumes | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-deployment.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-deployment.yaml new file mode 100644 index 000000000..a1943cd91 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-deployment.yaml @@ -0,0 +1,225 @@ +{{- if or (eq .Values.controller.kind "Deployment") (eq .Values.controller.kind "Both") -}} +{{- include "isControllerTagValid" . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} + {{- if .Values.controller.annotations }} + annotations: {{ toYaml .Values.controller.annotations | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller + {{- if not .Values.controller.autoscaling.enabled }} + replicas: {{ .Values.controller.replicaCount }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.controller.updateStrategy }} + strategy: + {{ toYaml .Values.controller.updateStrategy | nindent 4 }} + {{- end }} + minReadySeconds: {{ .Values.controller.minReadySeconds }} + template: + metadata: + {{- if .Values.controller.podAnnotations }} + annotations: + {{- range $key, $value := .Values.controller.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + labels: + {{- include "ingress-nginx.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.controller.podLabels }} + {{- toYaml .Values.controller.podLabels | nindent 8 }} + {{- end }} + spec: + {{- if .Values.controller.dnsConfig }} + dnsConfig: {{ toYaml .Values.controller.dnsConfig | nindent 8 }} + {{- end }} + {{- if .Values.controller.hostname }} + hostname: {{ toYaml .Values.controller.hostname | nindent 8 }} + {{- end }} + dnsPolicy: {{ .Values.controller.dnsPolicy }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if .Values.controller.priorityClassName }} + priorityClassName: {{ .Values.controller.priorityClassName | quote }} + {{- end }} + {{- if or .Values.controller.podSecurityContext .Values.controller.sysctls }} + securityContext: + {{- end }} + {{- if .Values.controller.podSecurityContext }} + {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + {{- end }} + {{- if .Values.controller.sysctls }} + sysctls: + {{- range $sysctl, $value := .Values.controller.sysctls }} + - name: {{ $sysctl | quote }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + containers: + - name: {{ .Values.controller.containerName }} + {{- with .Values.controller.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + {{- end }} + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} + {{- if .Values.controller.lifecycle }} + lifecycle: {{ toYaml .Values.controller.lifecycle | nindent 12 }} + {{- end }} + args: + {{- include "ingress-nginx.params" . | nindent 12 }} + securityContext: {{ include "controller.containerSecurityContext" . | nindent 12 }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.controller.enableMimalloc }} + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + {{- end }} + {{- if .Values.controller.extraEnvs }} + {{- toYaml .Values.controller.extraEnvs | nindent 12 }} + {{- end }} + {{- if .Values.controller.startupProbe }} + startupProbe: {{ toYaml .Values.controller.startupProbe | nindent 12 }} + {{- end }} + livenessProbe: {{ toYaml .Values.controller.livenessProbe | nindent 12 }} + readinessProbe: {{ toYaml .Values.controller.readinessProbe | nindent 12 }} + ports: + {{- range $key, $value := .Values.controller.containerPort }} + - name: {{ $key }} + containerPort: {{ $value }} + protocol: TCP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ index $.Values.controller.hostPort.ports $key | default $value }} + {{- end }} + {{- end }} + {{- if .Values.controller.metrics.enabled }} + - name: metrics + containerPort: {{ .Values.controller.metrics.port }} + protocol: TCP + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook + containerPort: {{ .Values.controller.admissionWebhooks.port }} + protocol: TCP + {{- end }} + {{- range $key, $value := .Values.tcp }} + - name: {{ $key }}-tcp + containerPort: {{ $key }} + protocol: TCP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ $key }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.udp }} + - name: {{ $key }}-udp + containerPort: {{ $key }} + protocol: UDP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ $key }} + {{- end }} + {{- end }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }} + volumeMounts: + {{- if .Values.controller.extraModules }} + - name: modules + mountPath: /modules_mount + {{- end }} + {{- if .Values.controller.customTemplate.configMapName }} + - mountPath: /etc/nginx/template + name: nginx-template-volume + readOnly: true + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + {{- end }} + {{- if .Values.controller.extraVolumeMounts }} + {{- toYaml .Values.controller.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.controller.resources }} + resources: {{ toYaml .Values.controller.resources | nindent 12 }} + {{- end }} + {{- if .Values.controller.extraContainers }} + {{ toYaml .Values.controller.extraContainers | nindent 8 }} + {{- end }} + {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }} + initContainers: + {{- if .Values.controller.extraInitContainers }} + {{ toYaml .Values.controller.extraInitContainers | nindent 8 }} + {{- end }} + {{- if .Values.controller.extraModules }} + {{- range .Values.controller.extraModules }} + - name: {{ .name }} + image: {{ .image }} + command: ['sh', '-c', '/usr/local/bin/init_module.sh'] + volumeMounts: + - name: modules + mountPath: /modules_mount + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.controller.hostNetwork }} + hostNetwork: {{ .Values.controller.hostNetwork }} + {{- end }} + {{- if .Values.controller.nodeSelector }} + nodeSelector: {{ toYaml .Values.controller.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.controller.tolerations }} + tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.controller.affinity }} + affinity: {{ toYaml .Values.controller.affinity | nindent 8 }} + {{- end }} + {{- if .Values.controller.topologySpreadConstraints }} + topologySpreadConstraints: {{ toYaml .Values.controller.topologySpreadConstraints | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }} + terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }} + volumes: + {{- if .Values.controller.extraModules }} + - name: modules + emptyDir: {} + {{- end }} + {{- if .Values.controller.customTemplate.configMapName }} + - name: nginx-template-volume + configMap: + name: {{ .Values.controller.customTemplate.configMapName }} + items: + - key: {{ .Values.controller.customTemplate.configMapKey }} + path: nginx.tmpl + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook-cert + secret: + secretName: {{ include "ingress-nginx.fullname" . }}-admission + {{- end }} + {{- if .Values.controller.extraVolumes }} + {{ toYaml .Values.controller.extraVolumes | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-hpa.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-hpa.yaml new file mode 100644 index 000000000..e0979f14b --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-hpa.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.controller.autoscaling.enabled (or (eq .Values.controller.kind "Deployment") (eq .Values.controller.kind "Both")) -}} +{{- if not .Values.controller.keda.enabled }} + +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + annotations: + {{- with .Values.controller.autoscaling.annotations }} + {{- toYaml . | trimSuffix "\n" | nindent 4 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "ingress-nginx.controller.fullname" . }} + minReplicas: {{ .Values.controller.autoscaling.minReplicas }} + maxReplicas: {{ .Values.controller.autoscaling.maxReplicas }} + metrics: + {{- with .Values.controller.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.controller.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.controller.autoscalingTemplate }} + {{- toYaml . | nindent 2 }} + {{- end }} + {{- with .Values.controller.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} + diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-ingressclass.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-ingressclass.yaml new file mode 100644 index 000000000..9492784a2 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-ingressclass.yaml @@ -0,0 +1,21 @@ +{{- if .Values.controller.ingressClassResource.enabled -}} +# We don't support namespaced ingressClass yet +# So a ClusterRole and a ClusterRoleBinding is required +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ .Values.controller.ingressClassResource.name }} +{{- if .Values.controller.ingressClassResource.default }} + annotations: + ingressclass.kubernetes.io/is-default-class: "true" +{{- end }} +spec: + controller: {{ .Values.controller.ingressClassResource.controllerValue }} + {{ template "ingressClass.parameters" . }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-keda.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-keda.yaml new file mode 100644 index 000000000..875157ea4 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-keda.yaml @@ -0,0 +1,42 @@ +{{- if and .Values.controller.keda.enabled (or (eq .Values.controller.kind "Deployment") (eq .Values.controller.kind "Both")) -}} +# https://keda.sh/docs/ + +apiVersion: {{ .Values.controller.keda.apiVersion }} +kind: ScaledObject +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + {{- if .Values.controller.keda.scaledObject.annotations }} + annotations: {{ toYaml .Values.controller.keda.scaledObject.annotations | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: +{{- if eq .Values.controller.keda.apiVersion "keda.k8s.io/v1alpha1" }} + deploymentName: {{ include "ingress-nginx.controller.fullname" . }} +{{- else if eq .Values.controller.keda.apiVersion "keda.sh/v1alpha1" }} + name: {{ include "ingress-nginx.controller.fullname" . }} +{{- end }} + pollingInterval: {{ .Values.controller.keda.pollingInterval }} + cooldownPeriod: {{ .Values.controller.keda.cooldownPeriod }} + minReplicaCount: {{ .Values.controller.keda.minReplicas }} + maxReplicaCount: {{ .Values.controller.keda.maxReplicas }} + triggers: +{{- with .Values.controller.keda.triggers }} +{{ toYaml . | indent 2 }} +{{ end }} + advanced: + restoreToOriginalReplicaCount: {{ .Values.controller.keda.restoreToOriginalReplicaCount }} +{{- if .Values.controller.keda.behavior }} + horizontalPodAutoscalerConfig: + behavior: +{{ with .Values.controller.keda.behavior -}} +{{ toYaml . | indent 8 }} +{{ end }} + +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-poddisruptionbudget.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-poddisruptionbudget.yaml new file mode 100644 index 000000000..8dfbe9891 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-poddisruptionbudget.yaml @@ -0,0 +1,19 @@ +{{- if or (and .Values.controller.autoscaling.enabled (gt (.Values.controller.autoscaling.minReplicas | int) 1)) (and (not .Values.controller.autoscaling.enabled) (gt (.Values.controller.replicaCount | int) 1)) }} +apiVersion: {{ ternary "policy/v1" "policy/v1beta1" (semverCompare ">=1.21.0-0" .Capabilities.KubeVersion.Version) }} +kind: PodDisruptionBudget +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller + minAvailable: {{ .Values.controller.minAvailable }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-prometheusrules.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-prometheusrules.yaml new file mode 100644 index 000000000..ca5427523 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-prometheusrules.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.controller.metrics.enabled .Values.controller.metrics.prometheusRule.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "ingress-nginx.controller.fullname" . }} +{{- if .Values.controller.metrics.prometheusRule.namespace }} + namespace: {{ .Values.controller.metrics.prometheusRule.namespace | quote }} +{{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- if .Values.controller.metrics.prometheusRule.additionalLabels }} + {{- toYaml .Values.controller.metrics.prometheusRule.additionalLabels | nindent 4 }} + {{- end }} +spec: +{{- if .Values.controller.metrics.prometheusRule.rules }} + groups: + - name: {{ template "ingress-nginx.name" . }} + rules: {{- toYaml .Values.controller.metrics.prometheusRule.rules | nindent 4 }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-psp.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-psp.yaml new file mode 100644 index 000000000..a859594d1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-psp.yaml @@ -0,0 +1,89 @@ +{{- if and .Values.podSecurityPolicy.enabled (empty .Values.controller.existingPsp) -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ingress-nginx.fullname" . }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + allowedCapabilities: + - NET_BIND_SERVICE +{{- if .Values.controller.sysctls }} + allowedUnsafeSysctls: + {{- range $sysctl, $value := .Values.controller.sysctls }} + - {{ $sysctl }} + {{- end }} +{{- end }} + privileged: false + allowPrivilegeEscalation: true + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + #- 'projected' + - 'secret' + #- 'downwardAPI' +{{- if .Values.controller.hostNetwork }} + hostNetwork: {{ .Values.controller.hostNetwork }} +{{- end }} +{{- if or .Values.controller.hostNetwork .Values.controller.hostPort.enabled }} + hostPorts: +{{- if .Values.controller.hostNetwork }} +{{- range $key, $value := .Values.controller.containerPort }} + # {{ $key }} + - min: {{ $value }} + max: {{ $value }} +{{- end }} +{{- else if .Values.controller.hostPort.enabled }} +{{- range $key, $value := .Values.controller.hostPort.ports }} + # {{ $key }} + - min: {{ $value }} + max: {{ $value }} +{{- end }} +{{- end }} +{{- if .Values.controller.metrics.enabled }} + # metrics + - min: {{ .Values.controller.metrics.port }} + max: {{ .Values.controller.metrics.port }} +{{- end }} +{{- if .Values.controller.admissionWebhooks.enabled }} + # admission webhooks + - min: {{ .Values.controller.admissionWebhooks.port }} + max: {{ .Values.controller.admissionWebhooks.port }} +{{- end }} +{{- range $key, $value := .Values.tcp }} + # {{ $key }}-tcp + - min: {{ $key }} + max: {{ $key }} +{{- end }} +{{- range $key, $value := .Values.udp }} + # {{ $key }}-udp + - min: {{ $key }} + max: {{ $key }} +{{- end }} +{{- end }} + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'MustRunAsNonRoot' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + seLinux: + rule: 'RunAsAny' +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-role.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-role.yaml new file mode 100644 index 000000000..47bbc32d0 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-role.yaml @@ -0,0 +1,93 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }} + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - {{ .Values.controller.electionID }} + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +{{- if .Values.podSecurityPolicy.enabled }} + - apiGroups: [{{ template "podSecurityPolicy.apiGroup" . }}] + resources: ['podsecuritypolicies'] + verbs: ['use'] + {{- with .Values.controller.existingPsp }} + resourceNames: [{{ . }}] + {{- else }} + resourceNames: [{{ include "ingress-nginx.fullname" . }}] + {{- end }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-rolebinding.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-rolebinding.yaml new file mode 100644 index 000000000..e846a1183 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "ingress-nginx.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "ingress-nginx.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-internal.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-internal.yaml new file mode 100644 index 000000000..599449836 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-internal.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.controller.service.enabled .Values.controller.service.internal.enabled .Values.controller.service.internal.annotations}} +apiVersion: v1 +kind: Service +metadata: + annotations: + {{- range $key, $value := .Values.controller.service.internal.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- if .Values.controller.service.labels }} + {{- toYaml .Values.controller.service.labels | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }}-internal + namespace: {{ .Release.Namespace }} +spec: + type: "{{ .Values.controller.service.type }}" +{{- if .Values.controller.service.internal.loadBalancerIP }} + loadBalancerIP: {{ .Values.controller.service.internal.loadBalancerIP }} +{{- end }} +{{- if .Values.controller.service.internal.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ toYaml .Values.controller.service.internal.loadBalancerSourceRanges | nindent 4 }} +{{- end }} +{{- if .Values.controller.service.internal.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.controller.service.internal.externalTrafficPolicy }} +{{- end }} + ports: + {{- $setNodePorts := (or (eq .Values.controller.service.type "NodePort") (eq .Values.controller.service.type "LoadBalancer")) }} + {{- if .Values.controller.service.enableHttp }} + - name: http + port: {{ .Values.controller.service.ports.http }} + protocol: TCP + targetPort: {{ .Values.controller.service.targetPorts.http }} + {{- if semverCompare ">=1.20" .Capabilities.KubeVersion.Version }} + appProtocol: http + {{- end }} + {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.http))) }} + nodePort: {{ .Values.controller.service.nodePorts.http }} + {{- end }} + {{- end }} + {{- if .Values.controller.service.enableHttps }} + - name: https + port: {{ .Values.controller.service.ports.https }} + protocol: TCP + targetPort: {{ .Values.controller.service.targetPorts.https }} + {{- if semverCompare ">=1.20" .Capabilities.KubeVersion.Version }} + appProtocol: https + {{- end }} + {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.https))) }} + nodePort: {{ .Values.controller.service.nodePorts.https }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.tcp }} + - name: {{ $key }}-tcp + port: {{ $key }} + protocol: TCP + targetPort: {{ $key }}-tcp + {{- if $.Values.controller.service.nodePorts.tcp }} + {{- if index $.Values.controller.service.nodePorts.tcp $key }} + nodePort: {{ index $.Values.controller.service.nodePorts.tcp $key }} + {{- end }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.udp }} + - name: {{ $key }}-udp + port: {{ $key }} + protocol: UDP + targetPort: {{ $key }}-udp + {{- if $.Values.controller.service.nodePorts.udp }} + {{- if index $.Values.controller.service.nodePorts.udp $key }} + nodePort: {{ index $.Values.controller.service.nodePorts.udp $key }} + {{- end }} + {{- end }} + {{- end }} + selector: + {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-metrics.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-metrics.yaml new file mode 100644 index 000000000..0aaf41473 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-metrics.yaml @@ -0,0 +1,45 @@ +{{- if .Values.controller.metrics.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.controller.metrics.service.annotations }} + annotations: {{ toYaml .Values.controller.metrics.service.annotations | nindent 4 }} +{{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- if .Values.controller.metrics.service.labels }} + {{- toYaml .Values.controller.metrics.service.labels | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }}-metrics + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.controller.metrics.service.type }} +{{- if .Values.controller.metrics.service.clusterIP }} + clusterIP: {{ .Values.controller.metrics.service.clusterIP }} +{{- end }} +{{- if .Values.controller.metrics.service.externalIPs }} + externalIPs: {{ toYaml .Values.controller.metrics.service.externalIPs | nindent 4 }} +{{- end }} +{{- if .Values.controller.metrics.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.controller.metrics.service.loadBalancerIP }} +{{- end }} +{{- if .Values.controller.metrics.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ toYaml .Values.controller.metrics.service.loadBalancerSourceRanges | nindent 4 }} +{{- end }} +{{- if .Values.controller.metrics.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.controller.metrics.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: metrics + port: {{ .Values.controller.metrics.service.servicePort }} + protocol: TCP + targetPort: metrics + {{- $setNodePorts := (or (eq .Values.controller.metrics.service.type "NodePort") (eq .Values.controller.metrics.service.type "LoadBalancer")) }} + {{- if (and $setNodePorts (not (empty .Values.controller.metrics.service.nodePort))) }} + nodePort: {{ .Values.controller.metrics.service.nodePort }} + {{- end }} + selector: + {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-webhook.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-webhook.yaml new file mode 100644 index 000000000..2aae24fcf --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-webhook.yaml @@ -0,0 +1,40 @@ +{{- if .Values.controller.admissionWebhooks.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.controller.admissionWebhooks.service.annotations }} + annotations: {{ toYaml .Values.controller.admissionWebhooks.service.annotations | nindent 4 }} +{{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }}-admission + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.controller.admissionWebhooks.service.type }} +{{- if .Values.controller.admissionWebhooks.service.clusterIP }} + clusterIP: {{ .Values.controller.admissionWebhooks.service.clusterIP }} +{{- end }} +{{- if .Values.controller.admissionWebhooks.service.externalIPs }} + externalIPs: {{ toYaml .Values.controller.admissionWebhooks.service.externalIPs | nindent 4 }} +{{- end }} +{{- if .Values.controller.admissionWebhooks.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.controller.admissionWebhooks.service.loadBalancerIP }} +{{- end }} +{{- if .Values.controller.admissionWebhooks.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ toYaml .Values.controller.admissionWebhooks.service.loadBalancerSourceRanges | nindent 4 }} +{{- end }} + ports: + - name: https-webhook + port: 443 + targetPort: webhook + {{- if semverCompare ">=1.20" .Capabilities.KubeVersion.Version }} + appProtocol: https + {{- end }} + selector: + {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service.yaml new file mode 100644 index 000000000..05fb2041e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service.yaml @@ -0,0 +1,101 @@ +{{- if and .Values.controller.service.enabled .Values.controller.service.external.enabled -}} +apiVersion: v1 +kind: Service +metadata: + annotations: + {{- range $key, $value := .Values.controller.service.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- if .Values.controller.service.labels }} + {{- toYaml .Values.controller.service.labels | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.controller.service.type }} +{{- if .Values.controller.service.clusterIP }} + clusterIP: {{ .Values.controller.service.clusterIP }} +{{- end }} +{{- if .Values.controller.service.externalIPs }} + externalIPs: {{ toYaml .Values.controller.service.externalIPs | nindent 4 }} +{{- end }} +{{- if .Values.controller.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.controller.service.loadBalancerIP }} +{{- end }} +{{- if .Values.controller.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ toYaml .Values.controller.service.loadBalancerSourceRanges | nindent 4 }} +{{- end }} +{{- if .Values.controller.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.controller.service.externalTrafficPolicy }} +{{- end }} +{{- if .Values.controller.service.sessionAffinity }} + sessionAffinity: {{ .Values.controller.service.sessionAffinity }} +{{- end }} +{{- if .Values.controller.service.healthCheckNodePort }} + healthCheckNodePort: {{ .Values.controller.service.healthCheckNodePort }} +{{- end }} +{{- if semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version -}} +{{- if .Values.controller.service.ipFamilyPolicy }} + ipFamilyPolicy: {{ .Values.controller.service.ipFamilyPolicy }} +{{- end }} +{{- end }} +{{- if semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version -}} +{{- if .Values.controller.service.ipFamilies }} + ipFamilies: {{ toYaml .Values.controller.service.ipFamilies | nindent 4 }} +{{- end }} +{{- end }} + ports: + {{- $setNodePorts := (or (eq .Values.controller.service.type "NodePort") (eq .Values.controller.service.type "LoadBalancer")) }} + {{- if .Values.controller.service.enableHttp }} + - name: http + port: {{ .Values.controller.service.ports.http }} + protocol: TCP + targetPort: {{ .Values.controller.service.targetPorts.http }} + {{- if and (semverCompare ">=1.20" .Capabilities.KubeVersion.Version) (.Values.controller.service.appProtocol) }} + appProtocol: http + {{- end }} + {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.http))) }} + nodePort: {{ .Values.controller.service.nodePorts.http }} + {{- end }} + {{- end }} + {{- if .Values.controller.service.enableHttps }} + - name: https + port: {{ .Values.controller.service.ports.https }} + protocol: TCP + targetPort: {{ .Values.controller.service.targetPorts.https }} + {{- if and (semverCompare ">=1.20" .Capabilities.KubeVersion.Version) (.Values.controller.service.appProtocol) }} + appProtocol: https + {{- end }} + {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.https))) }} + nodePort: {{ .Values.controller.service.nodePorts.https }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.tcp }} + - name: {{ $key }}-tcp + port: {{ $key }} + protocol: TCP + targetPort: {{ $key }}-tcp + {{- if $.Values.controller.service.nodePorts.tcp }} + {{- if index $.Values.controller.service.nodePorts.tcp $key }} + nodePort: {{ index $.Values.controller.service.nodePorts.tcp $key }} + {{- end }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.udp }} + - name: {{ $key }}-udp + port: {{ $key }} + protocol: UDP + targetPort: {{ $key }}-udp + {{- if $.Values.controller.service.nodePorts.udp }} + {{- if index $.Values.controller.service.nodePorts.udp $key }} + nodePort: {{ index $.Values.controller.service.nodePorts.udp $key }} + {{- end }} + {{- end }} + {{- end }} + selector: + {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-serviceaccount.yaml new file mode 100644 index 000000000..824b2a124 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if or .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "ingress-nginx.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + {{- if .Values.serviceAccount.annotations }} + annotations: + {{ toYaml .Values.serviceAccount.annotations | indent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-servicemonitor.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-servicemonitor.yaml new file mode 100644 index 000000000..4dbc6da9f --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.controller.metrics.enabled .Values.controller.metrics.serviceMonitor.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "ingress-nginx.controller.fullname" . }} +{{- if .Values.controller.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.controller.metrics.serviceMonitor.namespace | quote }} +{{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- if .Values.controller.metrics.serviceMonitor.additionalLabels }} + {{- toYaml .Values.controller.metrics.serviceMonitor.additionalLabels | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: metrics + interval: {{ .Values.controller.metrics.serviceMonitor.scrapeInterval }} + {{- if .Values.controller.metrics.serviceMonitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if .Values.controller.metrics.serviceMonitor.relabelings }} + relabelings: {{ toYaml .Values.controller.metrics.serviceMonitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.controller.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: {{ toYaml .Values.controller.metrics.serviceMonitor.metricRelabelings | nindent 8 }} + {{- end }} +{{- if .Values.controller.metrics.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.controller.metrics.serviceMonitor.jobLabel | quote }} +{{- end }} +{{- if .Values.controller.metrics.serviceMonitor.namespaceSelector }} + namespaceSelector: {{ toYaml .Values.controller.metrics.serviceMonitor.namespaceSelector | nindent 4 }} +{{- else }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} +{{- if .Values.controller.metrics.serviceMonitor.targetLabels }} + targetLabels: + {{- range .Values.controller.metrics.serviceMonitor.targetLabels }} + - {{ . }} + {{- end }} +{{- end }} + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-deployment.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-deployment.yaml new file mode 100644 index 000000000..fd3e96e9e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-deployment.yaml @@ -0,0 +1,118 @@ +{{- if .Values.defaultBackend.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.defaultBackend.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: default-backend +{{- if not .Values.defaultBackend.autoscaling.enabled }} + replicas: {{ .Values.defaultBackend.replicaCount }} +{{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + template: + metadata: + {{- if .Values.defaultBackend.podAnnotations }} + annotations: {{ toYaml .Values.defaultBackend.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "ingress-nginx.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.defaultBackend.podLabels }} + {{- toYaml .Values.defaultBackend.podLabels | nindent 8 }} + {{- end }} + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if .Values.defaultBackend.priorityClassName }} + priorityClassName: {{ .Values.defaultBackend.priorityClassName }} + {{- end }} + {{- if .Values.defaultBackend.podSecurityContext }} + securityContext: {{ toYaml .Values.defaultBackend.podSecurityContext | nindent 8 }} + {{- end }} + containers: + - name: {{ template "ingress-nginx.name" . }}-default-backend + {{- with .Values.defaultBackend.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + {{- end }} + imagePullPolicy: {{ .Values.defaultBackend.image.pullPolicy }} + {{- if .Values.defaultBackend.extraArgs }} + args: + {{- range $key, $value := .Values.defaultBackend.extraArgs }} + {{- /* Accept keys without values or with false as value */}} + {{- if eq ($value | quote | len) 2 }} + - --{{ $key }} + {{- else }} + - --{{ $key }}={{ $value }} + {{- end }} + {{- end }} + {{- end }} + securityContext: + capabilities: + drop: + - ALL + runAsUser: {{ .Values.defaultBackend.image.runAsUser }} + runAsNonRoot: {{ .Values.defaultBackend.image.runAsNonRoot }} + allowPrivilegeEscalation: {{ .Values.defaultBackend.image.allowPrivilegeEscalation }} + readOnlyRootFilesystem: {{ .Values.defaultBackend.image.readOnlyRootFilesystem}} + {{- if .Values.defaultBackend.extraEnvs }} + env: {{ toYaml .Values.defaultBackend.extraEnvs | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.defaultBackend.port }} + scheme: HTTP + initialDelaySeconds: {{ .Values.defaultBackend.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.defaultBackend.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.defaultBackend.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.defaultBackend.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.defaultBackend.livenessProbe.failureThreshold }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.defaultBackend.port }} + scheme: HTTP + initialDelaySeconds: {{ .Values.defaultBackend.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.defaultBackend.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.defaultBackend.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.defaultBackend.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.defaultBackend.readinessProbe.failureThreshold }} + ports: + - name: http + containerPort: {{ .Values.defaultBackend.port }} + protocol: TCP + {{- if .Values.defaultBackend.extraVolumeMounts }} + volumeMounts: {{- toYaml .Values.defaultBackend.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.defaultBackend.resources }} + resources: {{ toYaml .Values.defaultBackend.resources | nindent 12 }} + {{- end }} + {{- if .Values.defaultBackend.nodeSelector }} + nodeSelector: {{ toYaml .Values.defaultBackend.nodeSelector | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "ingress-nginx.defaultBackend.serviceAccountName" . }} + {{- if .Values.defaultBackend.tolerations }} + tolerations: {{ toYaml .Values.defaultBackend.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.defaultBackend.affinity }} + affinity: {{ toYaml .Values.defaultBackend.affinity | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: 60 + {{- if .Values.defaultBackend.extraVolumes }} + volumes: {{ toYaml .Values.defaultBackend.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-hpa.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-hpa.yaml new file mode 100644 index 000000000..594d26525 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-hpa.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.defaultBackend.enabled .Values.defaultBackend.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "ingress-nginx.defaultBackend.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "ingress-nginx.defaultBackend.fullname" . }} + minReplicas: {{ .Values.defaultBackend.autoscaling.minReplicas }} + maxReplicas: {{ .Values.defaultBackend.autoscaling.maxReplicas }} + metrics: +{{- with .Values.defaultBackend.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ . }} +{{- end }} +{{- with .Values.defaultBackend.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ . }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-poddisruptionbudget.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-poddisruptionbudget.yaml new file mode 100644 index 000000000..00891cee5 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-poddisruptionbudget.yaml @@ -0,0 +1,21 @@ +{{- if .Values.defaultBackend.enabled -}} +{{- if or (gt (.Values.defaultBackend.replicaCount | int) 1) (gt (.Values.defaultBackend.autoscaling.minReplicas | int) 1) }} +apiVersion: {{ ternary "policy/v1" "policy/v1beta1" (semverCompare ">=1.21.0-0" .Capabilities.KubeVersion.Version) }} +kind: PodDisruptionBudget +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.defaultBackend.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: default-backend + minAvailable: {{ .Values.defaultBackend.minAvailable }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-psp.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-psp.yaml new file mode 100644 index 000000000..42061c5d3 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-psp.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.podSecurityPolicy.enabled .Values.defaultBackend.enabled (empty .Values.defaultBackend.existingPsp) -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ingress-nginx.fullname" . }}-backend + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + allowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + requiredDropCapabilities: + - ALL + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - emptyDir + - projected + - secret + - downwardAPI +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-role.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-role.yaml new file mode 100644 index 000000000..a2b457c36 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-role.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.rbac.create .Values.podSecurityPolicy.enabled .Values.defaultBackend.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }}-backend + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: [{{ template "podSecurityPolicy.apiGroup" . }}] + resources: ['podsecuritypolicies'] + verbs: ['use'] + {{- with .Values.defaultBackend.existingPsp }} + resourceNames: [{{ . }}] + {{- else }} + resourceNames: [{{ include "ingress-nginx.fullname" . }}-backend] + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-rolebinding.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-rolebinding.yaml new file mode 100644 index 000000000..dbaa516b9 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.podSecurityPolicy.enabled .Values.defaultBackend.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }}-backend + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "ingress-nginx.fullname" . }}-backend +subjects: + - kind: ServiceAccount + name: {{ template "ingress-nginx.defaultBackend.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-service.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-service.yaml new file mode 100644 index 000000000..5f1d09a95 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-service.yaml @@ -0,0 +1,41 @@ +{{- if .Values.defaultBackend.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.defaultBackend.service.annotations }} + annotations: {{ toYaml .Values.defaultBackend.service.annotations | nindent 4 }} +{{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.defaultBackend.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.defaultBackend.service.type }} +{{- if .Values.defaultBackend.service.clusterIP }} + clusterIP: {{ .Values.defaultBackend.service.clusterIP }} +{{- end }} +{{- if .Values.defaultBackend.service.externalIPs }} + externalIPs: {{ toYaml .Values.defaultBackend.service.externalIPs | nindent 4 }} +{{- end }} +{{- if .Values.defaultBackend.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.defaultBackend.service.loadBalancerIP }} +{{- end }} +{{- if .Values.defaultBackend.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ toYaml .Values.defaultBackend.service.loadBalancerSourceRanges | nindent 4 }} +{{- end }} + ports: + - name: http + port: {{ .Values.defaultBackend.service.servicePort }} + protocol: TCP + targetPort: http + {{- if semverCompare ">=1.20" .Capabilities.KubeVersion.Version }} + appProtocol: http + {{- end }} + selector: + {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: default-backend +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml new file mode 100644 index 000000000..b45a95ad2 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.defaultBackend.enabled .Values.defaultBackend.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "ingress-nginx.defaultBackend.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +automountServiceAccountToken: {{ .Values.defaultBackend.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/dh-param-secret.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/dh-param-secret.yaml new file mode 100644 index 000000000..12e7a4f63 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/dh-param-secret.yaml @@ -0,0 +1,10 @@ +{{- with .Values.dhParam -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "ingress-nginx.controller.fullname" $ }} + labels: + {{- include "ingress-nginx.labels" $ | nindent 4 }} +data: + dhparam.pem: {{ . }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/values.yaml new file mode 100644 index 000000000..c887dc051 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/values.yaml @@ -0,0 +1,919 @@ +## nginx configuration +## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/index.md +## + +## Overrides for generated resource names +# See templates/_helpers.tpl +# nameOverride: +# fullnameOverride: + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +controller: + name: controller + image: + registry: k8s.gcr.io + image: ingress-nginx/controller + ## for backwards compatibility consider setting the full image url via the repository value below + ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + ## repository: + tag: "v1.1.2" + digest: sha256:28b11ce69e57843de44e3db6413e98d09de0f6688e33d4bd384002a44f78405c + pullPolicy: IfNotPresent + # www-data -> uid 101 + runAsUser: 101 + allowPrivilegeEscalation: true + + # -- Use an existing PSP instead of creating one + existingPsp: "" + + # -- Configures the controller container name + containerName: controller + + # -- Configures the ports that the nginx-controller listens on + containerPort: + http: 80 + https: 443 + + # -- Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/ + config: {} + + # -- Annotations to be added to the controller config configuration configmap. + configAnnotations: {} + + # -- Will add custom headers before sending traffic to backends according to https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/custom-headers + proxySetHeaders: {} + + # -- Will add custom headers before sending response traffic to the client according to: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#add-headers + addHeaders: {} + + # -- Optionally customize the pod dnsConfig. + dnsConfig: {} + + # -- Optionally customize the pod hostname. + hostname: {} + + # -- Optionally change this to ClusterFirstWithHostNet in case you have 'hostNetwork: true'. + # By default, while using host network, name resolution uses the host's DNS. If you wish nginx-controller + # to keep resolving names inside the k8s network, use ClusterFirstWithHostNet. + dnsPolicy: ClusterFirst + + # -- Bare-metal considerations via the host network https://kubernetes.github.io/ingress-nginx/deploy/baremetal/#via-the-host-network + # Ingress status was blank because there is no Service exposing the NGINX Ingress controller in a configuration using the host network, the default --publish-service flag used in standard cloud setups does not apply + reportNodeInternalIp: false + + # -- Process Ingress objects without ingressClass annotation/ingressClassName field + # Overrides value for --watch-ingress-without-class flag of the controller binary + # Defaults to false + watchIngressWithoutClass: false + + # -- Process IngressClass per name (additionally as per spec.controller). + ingressClassByName: false + + # -- This configuration defines if Ingress Controller should allow users to set + # their own *-snippet annotations, otherwise this is forbidden / dropped + # when users add those annotations. + # Global snippets in ConfigMap are still respected + allowSnippetAnnotations: true + + # -- Required for use with CNI based kubernetes installations (such as ones set up by kubeadm), + # since CNI and hostport don't mix yet. Can be deprecated once https://github.com/kubernetes/kubernetes/issues/23920 + # is merged + hostNetwork: false + + ## Use host ports 80 and 443 + ## Disabled by default + hostPort: + # -- Enable 'hostPort' or not + enabled: false + ports: + # -- 'hostPort' http port + http: 80 + # -- 'hostPort' https port + https: 443 + + # -- Election ID to use for status update + electionID: ingress-controller-leader + + ## This section refers to the creation of the IngressClass resource + ## IngressClass resources are supported since k8s >= 1.18 and required since k8s >= 1.19 + ingressClassResource: + # -- Name of the ingressClass + name: nginx + # -- Is this ingressClass enabled or not + enabled: true + # -- Is this the default ingressClass for the cluster + default: false + # -- Controller-value of the controller that is processing this ingressClass + controllerValue: "k8s.io/ingress-nginx" + + # -- Parameters is a link to a custom resource containing additional + # configuration for the controller. This is optional if the controller + # does not require extra parameters. + parameters: {} + + # -- For backwards compatibility with ingress.class annotation, use ingressClass. + # Algorithm is as follows, first ingressClassName is considered, if not present, controller looks for ingress.class annotation + ingressClass: nginx + + # -- Labels to add to the pod container metadata + podLabels: {} + # key: value + + # -- Security Context policies for controller pods + podSecurityContext: {} + + # -- See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for notes on enabling and using sysctls + sysctls: {} + # sysctls: + # "net.core.somaxconn": "8192" + + # -- Allows customization of the source of the IP address or FQDN to report + # in the ingress status field. By default, it reads the information provided + # by the service. If disable, the status field reports the IP address of the + # node or nodes where an ingress controller pod is running. + publishService: + # -- Enable 'publishService' or not + enabled: true + # -- Allows overriding of the publish service to bind to + # Must be / + pathOverride: "" + + # Limit the scope of the controller to a specific namespace + scope: + # -- Enable 'scope' or not + enabled: false + # -- Namespace to limit the controller to; defaults to $(POD_NAMESPACE) + namespace: "" + # -- When scope.enabled == false, instead of watching all namespaces, we watching namespaces whose labels + # only match with namespaceSelector. Format like foo=bar. Defaults to empty, means watching all namespaces. + namespaceSelector: "" + + # -- Allows customization of the configmap / nginx-configmap namespace; defaults to $(POD_NAMESPACE) + configMapNamespace: "" + + tcp: + # -- Allows customization of the tcp-services-configmap; defaults to $(POD_NAMESPACE) + configMapNamespace: "" + # -- Annotations to be added to the tcp config configmap + annotations: {} + + udp: + # -- Allows customization of the udp-services-configmap; defaults to $(POD_NAMESPACE) + configMapNamespace: "" + # -- Annotations to be added to the udp config configmap + annotations: {} + + # -- Maxmind license key to download GeoLite2 Databases. + ## https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases + maxmindLicenseKey: "" + + # -- Additional command line arguments to pass to nginx-ingress-controller + # E.g. to specify the default SSL certificate you can use + extraArgs: {} + ## extraArgs: + ## default-ssl-certificate: "/" + + # -- Additional environment variables to set + extraEnvs: [] + # extraEnvs: + # - name: FOO + # valueFrom: + # secretKeyRef: + # key: FOO + # name: secret-resource + + # -- Use a `DaemonSet` or `Deployment` + kind: Deployment + + # -- Annotations to be added to the controller Deployment or DaemonSet + ## + annotations: {} + # keel.sh/pollSchedule: "@every 60m" + + # -- Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels + ## + labels: {} + # keel.sh/policy: patch + # keel.sh/trigger: poll + + + # -- The update strategy to apply to the Deployment or DaemonSet + ## + updateStrategy: {} + # rollingUpdate: + # maxUnavailable: 1 + # type: RollingUpdate + + # -- `minReadySeconds` to avoid killing pods before we are ready + ## + minReadySeconds: 0 + + + # -- Node tolerations for server scheduling to nodes with taints + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- Affinity and anti-affinity rules for server scheduling to nodes + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + # # An example of preferred pod anti-affinity, weight is in the range 1-100 + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight: 100 + # podAffinityTerm: + # labelSelector: + # matchExpressions: + # - key: app.kubernetes.io/name + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/instance + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/component + # operator: In + # values: + # - controller + # topologyKey: kubernetes.io/hostname + + # # An example of required pod anti-affinity + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: app.kubernetes.io/name + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/instance + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/component + # operator: In + # values: + # - controller + # topologyKey: "kubernetes.io/hostname" + + # -- Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: failure-domain.beta.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app.kubernetes.io/instance: ingress-nginx-internal + + # -- `terminationGracePeriodSeconds` to avoid killing pods before we are ready + ## wait up to five minutes for the drain of connections + ## + terminationGracePeriodSeconds: 300 + + # -- Node labels for controller pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: + kubernetes.io/os: linux + + ## Liveness and readiness probe values + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes + ## + ## startupProbe: + ## httpGet: + ## # should match container.healthCheckPath + ## path: "/healthz" + ## port: 10254 + ## scheme: HTTP + ## initialDelaySeconds: 5 + ## periodSeconds: 5 + ## timeoutSeconds: 2 + ## successThreshold: 1 + ## failureThreshold: 5 + livenessProbe: + httpGet: + # should match container.healthCheckPath + path: "/healthz" + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + # should match container.healthCheckPath + path: "/healthz" + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + + + # -- Path of the health check endpoint. All requests received on the port defined by + # the healthz-port parameter are forwarded internally to this path. + healthCheckPath: "/healthz" + + # -- Address to bind the health check endpoint. + # It is better to set this option to the internal node address + # if the ingress nginx controller is running in the `hostNetwork: true` mode. + healthCheckHost: "" + + # -- Annotations to be added to controller pods + ## + podAnnotations: {} + + replicaCount: 1 + + minAvailable: 1 + + ## Define requests resources to avoid probe issues due to CPU utilization in busy nodes + ## ref: https://github.com/kubernetes/ingress-nginx/issues/4735#issuecomment-551204903 + ## Ideally, there should be no limits. + ## https://engineering.indeedblog.com/blog/2019/12/cpu-throttling-regression-fix/ + resources: + ## limits: + ## cpu: 100m + ## memory: 90Mi + requests: + cpu: 100m + memory: 90Mi + + # Mutually exclusive with keda autoscaling + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 11 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 + behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + + autoscalingTemplate: [] + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + # - type: Pods + # pods: + # metric: + # name: nginx_ingress_controller_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m + + # Mutually exclusive with hpa autoscaling + keda: + apiVersion: "keda.sh/v1alpha1" + ## apiVersion changes with keda 1.x vs 2.x + ## 2.x = keda.sh/v1alpha1 + ## 1.x = keda.k8s.io/v1alpha1 + enabled: false + minReplicas: 1 + maxReplicas: 11 + pollingInterval: 30 + cooldownPeriod: 300 + restoreToOriginalReplicaCount: false + scaledObject: + annotations: {} + # Custom annotations for ScaledObject resource + # annotations: + # key: value + triggers: [] + # - type: prometheus + # metadata: + # serverAddress: http://:9090 + # metricName: http_requests_total + # threshold: '100' + # query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + + behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + + # -- Enable mimalloc as a drop-in replacement for malloc. + ## ref: https://github.com/microsoft/mimalloc + ## + enableMimalloc: true + + ## Override NGINX template + customTemplate: + configMapName: "" + configMapKey: "" + + service: + enabled: true + + # -- If enabled is adding an appProtocol option for Kubernetes service. An appProtocol field replacing annotations that were + # using for setting a backend protocol. Here is an example for AWS: service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http + # It allows choosing the protocol for each backend specified in the Kubernetes service. + # See the following GitHub issue for more details about the purpose: https://github.com/kubernetes/kubernetes/issues/40244 + # Will be ignored for Kubernetes versions older than 1.20 + ## + appProtocol: true + + annotations: {} + labels: {} + # clusterIP: "" + + # -- List of IP addresses at which the controller services are available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + + enableHttp: true + enableHttps: true + + ## Set external traffic policy to: "Local" to preserve source IP on providers supporting it. + ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer + # externalTrafficPolicy: "" + + ## Must be either "None" or "ClientIP" if set. Kubernetes will default to "None". + ## Ref: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + # sessionAffinity: "" + + ## Specifies the health check node port (numeric port number) for the service. If healthCheckNodePort isn’t specified, + ## the service controller allocates a port from your cluster’s NodePort range. + ## Ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + # healthCheckNodePort: 0 + + # -- Represents the dual-stack-ness requested or required by this Service. Possible values are + # SingleStack, PreferDualStack or RequireDualStack. + # The ipFamilies and clusterIPs fields depend on the value of this field. + ## Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/ + ipFamilyPolicy: "SingleStack" + + # -- List of IP families (e.g. IPv4, IPv6) assigned to the service. This field is usually assigned automatically + # based on cluster configuration and the ipFamilyPolicy field. + ## Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/ + ipFamilies: + - IPv4 + + ports: + http: 80 + https: 443 + + targetPorts: + http: http + https: https + + type: LoadBalancer + + ## type: NodePort + ## nodePorts: + ## http: 32080 + ## https: 32443 + ## tcp: + ## 8080: 32808 + nodePorts: + http: "" + https: "" + tcp: {} + udp: {} + + external: + enabled: true + + internal: + # -- Enables an additional internal load balancer (besides the external one). + enabled: false + # -- Annotations are mandatory for the load balancer to come up. Varies with the cloud service. + annotations: {} + + # loadBalancerIP: "" + + # -- Restrict access For LoadBalancer service. Defaults to 0.0.0.0/0. + loadBalancerSourceRanges: [] + + ## Set external traffic policy to: "Local" to preserve source IP on + ## providers supporting it + ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer + # externalTrafficPolicy: "" + + # -- Additional containers to be added to the controller pod. + # See https://github.com/lemonldap-ng-controller/lemonldap-ng-controller as example. + extraContainers: [] + # - name: my-sidecar + # image: nginx:latest + # - name: lemonldap-ng-controller + # image: lemonldapng/lemonldap-ng-controller:0.2.0 + # args: + # - /lemonldap-ng-controller + # - --alsologtostderr + # - --configmap=$(POD_NAMESPACE)/lemonldap-ng-configuration + # env: + # - name: POD_NAME + # valueFrom: + # fieldRef: + # fieldPath: metadata.name + # - name: POD_NAMESPACE + # valueFrom: + # fieldRef: + # fieldPath: metadata.namespace + # volumeMounts: + # - name: copy-portal-skins + # mountPath: /srv/var/lib/lemonldap-ng/portal/skins + + # -- Additional volumeMounts to the controller main container. + extraVolumeMounts: [] + # - name: copy-portal-skins + # mountPath: /var/lib/lemonldap-ng/portal/skins + + # -- Additional volumes to the controller pod. + extraVolumes: [] + # - name: copy-portal-skins + # emptyDir: {} + + # -- Containers, which are run before the app containers are started. + extraInitContainers: [] + # - name: init-myservice + # image: busybox + # command: ['sh', '-c', 'until nslookup myservice; do echo waiting for myservice; sleep 2; done;'] + + extraModules: [] + ## Modules, which are mounted into the core nginx image + # - name: opentelemetry + # image: busybox + # + # The image must contain a `/usr/local/bin/init_module.sh` executable, which + # will be executed as initContainers, to move its config files within the + # mounted volume. + + admissionWebhooks: + annotations: {} + # ignore-check.kube-linter.io/no-read-only-rootfs: "This deployment needs write access to root filesystem". + + ## Additional annotations to the admission webhooks. + ## These annotations will be added to the ValidatingWebhookConfiguration and + ## the Jobs Spec of the admission webhooks. + enabled: true + failurePolicy: Fail + # timeoutSeconds: 10 + port: 8443 + certificate: "/usr/local/certificates/cert" + key: "/usr/local/certificates/key" + namespaceSelector: {} + objectSelector: {} + # -- Labels to be added to admission webhooks + labels: {} + + # -- Use an existing PSP instead of creating one + existingPsp: "" + + service: + annotations: {} + # clusterIP: "" + externalIPs: [] + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 443 + type: ClusterIP + + createSecretJob: + resources: {} + # limits: + # cpu: 10m + # memory: 20Mi + # requests: + # cpu: 10m + # memory: 20Mi + + patchWebhookJob: + resources: {} + + patch: + enabled: true + image: + registry: k8s.gcr.io + image: ingress-nginx/kube-webhook-certgen + ## for backwards compatibility consider setting the full image url via the repository value below + ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + ## repository: + tag: v1.1.1 + digest: sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660 + pullPolicy: IfNotPresent + # -- Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + podAnnotations: {} + nodeSelector: + kubernetes.io/os: linux + tolerations: [] + # -- Labels to be added to patch job resources + labels: {} + runAsUser: 2000 + fsGroup: 2000 + + metrics: + port: 10254 + # if this port is changed, change healthz-port: in extraArgs: accordingly + enabled: false + + service: + annotations: {} + # prometheus.io/scrape: "true" + # prometheus.io/port: "10254" + + # clusterIP: "" + + # -- List of IP addresses at which the stats-exporter service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 10254 + type: ClusterIP + # externalTrafficPolicy: "" + # nodePort: "" + + serviceMonitor: + enabled: false + additionalLabels: {} + ## The label to use to retrieve the job name from. + ## jobLabel: "app.kubernetes.io/name" + namespace: "" + namespaceSelector: {} + ## Default: scrape .Release.Namespace only + ## To scrape all, use the following: + ## namespaceSelector: + ## any: true + scrapeInterval: 30s + # honorLabels: true + targetLabels: [] + relabelings: [] + metricRelabelings: [] + + prometheusRule: + enabled: false + additionalLabels: {} + # namespace: "" + rules: [] + # # These are just examples rules, please adapt them to your needs + # - alert: NGINXConfigFailed + # expr: count(nginx_ingress_controller_config_last_reload_successful == 0) > 0 + # for: 1s + # labels: + # severity: critical + # annotations: + # description: bad ingress config - nginx config test failed + # summary: uninstall the latest ingress changes to allow config reloads to resume + # - alert: NGINXCertificateExpiry + # expr: (avg(nginx_ingress_controller_ssl_expire_time_seconds) by (host) - time()) < 604800 + # for: 1s + # labels: + # severity: critical + # annotations: + # description: ssl certificate(s) will expire in less then a week + # summary: renew expiring certificates to avoid downtime + # - alert: NGINXTooMany500s + # expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5 + # for: 1m + # labels: + # severity: warning + # annotations: + # description: Too many 5XXs + # summary: More than 5% of all requests returned 5XX, this requires your attention + # - alert: NGINXTooMany400s + # expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"4.+"} ) / sum(nginx_ingress_controller_requests) ) > 5 + # for: 1m + # labels: + # severity: warning + # annotations: + # description: Too many 4XXs + # summary: More than 5% of all requests returned 4XX, this requires your attention + + # -- Improve connection draining when ingress controller pod is deleted using a lifecycle hook: + # With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds + # to 300, allowing the draining of connections up to five minutes. + # If the active connections end before that, the pod will terminate gracefully at that time. + # To effectively take advantage of this feature, the Configmap feature + # worker-shutdown-timeout new value is 240s instead of 10s. + ## + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + + priorityClassName: "" + +# -- Rollback limit +## +revisionHistoryLimit: 10 + +## Default 404 backend +## +defaultBackend: + ## + enabled: false + + name: defaultbackend + image: + registry: k8s.gcr.io + image: defaultbackend-amd64 + ## for backwards compatibility consider setting the full image url via the repository value below + ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + ## repository: + tag: "1.5" + pullPolicy: IfNotPresent + # nobody user -> uid 65534 + runAsUser: 65534 + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + + # -- Use an existing PSP instead of creating one + existingPsp: "" + + extraArgs: {} + + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + # -- Additional environment variables to set for defaultBackend pods + extraEnvs: [] + + port: 8080 + + ## Readiness and liveness probes for default backend + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ + ## + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + failureThreshold: 6 + initialDelaySeconds: 0 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + + # -- Node tolerations for server scheduling to nodes with taints + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + affinity: {} + + # -- Security Context policies for controller pods + # See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for + # notes on enabling and using sysctls + ## + podSecurityContext: {} + + # -- Security Context policies for controller main container. + # See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for + # notes on enabling and using sysctls + ## + containerSecurityContext: {} + + # -- Labels to add to the pod container metadata + podLabels: {} + # key: value + + # -- Node labels for default backend pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: + kubernetes.io/os: linux + + # -- Annotations to be added to default backend pods + ## + podAnnotations: {} + + replicaCount: 1 + + minAvailable: 1 + + resources: {} + # limits: + # cpu: 10m + # memory: 20Mi + # requests: + # cpu: 10m + # memory: 20Mi + + extraVolumeMounts: [] + ## Additional volumeMounts to the default backend container. + # - name: copy-portal-skins + # mountPath: /var/lib/lemonldap-ng/portal/skins + + extraVolumes: [] + ## Additional volumes to the default backend pod. + # - name: copy-portal-skins + # emptyDir: {} + + autoscaling: + annotations: {} + enabled: false + minReplicas: 1 + maxReplicas: 2 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 + + service: + annotations: {} + + # clusterIP: "" + + # -- List of IP addresses at which the default backend service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 80 + type: ClusterIP + + priorityClassName: "" + # -- Labels to be added to the default backend resources + labels: {} + +## Enable RBAC as per https://github.com/kubernetes/ingress-nginx/blob/main/docs/deploy/rbac.md and https://github.com/kubernetes/ingress-nginx/issues/266 +rbac: + create: true + scope: false + +## If true, create & use Pod Security Policy resources +## https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +podSecurityPolicy: + enabled: false + +serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + # -- Annotations for the controller service account + annotations: {} + +# -- Optional array of imagePullSecrets containing private registry credentials +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# - name: secretName + +# -- TCP service key:value pairs +## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md +## +tcp: {} +# 8080: "default/example-tcp-svc:9000" + +# -- UDP service key:value pairs +## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md +## +udp: {} +# 53: "kube-system/kube-dns:53" + +# -- (string) A base64-encoded Diffie-Hellman parameter. +# This can be generated with: `openssl dhparam 4096 2> /dev/null | base64` +## Ref: https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/ssl-dh-param +dhParam: diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml deleted file mode 100644 index 8aa1dcf82..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: nginx-ingress -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -AppVersion: "v1.5.4" diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt deleted file mode 120000 index 12e23824a..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt +++ /dev/null @@ -1 +0,0 @@ -../../../files/site.crt \ No newline at end of file diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key deleted file mode 120000 index 3805a27d1..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key +++ /dev/null @@ -1 +0,0 @@ -../../../files/site.key \ No newline at end of file diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt deleted file mode 100644 index 8125afe9c..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "nginx-ingress.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "nginx-ingress.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "nginx-ingress.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "nginx-ingress.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl deleted file mode 100644 index b0f2808f1..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "nginx-ingress.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "nginx-ingress.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "nginx-ingress.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "nginx-ingress.labels" -}} -helm.sh/chart: {{ include "nginx-ingress.chart" . }} -{{ include "nginx-ingress.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "nginx-ingress.selectorLabels" -}} -app.kubernetes.io/name: {{ include "nginx-ingress.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "nginx-ingress.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "nginx-ingress.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml deleted file mode 100644 index 6322318e3..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml +++ /dev/null @@ -1,190 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx - namespace: {{ .Release.Namespace }} -data: - location.list: |- - location ~* /general_stats { - deny all; - } - location /healthz { - return 200 'OK'; - } - location ~ ^/(mobs|sessions-assets|frontend|static|sourcemaps|ios-images)/ { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $origin_forwarded_ip; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - - proxy_connect_timeout 300; - # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 - proxy_http_version 1.1; - proxy_set_header Connection ""; - chunked_transfer_encoding off; - - proxy_pass http://minio.db.svc.cluster.local:9000; - } - - location /minio/ { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_pass http://minio.db.svc.cluster.local:9000; - } - location /ingest/ { - rewrite ^/ingest/(.*) /$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header X-Forwarded-For $origin_forwarded_ip; - proxy_set_header X-Forwarded-Host $origin_forwarded_ip; - proxy_set_header X-Real-IP $origin_forwarded_ip; - proxy_set_header Host $host; - proxy_pass http://http-openreplay.app.svc.cluster.local; - proxy_read_timeout 300; - proxy_connect_timeout 120; - proxy_send_timeout 300; - } - location /grafana { - set $target http://monitoring-grafana.monitoring.svc.cluster.local; - rewrite ^/grafana/(.*) /$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_pass $target; - } - location /api/ { - rewrite ^/api/(.*) /$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $origin_proto; - proxy_pass http://chalice-openreplay.app.svc.cluster.local:8000; - } - location /assist/ { - rewrite ^/assist/(.*) /$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $origin_forwarded_ip; - proxy_pass http://utilities-openreplay.app.svc.cluster.local:9000; - } - location /ws-assist/ { - rewrite ^/ws-assist/(.*) /$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $origin_forwarded_ip; - proxy_set_header X-Real-IP $origin_forwarded_ip; - proxy_pass http://utilities-openreplay.app.svc.cluster.local:9001; - } - location /assets/ { - rewrite ^/assets/(.*) /sessions-assets/$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_pass http://minio.db.svc.cluster.local:9000; - } - location / { - index /index.html; - rewrite ^((?!.(js|css|png|svg|jpg|woff|woff2)).)*$ /frontend/index.html break; - proxy_http_version 1.1; - proxy_set_header Connection ""; - include /etc/nginx/conf.d/compression.conf; - proxy_set_header Host $http_host; - proxy_pass http://minio.db.svc.cluster.local:9000/frontend/; - proxy_intercept_errors on; # see http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_intercept_errors - error_page 404 =200 /index.html; - } - compression.conf: |- - # Compression - gzip on; - gzip_comp_level 5; - gzip_min_length 256; # 256Bytes - gzip_proxied any; - gzip_vary on; - # Content types for compression - gzip_types - application/atom+xml - application/javascript - application/json - application/ld+json - application/manifest+json - application/rss+xml - application/vnd.geo+json - application/vnd.ms-fontobject - application/x-font-ttf - application/x-web-app-manifest+json - application/xhtml+xml - application/xml - font/opentype - image/bmp - image/svg+xml - image/x-icon - text/cache-manifest - text/css - text/plain - ; - - sites.conf: |- - # Ref: https://github.com/openresty/openresty/#resolvconf-parsing - resolver local=on; - # Need real ip address for flags in replay. - # Some LBs will forward real ips as x-forwarded-for - # So making that as priority - map $http_x_forwarded_for $origin_forwarded_ip { - ~^(\d+\.\d+\.\d+\.\d+) $1; - default $remote_addr; - } - map $http_upgrade $connection_upgrade { - default upgrade; - '' close; - } - map $http_x_forwarded_proto $origin_proto { - default $http_x_forwarded_proto; - '' $scheme; - } - # Default server for helath check - server { - listen 80 default_server; - listen [::]:80; - location /healthz { - return 200 'OK'; - } - } - - upstream utilities-pool { - # comment this line if you have multiple pods - server utilities-openreplay-headless.app:9001; - # uncomment and change these lines if you have multiple pods - # server POD1_IP:9001; - # server POD2_IP:9001; - hash $origin_forwarded_ip consistent; - } - - server { - listen 80; - listen [::]:80; - server_name {{ .Values.global.domainName }}; - {{ .Values.customServerConfigs }} - include /etc/nginx/conf.d/location.list; - client_max_body_size 10M; - } - server { - listen 443 ssl; - server_name {{ .Values.global.domainName }}; - ssl_certificate /etc/secrets/site.crt; - ssl_certificate_key /etc/secrets/site.key; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA HIGH !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; - include /etc/nginx/conf.d/location.list; - client_max_body_size 10M; - } - diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml deleted file mode 100644 index 95c05aa67..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "nginx-ingress.fullname" . }} - labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "nginx-ingress.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - nginxRolloutID: {{ randAlphaNum 5 | quote }} # Restart nginx after every deployment - {{- with .Values.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "nginx-ingress.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "nginx-ingress.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: https - containerPort: 443 - protocol: TCP - - name: http - containerPort: 80 - protocol: TCP - - name: metrics - containerPort: 9145 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: http - readinessProbe: - httpGet: - path: /healthz - port: http - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: nginx - mountPath: /etc/nginx/conf.d/ - - name: ssl - mountPath: /etc/secrets/ - volumes: - - name: nginx - configMap: - name: nginx - - name: ssl - secret: - secretName: ssl - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml deleted file mode 100644 index 348f8f95b..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "nginx-ingress.fullname" . }} - labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "nginx-ingress.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml deleted file mode 100644 index 63cfce077..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "nginx-ingress.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml deleted file mode 100644 index 91b7cc09c..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: ssl -data: - ca.crt: '' - site.crt: '{{ .Files.Get "files/site.crt" | b64enc }}' - site.key: '{{ .Files.Get "files/site.key" | b64enc }}' diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml deleted file mode 100644 index f20d4fc38..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "nginx-ingress.fullname" . }} - labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} - annotations: - {{- with .Values.service.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.service.type }} - {{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort")}} - # Make sure to get client ip - externalTrafficPolicy: Local - {{- end}} - ports: - {{- range .Values.service.ports }} - - port: {{ .port }} - targetPort: {{ .targetPort }} - protocol: TCP - name: {{ .name }} - {{- end }} - selector: - {{- include "nginx-ingress.selectorLabels" . | nindent 4 }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "nginx-ingress.fullname" . }}-metrics - labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} - openreplay/monitoring: nginx-ingress-metrics -spec: - selector: - {{- include "nginx-ingress.selectorLabels" . | nindent 4 }} - ports: - - name: metrics - port: 9145 - targetPort: 9145 - protocol: TCP diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml deleted file mode 100644 index bc0091029..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "nginx-ingress.serviceAccountName" . }} - labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml deleted file mode 100644 index 074cec518..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "nginx-ingress.fullname" . }}-test-connection" - labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "nginx-ingress.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml deleted file mode 100644 index 1f7169c0d..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# Default values for nginx-ingress. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: nginx - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - annotations: {} - type: LoadBalancer - ports: - - port: 80 - targetPort: http - name: http - - port: 443 - targetPort: https - name: https - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -healthProbes: - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: http - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: http - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml index 567cac846..e17247411 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml @@ -1,61 +1,43 @@ -{{- if .Values.ingress.enabled -}} +{{- if .Values.ingress.enabled }} {{- $fullName := include "utilities.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +{{- $peerjsSvcPort := .Values.service.ports.peerjs -}} +{{- $socketioSvcPort := .Values.service.ports.socketio -}} apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} kind: Ingress metadata: name: {{ $fullName }} labels: {{- include "utilities.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/upstream-hash-by: $http_x_forwarded_for + {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} + ingressClassName: "{{ tpl .Values.ingress.className . }}" tls: - {{- range .Values.ingress.tls }} - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} + - host: {{ .Values.global.domainName }} http: paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} + - pathType: Prefix backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: name: {{ $fullName }} port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} + number: {{ $peerjsSvcPort }} + path: /assist/(.*) + - pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ $socketioSvcPort }} + path: /ws-assist/(.*) {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/utilities/values.yaml b/scripts/helmcharts/openreplay/charts/utilities/values.yaml index 8265b5aa5..3a252996c 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/values.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/values.yaml @@ -47,20 +47,13 @@ service: socketio: 9001 ingress: - enabled: false - className: "" - annotations: {} + enabled: true + className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}" + annotations: # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local + tls: + secretName: openreplay-ssl resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 24ea9344d..9c72d0d3a 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -41,8 +41,29 @@ minio: accessKey: "changeMeMinioAccessKey" secretKey: "changeMeMinioPassword" +ingress-nginx: &ingress-nginx + controller: + ingressClassResource: + # -- Name of the ingressClass + name: openreplay + # -- For backwards compatibility with ingress.class annotation, use ingressClass. + # Algorithm is as follows, first ingressClassName is considered, if not present, controller looks for ingress.class annotation + ingressClass: openreplay + service: + externalTrafficPolicy: "Local" + extraArgs: + default-ssl-certificate: "app/openreplay-ssl" + config: + enable-real-ip: true + forwarded-for-header: "proxy_protocol" + # If you want the ssl offload in LB layer + # Uncomment the following line + # ssl-redirect: false + # force-ssl-redirect: false + # Application specific variables global: + ingress: *ingress-nginx postgresql: *postgres kafka: *kafka redis: *redis From bd36b7c86422e264a8a1a7d8bd279644477f12e5 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 12:15:28 +0200 Subject: [PATCH 109/125] fix(helm): utilities annotation --- scripts/helmcharts/openreplay/charts/utilities/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/charts/utilities/values.yaml b/scripts/helmcharts/openreplay/charts/utilities/values.yaml index 3a252996c..5696ce401 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/values.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/values.yaml @@ -49,7 +49,7 @@ service: ingress: enabled: true className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}" - annotations: + annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" tls: From c48bc3e19744c56d798e7c0f2b7362aa666b35c9 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 14 Apr 2022 13:26:56 +0200 Subject: [PATCH 110/125] feat(api): dashboard fixed get_domains_errors/get_domains_errors_4xx/get_domains_errors_5xx wrong neutral --- api/chalicelib/core/dashboard.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index 719fabb3a..e5bc8003e 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -1,11 +1,12 @@ +import math + import schemas from chalicelib.core import metadata from chalicelib.utils import args_transformer -from chalicelib.utils import helper, dev +from chalicelib.utils import helper from chalicelib.utils import pg_client from chalicelib.utils.TimeUTC import TimeUTC from chalicelib.utils.metrics_helper import __get_step_size -import math # Written by David Aznaurov, inspired by numpy.quantile @@ -75,8 +76,6 @@ METADATA_FIELDS = {"userId": "user_id", "metadata9": "metadata_9", "metadata10": "metadata_10"} -from chalicelib.core import sessions_metas - def __get_meta_constraint(project_id, data): if len(data.get("filters", [])) == 0: @@ -127,6 +126,13 @@ SESSIONS_META_FIELDS = {"revId": "rev_id", "browser": "user_browser"} +def __get_domains_errors_neutral(rows): + neutral = {l: 0 for l in [i for k in [list(v.keys()) for v in rows] for i in k]} + if len(neutral.keys()) == 0: + neutral = {"All": 0} + return neutral + + def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -1591,7 +1597,7 @@ def get_crashes(project_id, startTimestamp=TimeUTC.now(delta_days=-1), def __get_neutral(rows, add_All_if_empty=True): neutral = {l: 0 for l in [i for k in [list(v.keys()) for v in rows] for i in k]} - if add_All_if_empty and len(neutral.keys()) == 0: + if add_All_if_empty and len(neutral.keys()) <= 1: neutral = {"All": 0} return neutral From 1f1a27797bb02482c4eca390214cd9654d134550 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Thu, 14 Apr 2022 11:56:24 +0000 Subject: [PATCH 111/125] Splitting utilities service to peers and assist (#427) * chore(helm): splitting utilities 1. peers -> handle peerjs connections 2. assist -> handle ws connections Signed-off-by: rjshrjndrn * build(utilities): rename utilities to assist Signed-off-by: rjshrjndrn * chore(build_deploy): include peers Signed-off-by: rjshrjndrn --- scripts/helm/build_deploy.sh | 2 + .../charts/{utilities => assist}/.helmignore | 0 .../charts/{utilities => assist}/Chart.yaml | 6 +- .../{utilities => assist}/templates/NOTES.txt | 8 +- .../templates/_helpers.tpl | 20 ++--- .../templates/deployment.yaml | 10 +-- .../{utilities => assist}/templates/hpa.yaml | 6 +- .../templates/ingress.yaml | 4 +- .../templates/service.yaml | 12 +-- .../templates/serviceaccount.yaml | 4 +- .../charts/{utilities => assist}/values.yaml | 6 +- .../openreplay/charts/peers/.helmignore | 23 +++++ .../openreplay/charts/peers/Chart.yaml | 24 ++++++ .../charts/peers/templates/NOTES.txt | 22 +++++ .../charts/peers/templates/_helpers.tpl | 62 ++++++++++++++ .../charts/peers/templates/deployment.yaml | 66 +++++++++++++++ .../charts/peers/templates/hpa.yaml | 28 +++++++ .../charts/peers/templates/ingress.yaml | 34 ++++++++ .../charts/peers/templates/service.yaml | 37 +++++++++ .../peers/templates/serviceaccount.yaml | 12 +++ .../templates/tests/test-connection.yaml | 15 ++++ .../openreplay/charts/peers/values.yaml | 83 +++++++++++++++++++ .../templates/tests/test-connection.yaml | 15 ---- utilities/build.sh | 8 +- 24 files changed, 450 insertions(+), 57 deletions(-) rename scripts/helmcharts/openreplay/charts/{utilities => assist}/.helmignore (100%) rename scripts/helmcharts/openreplay/charts/{utilities => assist}/Chart.yaml (81%) rename scripts/helmcharts/openreplay/charts/{utilities => assist}/templates/NOTES.txt (77%) rename scripts/helmcharts/openreplay/charts/{utilities => assist}/templates/_helpers.tpl (75%) rename scripts/helmcharts/openreplay/charts/{utilities => assist}/templates/deployment.yaml (88%) rename scripts/helmcharts/openreplay/charts/{utilities => assist}/templates/hpa.yaml (84%) rename scripts/helmcharts/openreplay/charts/{utilities => assist}/templates/ingress.yaml (92%) rename scripts/helmcharts/openreplay/charts/{utilities => assist}/templates/service.yaml (63%) rename scripts/helmcharts/openreplay/charts/{utilities => assist}/templates/serviceaccount.yaml (66%) rename scripts/helmcharts/openreplay/charts/{utilities => assist}/values.yaml (94%) create mode 100644 scripts/helmcharts/openreplay/charts/peers/.helmignore create mode 100644 scripts/helmcharts/openreplay/charts/peers/Chart.yaml create mode 100644 scripts/helmcharts/openreplay/charts/peers/templates/NOTES.txt create mode 100644 scripts/helmcharts/openreplay/charts/peers/templates/_helpers.tpl create mode 100644 scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml create mode 100644 scripts/helmcharts/openreplay/charts/peers/templates/hpa.yaml create mode 100644 scripts/helmcharts/openreplay/charts/peers/templates/ingress.yaml create mode 100644 scripts/helmcharts/openreplay/charts/peers/templates/service.yaml create mode 100644 scripts/helmcharts/openreplay/charts/peers/templates/serviceaccount.yaml create mode 100644 scripts/helmcharts/openreplay/charts/peers/templates/tests/test-connection.yaml create mode 100644 scripts/helmcharts/openreplay/charts/peers/values.yaml delete mode 100644 scripts/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml diff --git a/scripts/helm/build_deploy.sh b/scripts/helm/build_deploy.sh index 86925ebe8..7a75fad8b 100644 --- a/scripts/helm/build_deploy.sh +++ b/scripts/helm/build_deploy.sh @@ -17,4 +17,6 @@ echo $DOCKER_REPO PUSH_IMAGE=1 bash build.sh $@ cd ../utilities PUSH_IMAGE=1 bash build.sh $@ + cd ../peers + PUSH_IMAGE=1 bash build.sh $@ } diff --git a/scripts/helmcharts/openreplay/charts/utilities/.helmignore b/scripts/helmcharts/openreplay/charts/assist/.helmignore similarity index 100% rename from scripts/helmcharts/openreplay/charts/utilities/.helmignore rename to scripts/helmcharts/openreplay/charts/assist/.helmignore diff --git a/scripts/helmcharts/openreplay/charts/utilities/Chart.yaml b/scripts/helmcharts/openreplay/charts/assist/Chart.yaml similarity index 81% rename from scripts/helmcharts/openreplay/charts/utilities/Chart.yaml rename to scripts/helmcharts/openreplay/charts/assist/Chart.yaml index 203871558..0bcde4fd3 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/Chart.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: utilities +name: assist description: A Helm chart for Kubernetes # A chart can be either an 'application' or a 'library' chart. @@ -7,8 +7,8 @@ description: A Helm chart for Kubernetes # Application charts are a collection of templates that can be packaged into versioned archives # to be deployed. # -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering +# Library charts provide useful assist or functions for the chart developer. They're included as +# a dependency of application charts to inject those assist and functions into the rendering # pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/assist/templates/NOTES.txt similarity index 77% rename from scripts/helmcharts/openreplay/charts/utilities/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/assist/templates/NOTES.txt index 323bc20c1..57c8efcf4 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/NOTES.txt +++ b/scripts/helmcharts/openreplay/charts/assist/templates/NOTES.txt @@ -6,16 +6,16 @@ {{- end }} {{- end }} {{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "utilities.fullname" . }}) + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "assist.fullname" . }}) export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo http://$NODE_IP:$NODE_PORT {{- else if contains "LoadBalancer" .Values.service.type }} NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "utilities.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "utilities.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "assist.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "assist.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") echo http://$SERVICE_IP:{{ .Values.service.port }} {{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "utilities.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "assist.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") echo "Visit http://127.0.0.1:8080 to use your application" kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/assist/templates/_helpers.tpl similarity index 75% rename from scripts/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/assist/templates/_helpers.tpl index 8999db4be..95cab7e2b 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl +++ b/scripts/helmcharts/openreplay/charts/assist/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "utilities.name" -}} +{{- define "assist.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "utilities.fullname" -}} +{{- define "assist.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "utilities.chart" -}} +{{- define "assist.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "utilities.labels" -}} -helm.sh/chart: {{ include "utilities.chart" . }} -{{ include "utilities.selectorLabels" . }} +{{- define "assist.labels" -}} +helm.sh/chart: {{ include "assist.chart" . }} +{{ include "assist.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "utilities.selectorLabels" -}} -app.kubernetes.io/name: {{ include "utilities.name" . }} +{{- define "assist.selectorLabels" -}} +app.kubernetes.io/name: {{ include "assist.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Create the name of the service account to use */}} -{{- define "utilities.serviceAccountName" -}} +{{- define "assist.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} -{{- default (include "utilities.fullname" .) .Values.serviceAccount.name }} +{{- default (include "assist.fullname" .) .Values.serviceAccount.name }} {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml similarity index 88% rename from scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml index 7a3d5b0b6..be98db206 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml @@ -1,16 +1,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "utilities.fullname" . }} + name: {{ include "assist.fullname" . }} labels: - {{- include "utilities.labels" . | nindent 4 }} + {{- include "assist.labels" . | nindent 4 }} spec: {{- if not .Values.autoscaling.enabled }} replicas: {{ .Values.replicaCount }} {{- end }} selector: matchLabels: - {{- include "utilities.selectorLabels" . | nindent 6 }} + {{- include "assist.selectorLabels" . | nindent 6 }} template: metadata: {{- with .Values.podAnnotations }} @@ -18,13 +18,13 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "utilities.selectorLabels" . | nindent 8 }} + {{- include "assist.selectorLabels" . | nindent 8 }} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "utilities.serviceAccountName" . }} + serviceAccountName: {{ include "assist.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/hpa.yaml similarity index 84% rename from scripts/helmcharts/openreplay/charts/utilities/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/assist/templates/hpa.yaml index 8944056ea..b0b394a20 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/hpa.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/hpa.yaml @@ -2,14 +2,14 @@ apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: - name: {{ include "utilities.fullname" . }} + name: {{ include "assist.fullname" . }} labels: - {{- include "utilities.labels" . | nindent 4 }} + {{- include "assist.labels" . | nindent 4 }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ include "utilities.fullname" . }} + name: {{ include "assist.fullname" . }} minReplicas: {{ .Values.autoscaling.minReplicas }} maxReplicas: {{ .Values.autoscaling.maxReplicas }} metrics: diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml similarity index 92% rename from scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml index e17247411..48e568298 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml @@ -1,5 +1,5 @@ {{- if .Values.ingress.enabled }} -{{- $fullName := include "utilities.fullname" . -}} +{{- $fullName := include "assist.fullname" . -}} {{- $peerjsSvcPort := .Values.service.ports.peerjs -}} {{- $socketioSvcPort := .Values.service.ports.socketio -}} apiVersion: networking.k8s.io/v1 @@ -7,7 +7,7 @@ kind: Ingress metadata: name: {{ $fullName }} labels: - {{- include "utilities.labels" . | nindent 4 }} + {{- include "assist.labels" . | nindent 4 }} annotations: nginx.ingress.kubernetes.io/rewrite-target: /$1 nginx.ingress.kubernetes.io/upstream-hash-by: $http_x_forwarded_for diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/service.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/service.yaml similarity index 63% rename from scripts/helmcharts/openreplay/charts/utilities/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/assist/templates/service.yaml index 8b1dc8753..699d9d149 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/service.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/service.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "utilities.fullname" . }} + name: {{ include "assist.fullname" . }} labels: - {{- include "utilities.labels" . | nindent 4 }} + {{- include "assist.labels" . | nindent 4 }} spec: type: {{ .Values.service.type }} ports: @@ -14,15 +14,15 @@ spec: name: {{ $key }} {{- end}} selector: - {{- include "utilities.selectorLabels" . | nindent 4 }} + {{- include "assist.selectorLabels" . | nindent 4 }} --- apiVersion: v1 kind: Service metadata: - name: {{ include "utilities.fullname" . }}-headless + name: {{ include "assist.fullname" . }}-headless labels: - {{- include "utilities.labels" . | nindent 4 }} + {{- include "assist.labels" . | nindent 4 }} spec: type: ClusterIP clusterIP: None @@ -34,4 +34,4 @@ spec: name: {{ $key }} {{- end}} selector: - {{- include "utilities.selectorLabels" . | nindent 4 }} + {{- include "assist.selectorLabels" . | nindent 4 }} diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/serviceaccount.yaml similarity index 66% rename from scripts/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/assist/templates/serviceaccount.yaml index dd5c35012..18e86a3e8 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/serviceaccount.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "utilities.serviceAccountName" . }} + name: {{ include "assist.serviceAccountName" . }} labels: - {{- include "utilities.labels" . | nindent 4 }} + {{- include "assist.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/scripts/helmcharts/openreplay/charts/utilities/values.yaml b/scripts/helmcharts/openreplay/charts/assist/values.yaml similarity index 94% rename from scripts/helmcharts/openreplay/charts/utilities/values.yaml rename to scripts/helmcharts/openreplay/charts/assist/values.yaml index 5696ce401..861d715f9 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/values.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/values.yaml @@ -5,14 +5,14 @@ replicaCount: 1 image: - repository: rg.fr-par.scw.cloud/foss/utilities + repository: rg.fr-par.scw.cloud/foss/assist pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "" imagePullSecrets: [] -nameOverride: "utililities" -fullnameOverride: "utililities" +nameOverride: "assist" +fullnameOverride: "assist" serviceAccount: # Specifies whether a service account should be created diff --git a/scripts/helmcharts/openreplay/charts/peers/.helmignore b/scripts/helmcharts/openreplay/charts/peers/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helmcharts/openreplay/charts/peers/Chart.yaml b/scripts/helmcharts/openreplay/charts/peers/Chart.yaml new file mode 100644 index 000000000..b482f5986 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: peers +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful peers or functions for the chart developer. They're included as +# a dependency of application charts to inject those peers and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +AppVersion: "v1.5.4" diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/peers/templates/NOTES.txt new file mode 100644 index 000000000..fbe8dd585 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "peers.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "peers.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "peers.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "peers.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/peers/templates/_helpers.tpl new file mode 100644 index 000000000..9dce56cdd --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "peers.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "peers.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "peers.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "peers.labels" -}} +helm.sh/chart: {{ include "peers.chart" . }} +{{ include "peers.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "peers.selectorLabels" -}} +app.kubernetes.io/name: {{ include "peers.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "peers.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "peers.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml new file mode 100644 index 000000000..8ad4364d8 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "peers.fullname" . }} + labels: + {{- include "peers.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "peers.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "peers.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "peers.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: S3_KEY + value: {{ .Values.global.s3.accessKey }} + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} + ports: + {{- range $key, $val := .Values.service.ports }} + - name: {{ $key }} + containerPort: {{ $val }} + protocol: TCP + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/hpa.yaml new file mode 100644 index 000000000..7191631ee --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "peers.fullname" . }} + labels: + {{- include "peers.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "peers.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/ingress.yaml new file mode 100644 index 000000000..0da5eca0d --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/templates/ingress.yaml @@ -0,0 +1,34 @@ +{{- if .Values.ingress.enabled }} +{{- $fullName := include "peers.fullname" . -}} +{{- $peerjsSvcPort := .Values.service.ports.peerjs -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "peers.labels" . | nindent 4 }} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: "{{ tpl .Values.ingress.className . }}" + tls: + - hosts: + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} + rules: + - host: {{ .Values.global.domainName }} + http: + paths: + - pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ $peerjsSvcPort }} + path: /assist/(.*) +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/service.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/service.yaml new file mode 100644 index 000000000..6fa2a69fb --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/templates/service.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "peers.fullname" . }} + labels: + {{- include "peers.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + {{- range $key, $val := .Values.service.ports }} + - port: {{ $val }} + targetPort: {{ $key }} + protocol: TCP + name: {{ $key }} + {{- end}} + selector: + {{- include "peers.selectorLabels" . | nindent 4 }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "peers.fullname" . }}-headless + labels: + {{- include "peers.labels" . | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + {{- range $key, $val := .Values.service.ports }} + - port: {{ $val }} + targetPort: {{ $key }} + protocol: TCP + name: {{ $key }} + {{- end}} + selector: + {{- include "peers.selectorLabels" . | nindent 4 }} diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/serviceaccount.yaml new file mode 100644 index 000000000..4b6cc04fd --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "peers.serviceAccountName" . }} + labels: + {{- include "peers.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/tests/test-connection.yaml new file mode 100644 index 000000000..e8dd6a063 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "peers.fullname" . }}-test-connection" + labels: + {{- include "peers.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "peers.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helmcharts/openreplay/charts/peers/values.yaml b/scripts/helmcharts/openreplay/charts/peers/values.yaml new file mode 100644 index 000000000..ab74348fc --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/values.yaml @@ -0,0 +1,83 @@ +# Default values for openreplay. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rg.fr-par.scw.cloud/foss/peers + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "peers" +fullnameOverride: "peers" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +#service: +# type: ClusterIP +# port: 9000 + +service: + type: ClusterIP + ports: + peerjs: 9000 + +ingress: + enabled: true + className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + tls: + secretName: openreplay-ssl + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: + debug: 0 + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml deleted file mode 100644 index 44b72f68d..000000000 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "utilities.fullname" . }}-test-connection" - labels: - {{- include "utilities.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "utilities.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/utilities/build.sh b/utilities/build.sh index 99be144fe..f7d003ed3 100644 --- a/utilities/build.sh +++ b/utilities/build.sh @@ -20,11 +20,11 @@ function build_api(){ [[ $1 == "ee" ]] && { cp -rf ../ee/utilities/* ./ } - docker build -f ./Dockerfile -t ${DOCKER_REPO:-'local'}/utilities:${git_sha1} . + docker build -f ./Dockerfile -t ${DOCKER_REPO:-'local'}/assist:${git_sha1} . [[ $PUSH_IMAGE -eq 1 ]] && { - docker push ${DOCKER_REPO:-'local'}/utilities:${git_sha1} - docker tag ${DOCKER_REPO:-'local'}/utilities:${git_sha1} ${DOCKER_REPO:-'local'}/utilities:latest - docker push ${DOCKER_REPO:-'local'}/utilities:latest + docker push ${DOCKER_REPO:-'local'}/assist:${git_sha1} + docker tag ${DOCKER_REPO:-'local'}/assist:${git_sha1} ${DOCKER_REPO:-'local'}/assist:latest + docker push ${DOCKER_REPO:-'local'}/assist:latest } } From b7b1d2b315443e1854403c8fe8f871c4632b5d31 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 14 Apr 2022 14:00:17 +0200 Subject: [PATCH 112/125] feat(api): dashboard removed unused code --- api/chalicelib/core/dashboard.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index e5bc8003e..1afa10538 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -126,13 +126,6 @@ SESSIONS_META_FIELDS = {"revId": "rev_id", "browser": "user_browser"} -def __get_domains_errors_neutral(rows): - neutral = {l: 0 for l in [i for k in [list(v.keys()) for v in rows] for i in k]} - if len(neutral.keys()) == 0: - neutral = {"All": 0} - return neutral - - def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): From 53c83f99a6b7e05f09aff22fa14d6a37e7439edb Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 14:06:58 +0200 Subject: [PATCH 113/125] chore(helm): nginx change max worker connections Signed-off-by: rjshrjndrn --- scripts/helmcharts/vars.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 9c72d0d3a..1e8f268d3 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -56,6 +56,8 @@ ingress-nginx: &ingress-nginx config: enable-real-ip: true forwarded-for-header: "proxy_protocol" + # Ref: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#max-worker-connections + max-worker-connections: 0 # If you want the ssl offload in LB layer # Uncomment the following line # ssl-redirect: false From c174b0a63b024e8832640f8a03c59c085a6a882b Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 14:56:44 +0200 Subject: [PATCH 114/125] chore(assist): fix ports Signed-off-by: rjshrjndrn --- .../openreplay/charts/assist/templates/ingress.yaml | 10 +--------- .../helmcharts/openreplay/charts/assist/values.yaml | 1 - 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml index 48e568298..f5e2a3c03 100644 --- a/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml @@ -1,6 +1,5 @@ {{- if .Values.ingress.enabled }} {{- $fullName := include "assist.fullname" . -}} -{{- $peerjsSvcPort := .Values.service.ports.peerjs -}} {{- $socketioSvcPort := .Values.service.ports.socketio -}} apiVersion: networking.k8s.io/v1 kind: Ingress @@ -26,18 +25,11 @@ spec: - host: {{ .Values.global.domainName }} http: paths: - - pathType: Prefix - backend: - service: - name: {{ $fullName }} - port: - number: {{ $peerjsSvcPort }} - path: /assist/(.*) - pathType: Prefix backend: service: name: {{ $fullName }} port: number: {{ $socketioSvcPort }} - path: /ws-assist/(.*) + path: /assist/(.*) {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/assist/values.yaml b/scripts/helmcharts/openreplay/charts/assist/values.yaml index 861d715f9..a78be6a33 100644 --- a/scripts/helmcharts/openreplay/charts/assist/values.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/values.yaml @@ -43,7 +43,6 @@ securityContext: {} service: type: ClusterIP ports: - peerjs: 9000 socketio: 9001 ingress: From 02067df36b882b141d89a7bb06eb6c76a3c81d17 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 14:58:23 +0200 Subject: [PATCH 115/125] chore(assist): remove headless port Signed-off-by: rjshrjndrn --- .../charts/assist/templates/service.yaml | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/assist/templates/service.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/service.yaml index 699d9d149..13b78654c 100644 --- a/scripts/helmcharts/openreplay/charts/assist/templates/service.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/service.yaml @@ -15,23 +15,3 @@ spec: {{- end}} selector: {{- include "assist.selectorLabels" . | nindent 4 }} - ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "assist.fullname" . }}-headless - labels: - {{- include "assist.labels" . | nindent 4 }} -spec: - type: ClusterIP - clusterIP: None - ports: - {{- range $key, $val := .Values.service.ports }} - - port: {{ $val }} - targetPort: {{ $key }} - protocol: TCP - name: {{ $key }} - {{- end}} - selector: - {{- include "assist.selectorLabels" . | nindent 4 }} From 0b16448e419339bedb3473681ab989213049ff1e Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 15:02:55 +0200 Subject: [PATCH 116/125] chore(peers): remove headless svc Signed-off-by: rjshrjndrn --- .../charts/peers/templates/service.yaml | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/service.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/service.yaml index 6fa2a69fb..3a069c2c2 100644 --- a/scripts/helmcharts/openreplay/charts/peers/templates/service.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/templates/service.yaml @@ -15,23 +15,3 @@ spec: {{- end}} selector: {{- include "peers.selectorLabels" . | nindent 4 }} - ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "peers.fullname" . }}-headless - labels: - {{- include "peers.labels" . | nindent 4 }} -spec: - type: ClusterIP - clusterIP: None - ports: - {{- range $key, $val := .Values.service.ports }} - - port: {{ $val }} - targetPort: {{ $key }} - protocol: TCP - name: {{ $key }} - {{- end}} - selector: - {{- include "peers.selectorLabels" . | nindent 4 }} From 61247e5d416d9e5cfd5d456df4d3a1765e773e1b Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 15:29:44 +0200 Subject: [PATCH 117/125] chore(kubernetes): upgrade kube Signed-off-by: rjshrjndrn --- scripts/helmcharts/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/init.sh b/scripts/helmcharts/init.sh index c772629fc..078723007 100644 --- a/scripts/helmcharts/init.sh +++ b/scripts/helmcharts/init.sh @@ -19,7 +19,7 @@ version="v1.5.4" usr=`whoami` # Installing k3s -curl -sL https://get.k3s.io | sudo K3S_KUBECONFIG_MODE="644" INSTALL_K3S_VERSION='v1.19.5+k3s2' INSTALL_K3S_EXEC="--no-deploy=traefik" sh - +curl -sL https://get.k3s.io | sudo K3S_KUBECONFIG_MODE="644" INSTALL_K3S_VERSION='v1.22.8+k3s1' INSTALL_K3S_EXEC="--no-deploy=traefik" sh - mkdir ~/.kube sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config chmod 0644 ~/.kube/config From 44d5f8bc104e8ea0a8a5ceda24fafa6ff3b5a96e Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 15:39:04 +0200 Subject: [PATCH 118/125] chore(helm): adding clusterIssuer for ssl Signed-off-by: rjshrjndrn --- scripts/helmcharts/clusterIssuer.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 scripts/helmcharts/clusterIssuer.yaml diff --git a/scripts/helmcharts/clusterIssuer.yaml b/scripts/helmcharts/clusterIssuer.yaml new file mode 100644 index 000000000..fa18dfb3e --- /dev/null +++ b/scripts/helmcharts/clusterIssuer.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + # The ACME server URL + server: https://acme-v02.api.letsencrypt.org/directory + # Email address used for ACME registration + email: + # Name of a secret used to store the ACME account private key + privateKeySecretRef: + name: letsencrypt-prod + # Enable the HTTP-01 challenge provider + solvers: + - http01: + ingress: + class: openreplay From ac9fdf1091735c6a6ec118b7b2396ef624d2bcb2 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 15:43:06 +0200 Subject: [PATCH 119/125] fix(certmanager): email update Signed-off-by: rjshrjndrn --- scripts/helmcharts/certmanager.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/certmanager.sh b/scripts/helmcharts/certmanager.sh index 23f138084..1b8a57422 100644 --- a/scripts/helmcharts/certmanager.sh +++ b/scripts/helmcharts/certmanager.sh @@ -25,7 +25,7 @@ else fatal "Email address $EMAIL_ADDRESS is invalid." fi -sed -i "s/email: \"\"/email: \"${EMAIL_ADDRESS}\"/g" clusterIssuer.yaml +sed -i "s/email: .*/email: \"${EMAIL_ADDRESS}\"/g" clusterIssuer.yaml info "Installing cert-manager for auto letsencrypt certificate" kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.0/cert-manager.crds.yaml helm repo add jetstack https://charts.jetstack.io From 6f5bda117c4fd2fe6767c4d19bc6146570c18f83 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 15:48:37 +0200 Subject: [PATCH 120/125] fix(ingress): assist path Signed-off-by: rjshrjndrn --- .../helmcharts/openreplay/charts/assist/templates/ingress.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml index f5e2a3c03..f791024bb 100644 --- a/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml @@ -31,5 +31,5 @@ spec: name: {{ $fullName }} port: number: {{ $socketioSvcPort }} - path: /assist/(.*) + path: /ws-assist/(.*) {{- end }} From 87049b4633e9df117fd1b2491b739e1c950e87ef Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 16:26:17 +0200 Subject: [PATCH 121/125] chore(certmanager): check domain name has ip, then proceed Signed-off-by: rjshrjndrn --- scripts/helmcharts/certmanager.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/helmcharts/certmanager.sh b/scripts/helmcharts/certmanager.sh index 1b8a57422..64ba77b14 100644 --- a/scripts/helmcharts/certmanager.sh +++ b/scripts/helmcharts/certmanager.sh @@ -13,6 +13,11 @@ fatal() exit 1 } +read -p "enter openreplay domain name: " domain +nslookup domain > /dev/null || { + fatal "Domain name doesn't have ip associated with it. Please check your DNS record." +} + # Reading email address for ssl certificate [[ -z $EMAIL_ADDRESS ]] && { read -p "Enter your email address for letsencrypt certificate: " EMAIL_ADDRESS From 5cc49b9e9b430247409e49f896725833819622d7 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 17:31:49 +0200 Subject: [PATCH 122/125] chore(helm): overriding svc name template for assist/peers Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/values.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/openreplay/values.yaml b/scripts/helmcharts/openreplay/values.yaml index 7fcd4121e..f702c1a49 100644 --- a/scripts/helmcharts/openreplay/values.yaml +++ b/scripts/helmcharts/openreplay/values.yaml @@ -110,5 +110,7 @@ sink: fullnameOverride: sink-openreplay storage: fullnameOverride: storage-openreplay -utilities: - fullnameOverride: utilities-openreplay +assist: + fullnameOverride: assist-openreplay +peers: + fullnameOverride: peers-openreplay From e35cc072e8f616bd20bb33d0985d7df1183c5d88 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 19:53:51 +0200 Subject: [PATCH 123/125] fix(certmanager): check domain ip Signed-off-by: rjshrjndrn --- scripts/helmcharts/certmanager.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/certmanager.sh b/scripts/helmcharts/certmanager.sh index 64ba77b14..daab4d661 100644 --- a/scripts/helmcharts/certmanager.sh +++ b/scripts/helmcharts/certmanager.sh @@ -14,8 +14,8 @@ fatal() } read -p "enter openreplay domain name: " domain -nslookup domain > /dev/null || { - fatal "Domain name doesn't have ip associated with it. Please check your DNS record." +nslookup $domain > /dev/null || { + fatal "Domain name does not have ip associated with it. Please check your DNS record." } # Reading email address for ssl certificate From 07b38c8971058e26add705830cb73346b08b52c3 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 20:10:26 +0200 Subject: [PATCH 124/125] chore(helm): removed deprecated Nginx config Signed-off-by: rjshrjndrn --- scripts/helmcharts/vars.yaml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 1e8f268d3..67c1991b0 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -78,7 +78,7 @@ global: # if you're using one node installation, where # you're using local s3, make sure these variables # are same as minio.global.minio.accesskey and secretKey - accessKey: "changeMeMinioAccessKey" + accessKey: "YkkPAPYjogRlicqvCuNSHkfsdGtCCq" secretKey: "changeMeMinioPassword" email: emailHost: '' @@ -92,7 +92,7 @@ global: emailFrom: 'OpenReplay' enterpriseEditionLicense: "" - domainName: "" + domainName: "foss.openreplay.com" # If there is multiple nodes in the kubernetes cluster, # we'll have to create a NFS share PVC for both the containers to share data. @@ -132,14 +132,3 @@ chalice: # requests: # cpu: 512m # memory: 2056Mi - -## Changes to nginx -# -# nginx-ingress: -# # Key and certificate files must be named site.key and site.crt -# # and copied to ../openreplay/files/ -# sslKey: site.key -# sslCert: site.crt -# # Redirecting http to https -# customServerConfigs: | -# return 301 https://$host$request_uri; From 7c7aee64591cd1c7492505d63689b6b99bfd07e6 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 14 Apr 2022 20:40:57 +0200 Subject: [PATCH 125/125] chore(nginx): disable ssl redirection by default Signed-off-by: rjshrjndrn --- scripts/helmcharts/vars.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 67c1991b0..7f5aaf424 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -58,10 +58,10 @@ ingress-nginx: &ingress-nginx forwarded-for-header: "proxy_protocol" # Ref: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#max-worker-connections max-worker-connections: 0 - # If you want the ssl offload in LB layer - # Uncomment the following line - # ssl-redirect: false - # force-ssl-redirect: false + # If you want the ssl redirection + # Comment the following line + ssl-redirect: false + force-ssl-redirect: false # Application specific variables global: