diff --git a/api/.env.default b/api/.env.default index 6ae959a7d..6caeb96d8 100644 --- a/api/.env.default +++ b/api/.env.default @@ -37,11 +37,13 @@ pg_port=5432 pg_user=postgres pg_timeout=30 pg_minconn=45 +PG_RETRY_MAX=50 +PG_RETRY_INTERVAL=2 put_S3_TTL=20 sentryURL= sessions_bucket=mobs sessions_region=us-east-1 sourcemaps_bucket=sourcemaps -sourcemaps_reader=http://utilities-openreplay.app.svc.cluster.local:9000/sourcemaps +sourcemaps_reader=http://127.0.0.1:9000/ stage=default-foss version_number=1.4.0 \ No newline at end of file diff --git a/api/Dockerfile b/api/Dockerfile index 780518ff3..0673ab2b5 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -5,6 +5,15 @@ WORKDIR /work COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env +ENV APP_NAME chalice +# Installing Nodejs +RUN apt update && apt install -y curl && \ + curl -fsSL https://deb.nodesource.com/setup_12.x | bash - && \ + apt install -y nodejs && \ + apt remove --purge -y curl && \ + rm -rf /var/lib/apt/lists/* && \ + cd sourcemap-reader && \ + npm install # Add Tini # Startup daemon diff --git a/api/Dockerfile.alerts b/api/Dockerfile.alerts index ed8f06eac..bdd1772ba 100644 --- a/api/Dockerfile.alerts +++ b/api/Dockerfile.alerts @@ -6,6 +6,7 @@ COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env && mv app_alerts.py app.py ENV pg_minconn 2 +ENV APP_NAME alerts # Add Tini # Startup daemon diff --git a/api/app.py b/api/app.py index d261dadac..959f1ef8f 100644 --- a/api/app.py +++ b/api/app.py @@ -9,12 +9,11 @@ from starlette.responses import StreamingResponse from chalicelib.utils import helper from chalicelib.utils import pg_client from routers import core, core_dynamic -from routers.app import v1_api from routers.crons import core_crons from routers.crons import core_dynamic_crons -from routers.subs import dashboard +from routers.subs import dashboard, insights, metrics, v1_api -app = FastAPI() +app = FastAPI(root_path="/api") @app.middleware('http') @@ -54,7 +53,8 @@ app.include_router(core_dynamic.public_app) app.include_router(core_dynamic.app) app.include_router(core_dynamic.app_apikey) app.include_router(dashboard.app) -# app.include_router(insights.app) +app.include_router(metrics.app) +app.include_router(insights.app) app.include_router(v1_api.app_apikey) Schedule = AsyncIOScheduler() diff --git a/api/auth/auth_project.py b/api/auth/auth_project.py new file mode 100644 index 000000000..98a495bbb --- /dev/null +++ b/api/auth/auth_project.py @@ -0,0 +1,24 @@ +from fastapi import Request +from starlette import status +from starlette.exceptions import HTTPException + +import schemas +from chalicelib.core import projects +from or_dependencies import OR_context + + +class ProjectAuthorizer: + def __init__(self, project_identifier): + self.project_identifier: str = project_identifier + + async def __call__(self, request: Request) -> None: + if len(request.path_params.keys()) == 0 or request.path_params.get(self.project_identifier) is None: + return + current_user: schemas.CurrentContext = await OR_context(request) + project_identifier = request.path_params[self.project_identifier] + if (self.project_identifier == "projectId" \ + and projects.get_project(project_id=project_identifier, tenant_id=current_user.tenant_id) is None) \ + or (self.project_identifier.lower() == "projectKey" \ + and projects.get_internal_project_id(project_key=project_identifier) is None): + print("project not found") + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="project not found.") diff --git a/api/build.sh b/api/build.sh index 29b8911ca..cec7525f5 100644 --- a/api/build.sh +++ b/api/build.sh @@ -18,6 +18,8 @@ check_prereq() { } function build_api(){ + cp -R ../utilities/utils ../sourcemap-reader/. + cp -R ../sourcemap-reader . tag="" # Copy enterprise code [[ $1 == "ee" ]] && { diff --git a/api/chalicelib/core/authorizers.py b/api/chalicelib/core/authorizers.py index 33a859cc8..5756e82ab 100644 --- a/api/chalicelib/core/authorizers.py +++ b/api/chalicelib/core/authorizers.py @@ -42,7 +42,7 @@ def generate_jwt(id, tenant_id, iat, aud): payload={ "userId": id, "tenantId": tenant_id, - "exp": iat // 1000 + config("jwt_exp_delta_seconds",cast=int) + TimeUTC.get_utc_offset() // 1000, + "exp": iat // 1000 + config("jwt_exp_delta_seconds", cast=int) + TimeUTC.get_utc_offset() // 1000, "iss": config("jwt_issuer"), "iat": iat // 1000, "aud": aud diff --git a/api/chalicelib/core/boarding.py b/api/chalicelib/core/boarding.py index c303643c8..68843b2f8 100644 --- a/api/chalicelib/core/boarding.py +++ b/api/chalicelib/core/boarding.py @@ -5,39 +5,38 @@ from chalicelib.core import users def get_state(tenant_id): - my_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) - pids = [s["projectId"] for s in my_projects] + pids = projects.get_projects_ids(tenant_id=tenant_id) with pg_client.PostgresClient() as cur: recorded = False meta = False if len(pids) > 0: cur.execute( - cur.mogrify("""\ - SELECT - COUNT(*) - FROM public.sessions AS s - where s.project_id IN %(ids)s - LIMIT 1;""", + cur.mogrify("""SELECT EXISTS(( SELECT 1 + FROM public.sessions AS s + WHERE s.project_id IN %(ids)s)) AS exists;""", {"ids": tuple(pids)}) ) - recorded = cur.fetchone()["count"] > 0 + recorded = cur.fetchone()["exists"] meta = False if recorded: - cur.execute("""SELECT SUM((SELECT COUNT(t.meta) - FROM (VALUES (p.metadata_1), (p.metadata_2), (p.metadata_3), (p.metadata_4), (p.metadata_5), - (p.metadata_6), (p.metadata_7), (p.metadata_8), (p.metadata_9), (p.metadata_10), - (sessions.user_id)) AS t(meta) - WHERE t.meta NOTNULL)) - FROM public.projects AS p - LEFT JOIN LATERAL ( SELECT 'defined' - FROM public.sessions - WHERE sessions.project_id=p.project_id AND sessions.user_id IS NOT NULL - LIMIT 1) AS sessions(user_id) ON(TRUE) - WHERE p.deleted_at ISNULL;""" - ) + cur.execute("""SELECT EXISTS((SELECT 1 + FROM public.projects AS p + LEFT JOIN LATERAL ( SELECT 1 + FROM public.sessions + WHERE sessions.project_id = p.project_id + AND sessions.user_id IS NOT NULL + LIMIT 1) AS sessions(user_id) ON (TRUE) + WHERE p.deleted_at ISNULL + AND ( sessions.user_id IS NOT NULL OR p.metadata_1 IS NOT NULL + OR p.metadata_2 IS NOT NULL OR p.metadata_3 IS NOT NULL + OR p.metadata_4 IS NOT NULL OR p.metadata_5 IS NOT NULL + OR p.metadata_6 IS NOT NULL OR p.metadata_7 IS NOT NULL + OR p.metadata_8 IS NOT NULL OR p.metadata_9 IS NOT NULL + OR p.metadata_10 IS NOT NULL ) + )) AS exists;""") - meta = cur.fetchone()["sum"] > 0 + meta = cur.fetchone()["exists"] return [ {"task": "Install OpenReplay", @@ -58,22 +57,18 @@ def get_state(tenant_id): def get_state_installing(tenant_id): - my_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) - pids = [s["projectId"] for s in my_projects] + pids = projects.get_projects_ids(tenant_id=tenant_id) with pg_client.PostgresClient() as cur: recorded = False if len(pids) > 0: cur.execute( - cur.mogrify("""\ - SELECT - COUNT(*) - FROM public.sessions AS s - where s.project_id IN %(ids)s - LIMIT 1;""", + cur.mogrify("""SELECT EXISTS(( SELECT 1 + FROM public.sessions AS s + WHERE s.project_id IN %(ids)s)) AS exists;""", {"ids": tuple(pids)}) ) - recorded = cur.fetchone()["count"] > 0 + recorded = cur.fetchone()["exists"] return {"task": "Install OpenReplay", "done": recorded, @@ -82,20 +77,23 @@ def get_state_installing(tenant_id): def get_state_identify_users(tenant_id): with pg_client.PostgresClient() as cur: - cur.execute( - """SELECT SUM((SELECT COUNT(t.meta) - FROM (VALUES (p.metadata_1), (p.metadata_2), (p.metadata_3), (p.metadata_4), (p.metadata_5), - (p.metadata_6), (p.metadata_7), (p.metadata_8), (p.metadata_9), (p.metadata_10), - (sessions.user_id)) AS t(meta) - WHERE t.meta NOTNULL)) - FROM public.projects AS p - LEFT JOIN LATERAL ( SELECT 'defined' - FROM public.sessions - WHERE sessions.project_id=p.project_id AND sessions.user_id IS NOT NULL - LIMIT 1) AS sessions(user_id) ON(TRUE) - WHERE p.deleted_at ISNULL;""") + cur.execute("""SELECT EXISTS((SELECT 1 + FROM public.projects AS p + LEFT JOIN LATERAL ( SELECT 1 + FROM public.sessions + WHERE sessions.project_id = p.project_id + AND sessions.user_id IS NOT NULL + LIMIT 1) AS sessions(user_id) ON (TRUE) + WHERE p.deleted_at ISNULL + AND ( sessions.user_id IS NOT NULL OR p.metadata_1 IS NOT NULL + OR p.metadata_2 IS NOT NULL OR p.metadata_3 IS NOT NULL + OR p.metadata_4 IS NOT NULL OR p.metadata_5 IS NOT NULL + OR p.metadata_6 IS NOT NULL OR p.metadata_7 IS NOT NULL + OR p.metadata_8 IS NOT NULL OR p.metadata_9 IS NOT NULL + OR p.metadata_10 IS NOT NULL ) + )) AS exists;""") - meta = cur.fetchone()["sum"] > 0 + meta = cur.fetchone()["exists"] return {"task": "Identify Users", "done": meta, diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index e0b0ed432..15c2ffc49 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -9,11 +9,11 @@ from chalicelib.utils.TimeUTC import TimeUTC PIE_CHART_GROUP = 5 -def __try_live(project_id, data: schemas.CreateCustomMetricsSchema): +def __try_live(project_id, data: schemas.TryCustomMetricsPayloadSchema): results = [] for i, s in enumerate(data.series): - s.filter.startDate = data.startDate - s.filter.endDate = data.endDate + s.filter.startDate = data.startTimestamp + s.filter.endDate = data.endTimestamp results.append(sessions.search2_series(data=s.filter, project_id=project_id, density=data.density, view_type=data.view_type, metric_type=data.metric_type, metric_of=data.metric_of, metric_value=data.metric_value)) @@ -42,7 +42,7 @@ def __try_live(project_id, data: schemas.CreateCustomMetricsSchema): return results -def merged_live(project_id, data: schemas.CreateCustomMetricsSchema): +def merged_live(project_id, data: schemas.TryCustomMetricsPayloadSchema): series_charts = __try_live(project_id=project_id, data=data) if data.view_type == schemas.MetricTimeseriesViewType.progress or data.metric_type == schemas.MetricType.table: return series_charts @@ -54,13 +54,9 @@ def merged_live(project_id, data: schemas.CreateCustomMetricsSchema): return results -def __get_merged_metric(project_id, user_id, metric_id, - data: Union[schemas.CustomMetricChartPayloadSchema, - schemas.CustomMetricSessionsPayloadSchema]) \ +def __merge_metric_with_data(metric, data: Union[schemas.CustomMetricChartPayloadSchema, + schemas.CustomMetricSessionsPayloadSchema]) \ -> Union[schemas.CreateCustomMetricsSchema, None]: - metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) - if metric is None: - return None metric: schemas.CreateCustomMetricsSchema = schemas.CreateCustomMetricsSchema.parse_obj({**data.dict(), **metric}) if len(data.filters) > 0 or len(data.events) > 0: for s in metric.series: @@ -71,11 +67,12 @@ def __get_merged_metric(project_id, user_id, metric_id, return metric -def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema): - metric: schemas.CreateCustomMetricsSchema = __get_merged_metric(project_id=project_id, user_id=user_id, - metric_id=metric_id, data=data) +def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema, metric=None): + if metric is None: + metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) if metric is None: return None + metric: schemas.CreateCustomMetricsSchema = __merge_metric_with_data(metric=metric, data=data) series_charts = __try_live(project_id=project_id, data=metric) if metric.view_type == schemas.MetricTimeseriesViewType.progress or metric.metric_type == schemas.MetricType.table: return series_charts @@ -88,21 +85,23 @@ def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPa def get_sessions(project_id, user_id, metric_id, data: schemas.CustomMetricSessionsPayloadSchema): - metric: schemas.CreateCustomMetricsSchema = __get_merged_metric(project_id=project_id, user_id=user_id, - metric_id=metric_id, data=data) + metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) + if metric is None: + return None + metric: schemas.CreateCustomMetricsSchema = __merge_metric_with_data(metric=metric, data=data) if metric is None: return None results = [] for s in metric.series: - s.filter.startDate = data.startDate - s.filter.endDate = data.endDate + s.filter.startDate = data.startTimestamp + s.filter.endDate = data.endTimestamp results.append({"seriesId": s.series_id, "seriesName": s.name, **sessions.search2_pg(data=s.filter, project_id=project_id, user_id=user_id)}) return results -def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): +def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema, dashboard=False): with pg_client.PostgresClient() as cur: _data = {} for i, s in enumerate(data.series): @@ -129,6 +128,8 @@ def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): query ) r = cur.fetchone() + if dashboard: + return r["metric_id"] return {"data": get(metric_id=r["metric_id"], project_id=project_id, user_id=user_id)} @@ -147,10 +148,11 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche "metric_value": data.metric_value, "metric_format": data.metric_format} for i, s in enumerate(data.series): prefix = "u_" + if s.index is None: + s.index = i if s.series_id is None or s.series_id not in series_ids: n_series.append({"i": i, "s": s}) prefix = "n_" - s.index = i else: u_series.append({"i": i, "s": s}) u_series_ids.append(s.series_id) @@ -192,40 +194,60 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche SET name = %(name)s, is_public= %(is_public)s, view_type= %(view_type)s, metric_type= %(metric_type)s, metric_of= %(metric_of)s, metric_value= %(metric_value)s, - metric_format= %(metric_format)s + metric_format= %(metric_format)s, + edited_at = timezone('utc'::text, now()) WHERE metric_id = %(metric_id)s AND project_id = %(project_id)s AND (user_id = %(user_id)s OR is_public) RETURNING metric_id;""", params) - cur.execute( - query - ) + cur.execute(query) return get(metric_id=metric_id, project_id=project_id, user_id=user_id) -def get_all(project_id, user_id): +def get_all(project_id, user_id, include_series=False): with pg_client.PostgresClient() as cur: - cur.execute( - cur.mogrify( - """SELECT * - FROM metrics - LEFT JOIN LATERAL (SELECT jsonb_agg(metric_series.* ORDER BY index) AS series + sub_join = "" + if include_series: + sub_join = """LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(metric_series.* ORDER BY index),'[]'::jsonb) AS series FROM metric_series WHERE metric_series.metric_id = metrics.metric_id AND metric_series.deleted_at ISNULL - ) AS metric_series ON (TRUE) + ) AS metric_series ON (TRUE)""" + cur.execute( + cur.mogrify( + f"""SELECT * + FROM metrics + {sub_join} + LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards + FROM (SELECT DISTINCT dashboard_id, name, is_public + FROM dashboards INNER JOIN dashboard_widgets USING (dashboard_id) + WHERE deleted_at ISNULL + AND dashboard_widgets.metric_id = metrics.metric_id + AND project_id = %(project_id)s + AND ((dashboards.user_id = %(user_id)s OR is_public))) AS connected_dashboards + ) AS connected_dashboards ON (TRUE) + LEFT JOIN LATERAL (SELECT email AS owner_email + FROM users + WHERE deleted_at ISNULL + AND users.user_id = metrics.user_id + ) AS owner ON (TRUE) WHERE metrics.project_id = %(project_id)s AND metrics.deleted_at ISNULL - AND (user_id = %(user_id)s OR is_public) - ORDER BY created_at;""", + AND (user_id = %(user_id)s OR metrics.is_public) + ORDER BY metrics.edited_at, metrics.created_at;""", {"project_id": project_id, "user_id": user_id} ) ) rows = cur.fetchall() - for r in rows: - r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) - for s in r["series"]: - s["filter"] = helper.old_search_payload_to_flat(s["filter"]) + if include_series: + for r in rows: + # r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + for s in r["series"]: + s["filter"] = helper.old_search_payload_to_flat(s["filter"]) + else: + for r in rows: + r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r["edited_at"] = TimeUTC.datetime_to_timestamp(r["edited_at"]) rows = helper.list_to_camel_case(rows) return rows @@ -235,7 +257,7 @@ def delete(project_id, metric_id, user_id): cur.execute( cur.mogrify("""\ UPDATE public.metrics - SET deleted_at = timezone('utc'::text, now()) + SET deleted_at = timezone('utc'::text, now()), edited_at = timezone('utc'::text, now()) WHERE project_id = %(project_id)s AND metric_id = %(metric_id)s AND (user_id = %(user_id)s OR is_public);""", @@ -256,6 +278,18 @@ def get(metric_id, project_id, user_id, flatten=True): WHERE metric_series.metric_id = metrics.metric_id AND metric_series.deleted_at ISNULL ) AS metric_series ON (TRUE) + LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards + FROM (SELECT dashboard_id, name, is_public + FROM dashboards + WHERE deleted_at ISNULL + AND project_id = %(project_id)s + AND ((user_id = %(user_id)s OR is_public))) AS connected_dashboards + ) AS connected_dashboards ON (TRUE) + LEFT JOIN LATERAL (SELECT email AS owner_email + FROM users + WHERE deleted_at ISNULL + AND users.user_id = metrics.user_id + ) AS owner ON (TRUE) WHERE metrics.project_id = %(project_id)s AND metrics.deleted_at ISNULL AND (metrics.user_id = %(user_id)s OR metrics.is_public) @@ -268,12 +302,46 @@ def get(metric_id, project_id, user_id, flatten=True): if row is None: return None row["created_at"] = TimeUTC.datetime_to_timestamp(row["created_at"]) + row["edited_at"] = TimeUTC.datetime_to_timestamp(row["edited_at"]) if flatten: for s in row["series"]: s["filter"] = helper.old_search_payload_to_flat(s["filter"]) return helper.dict_to_camel_case(row) +def get_with_template(metric_id, project_id, user_id, include_dashboard=True): + with pg_client.PostgresClient() as cur: + sub_query = "" + if include_dashboard: + sub_query = """LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards + FROM (SELECT dashboard_id, name, is_public + FROM dashboards + WHERE deleted_at ISNULL + AND project_id = %(project_id)s + AND ((user_id = %(user_id)s OR is_public))) AS connected_dashboards + ) AS connected_dashboards ON (TRUE)""" + cur.execute( + cur.mogrify( + f"""SELECT * + FROM metrics + LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(metric_series.* ORDER BY index),'[]'::jsonb) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + {sub_query} + WHERE (metrics.project_id = %(project_id)s OR metrics.project_id ISNULL) + AND metrics.deleted_at ISNULL + AND (metrics.user_id = %(user_id)s OR metrics.is_public) + AND metrics.metric_id = %(metric_id)s + ORDER BY created_at;""", + {"metric_id": metric_id, "project_id": project_id, "user_id": user_id} + ) + ) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + def get_series_for_alert(project_id, user_id): with pg_client.PostgresClient() as cur: cur.execute( diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index 9cd88eb6a..1afa10538 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -1,11 +1,12 @@ +import math + import schemas from chalicelib.core import metadata from chalicelib.utils import args_transformer -from chalicelib.utils import helper, dev +from chalicelib.utils import helper from chalicelib.utils import pg_client from chalicelib.utils.TimeUTC import TimeUTC from chalicelib.utils.metrics_helper import __get_step_size -import math # Written by David Aznaurov, inspired by numpy.quantile @@ -75,8 +76,6 @@ METADATA_FIELDS = {"userId": "user_id", "metadata9": "metadata_9", "metadata10": "metadata_10"} -from chalicelib.core import sessions_metas - def __get_meta_constraint(project_id, data): if len(data.get("filters", [])) == 0: @@ -127,7 +126,6 @@ SESSIONS_META_FIELDS = {"revId": "rev_id", "browser": "user_browser"} -@dev.timed def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -138,7 +136,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) with pg_client.PostgresClient() as cur: pg_query = f"""\ SELECT generated_timestamp AS timestamp, - COALESCE(COUNT(sessions), 0) AS count + COALESCE(COUNT(sessions), 0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT 1 FROM public.sessions @@ -151,7 +149,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) cur.execute(cur.mogrify(pg_query, params)) rows = cur.fetchall() results = { - "count": sum([r["count"] for r in rows]), + "value": sum([r["value"] for r in rows]), "chart": rows } @@ -170,12 +168,11 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) count = cur.fetchone()["count"] - results["countProgress"] = helper.__progress(old_val=count, new_val=results["count"]) - + results["progress"] = helper.__progress(old_val=count, new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.count return results -@dev.timed def get_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -234,7 +231,6 @@ def __count_distinct_errors(cur, project_id, startTimestamp, endTimestamp, pg_su return cur.fetchone()["count"] -@dev.timed def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -298,7 +294,6 @@ def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return rows -@dev.timed def get_page_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: @@ -316,7 +311,6 @@ def get_page_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return results -@dev.timed def __get_page_metrics(cur, project_id, startTimestamp, endTimestamp, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) pg_sub_query.append("pages.timestamp>=%(startTimestamp)s") @@ -336,7 +330,6 @@ def __get_page_metrics(cur, project_id, startTimestamp, endTimestamp, **args): return rows -@dev.timed def get_application_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: @@ -390,7 +383,6 @@ def __get_application_activity(cur, project_id, startTimestamp, endTimestamp, ** return result -@dev.timed def get_user_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): with pg_client.PostgresClient() as cur: @@ -423,7 +415,6 @@ def __get_user_activity(cur, project_id, startTimestamp, endTimestamp, **args): return row -@dev.timed def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -468,8 +459,9 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ORDER BY generated_timestamp) AS chart ) AS chart ON (TRUE);""" - cur.execute(cur.mogrify(pg_query, {"step_size": step_size,"project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + cur.execute( + cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) rows = cur.fetchall() for i in range(len(rows)): rows[i]["sessions"] = rows[i].pop("sessions_count") @@ -478,7 +470,6 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return sorted(rows, key=lambda k: k["sessions"], reverse=True) -@dev.timed def __get_performance_constraint(l): if len(l) == 0: return "" @@ -486,7 +477,6 @@ def __get_performance_constraint(l): return f"AND ({' OR '.join(l)})" -@dev.timed def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, resources=None, **args): step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) @@ -621,7 +611,6 @@ def __get_resource_db_type_from_type(resource_type): return {v: k for k, v in RESOURCS_TYPE_TO_DB_TYPE.items()}.get(resource_type, resource_type) -@dev.timed def search(text, resource_type, project_id, performance=False, pages_only=False, events_only=False, metadata=False, key=None, platform=None): if not resource_type: @@ -672,8 +661,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} AND positionUTF8(url_path, %(value)s) != 0 LIMIT 10);""" print(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text.lower()), - "platform_0": platform})) + "value": helper.string_to_sql_like(text.lower()), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text.lower()), "platform_0": platform})) @@ -691,9 +680,9 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" print(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "resource_type": resource_type, - "platform_0": platform})) + "value": helper.string_to_sql_like(text), + "resource_type": resource_type, + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "resource_type": resource_type, @@ -709,8 +698,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" print(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "platform_0": platform})) + "value": helper.string_to_sql_like(text), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "platform_0": platform})) @@ -723,8 +712,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" print(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "platform_0": platform})) + "value": helper.string_to_sql_like(text), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "platform_0": platform})) @@ -737,8 +726,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" print(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "platform_0": platform})) + "value": helper.string_to_sql_like(text), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "platform_0": platform})) @@ -758,8 +747,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" print(cur.mogrify(pg_query, - {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, - "platform_0": platform})) + {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, "platform_0": platform})) @@ -785,9 +774,9 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, LIMIT 10)""") pg_query = " UNION ALL ".join(pg_query) print(cur.mogrify(pg_query, - {"project_id": project_id, "value": helper.string_to_sql_like(text), - "key": key, - "platform_0": platform})) + {"project_id": project_id, "value": helper.string_to_sql_like(text), + "key": key, + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, @@ -798,7 +787,6 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, return [helper.dict_to_camel_case(row) for row in rows] -@dev.timed def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -854,7 +842,6 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day return rows -@dev.timed def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -866,8 +853,6 @@ def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1), pg_sub_query_subset.append("resources.timestamp>=%(startTimestamp)s") pg_sub_query_subset.append("resources.timestamp<%(endTimestamp)s") - - with pg_client.PostgresClient() as cur: pg_query = f"""WITH resources AS (SELECT resources.session_id, resources.url_hostpath, @@ -922,7 +907,6 @@ def dashboard_args(params): return args -@dev.timed def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, type=None, url=None, **args): @@ -971,7 +955,6 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days return {"avg": avg, "chart": rows} -@dev.timed def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=19, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -993,13 +976,13 @@ def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=- FROM public.sessions INNER JOIN events.pages USING (session_id) WHERE {" AND ".join(pg_sub_query_subset)}) - SELECT COALESCE(avg, 0) AS avg, chart + SELECT COALESCE(avg, 0) AS value, chart FROM (SELECT AVG(dom_building_time) FROM pages) AS avg LEFT JOIN (SELECT jsonb_agg(chart) AS chart FROM ( SELECT generated_timestamp AS timestamp, - COALESCE(AVG(dom_building_time), 0) AS avg + COALESCE(AVG(dom_building_time), 0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT pages.dom_building_time FROM pages @@ -1014,10 +997,10 @@ def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=- cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond return row -@dev.timed def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), type="all", density=19, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1091,7 +1074,6 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return rows -@dev.timed def get_sessions_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1110,7 +1092,6 @@ def get_sessions_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return {"count": sum(i["count"] for i in rows), "chart": helper.list_to_camel_case(rows)} -@dev.timed def get_speed_index_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1139,7 +1120,6 @@ def get_speed_index_location(project_id, startTimestamp=TimeUTC.now(delta_days=- return {"avg": avg, "chart": helper.list_to_camel_case(rows)} -@dev.timed def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1155,7 +1135,7 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 pg_sub_query_chart.append(f"url = %(value)s") with pg_client.PostgresClient() as cur: pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(pages.response_time),0) AS avg + COALESCE(AVG(pages.response_time),0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT response_time @@ -1176,10 +1156,9 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"avg": avg, "chart": rows} + return {"value": avg, "chart": rows, "unit": schemas.TemplatePredefinedUnits.millisecond} -@dev.timed def get_pages_response_time_distribution(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=20, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1297,7 +1276,6 @@ def get_pages_response_time_distribution(project_id, startTimestamp=TimeUTC.now( return result -@dev.timed def get_busiest_time_of_day(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1317,7 +1295,6 @@ def get_busiest_time_of_day(project_id, startTimestamp=TimeUTC.now(delta_days=-1 return rows -@dev.timed def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), value=None, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1368,7 +1345,6 @@ def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return helper.dict_to_camel_case(row) -@dev.timed def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, url=None, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1384,11 +1360,11 @@ def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), pg_query = f"""WITH pages AS(SELECT pages.visually_complete,pages.timestamp FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query_subset)}) - SELECT COALESCE((SELECT AVG(pages.visually_complete) FROM pages),0) AS avg, + SELECT COALESCE((SELECT AVG(pages.visually_complete) FROM pages),0) AS value, jsonb_agg(chart) AS chart FROM (SELECT generated_timestamp AS timestamp, - COALESCE(AVG(visually_complete), 0) AS avg + COALESCE(AVG(visually_complete), 0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT pages.visually_complete FROM pages @@ -1402,10 +1378,10 @@ def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), "endTimestamp": endTimestamp, "value": url, **__get_constraint_values(args)} cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond return row -@dev.timed def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), value=None, density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1444,7 +1420,6 @@ def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(d return rows -@dev.timed def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1454,7 +1429,7 @@ def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1) with pg_client.PostgresClient() as cur: pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(performance.avg_used_js_heap_size),0) AS avg_used_js_heap_size + COALESCE(AVG(performance.avg_used_js_heap_size),0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT avg_used_js_heap_size @@ -1474,10 +1449,9 @@ def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1) WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"avgUsedJsHeapSize": avg, "chart": helper.list_to_camel_case(rows)} + return {"value": avg, "chart": helper.list_to_camel_case(rows), "unit": schemas.TemplatePredefinedUnits.memory} -@dev.timed def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1487,7 +1461,7 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with pg_client.PostgresClient() as cur: pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(performance.avg_cpu),0) AS avg_cpu + COALESCE(AVG(performance.avg_cpu),0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT avg_cpu @@ -1507,10 +1481,10 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"avgCpu": avg, "chart": helper.list_to_camel_case(rows)} + return {"value": avg, "chart": helper.list_to_camel_case(rows), + "unit": schemas.TemplatePredefinedUnits.percentage} -@dev.timed def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1520,7 +1494,7 @@ def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with pg_client.PostgresClient() as cur: pg_query = f"""SELECT generated_timestamp AS timestamp, - COALESCE(AVG(NULLIF(performance.avg_fps,0)),0) AS avg_fps + COALESCE(AVG(NULLIF(performance.avg_fps,0)),0) AS value FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp LEFT JOIN LATERAL ( SELECT avg_fps @@ -1540,10 +1514,9 @@ def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), WHERE {" AND ".join(pg_sub_query)};""" cur.execute(cur.mogrify(pg_query, params)) avg = cur.fetchone()["avg"] - return {"avgFps": avg, "chart": helper.list_to_camel_case(rows)} + return {"value": avg, "chart": helper.list_to_camel_case(rows), "unit": schemas.TemplatePredefinedUnits.frame} -@dev.timed def get_crashes(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1617,7 +1590,7 @@ def get_crashes(project_id, startTimestamp=TimeUTC.now(delta_days=-1), def __get_neutral(rows, add_All_if_empty=True): neutral = {l: 0 for l in [i for k in [list(v.keys()) for v in rows] for i in k]} - if add_All_if_empty and len(neutral.keys()) == 0: + if add_All_if_empty and len(neutral.keys()) <= 1: neutral = {"All": 0} return neutral @@ -1628,7 +1601,6 @@ def __merge_rows_with_neutral(rows, neutral): return rows -@dev.timed def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1679,7 +1651,6 @@ def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return result -@dev.timed def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1720,7 +1691,6 @@ def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) return rows -@dev.timed def get_domains_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1769,7 +1739,6 @@ def __nested_array_to_dict_array(rows, key="url_host", value="count"): return rows -@dev.timed def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1801,7 +1770,6 @@ def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return {"avg": avg, "partition": rows} -@dev.timed def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1815,7 +1783,7 @@ def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1) WHERE {" AND ".join(pg_sub_query)} GROUP BY resources.url_host ORDER BY errors_count DESC - LIMIT 10;""" + LIMIT 5;""" cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)})) @@ -1823,7 +1791,6 @@ def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1) return helper.list_to_camel_case(rows) -@dev.timed def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1866,7 +1833,6 @@ def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=- return {"count": sum(i["count"] for i in rows), "chart": rows} -@dev.timed def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1892,7 +1858,6 @@ def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endT return helper.list_to_camel_case(rows) -@dev.timed def get_calls_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1916,7 +1881,6 @@ def get_calls_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return helper.list_to_camel_case(rows) -@dev.timed def get_calls_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) @@ -1940,7 +1904,6 @@ def get_calls_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return helper.list_to_camel_case(rows) -@dev.timed def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), platform=None, density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -1952,7 +1915,7 @@ def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), e pg_sub_query_subset.append("resources.status > 200") pg_sub_query_subset_e = __get_constraints(project_id=project_id, data=args, duration=False, main_table="m_errors", - time_constraint=False) + time_constraint=False) pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, chart=True, data=args, main_table="", time_column="timestamp", project=False, duration=False) @@ -2005,7 +1968,6 @@ def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), e return rows -@dev.timed def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -2060,7 +2022,6 @@ def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_d return helper.list_to_camel_case(__merge_charts(response_end, actions)) -@dev.timed def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -2142,7 +2103,6 @@ def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(de return {**row_sessions, **row_errors, "chart": chart} -@dev.timed def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -2193,7 +2153,6 @@ def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(de return helper.list_to_camel_case(rows) -@dev.timed def get_resources_count_by_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -2230,7 +2189,6 @@ def get_resources_count_by_type(project_id, startTimestamp=TimeUTC.now(delta_day return rows -@dev.timed def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) @@ -2284,3 +2242,537 @@ def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1) rows = cur.fetchall() return rows + + +def __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("resources.duration > 0") + pg_sub_query.append("resources.type= %(type)s") + pg_query = f"""\ + SELECT COALESCE(AVG(resources.duration),0) AS value + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + row = cur.fetchone() + return row + + +def get_application_activity_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + results = row + results["chart"] = get_performance_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond + return results + + +def get_performance_avg_image_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) + img_constraints = [] + + img_constraints_vals = {} + + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") + + pg_query = f"""WITH resources AS (SELECT resources.duration, resources.timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + AND resources.type = 'img' AND resources.duration>0 + {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} + ) + SELECT generated_timestamp AS timestamp, + COALESCE(AVG(resources.duration),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT resources.duration + FROM resources + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS resources ON (TRUE) + GROUP BY timestamp + ORDER BY timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **img_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() + rows = helper.list_to_camel_case(rows) + + return rows + + +def __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("pages.timestamp >= %(startTimestamp)s") + pg_sub_query.append("pages.timestamp > %(endTimestamp)s") + pg_sub_query.append("pages.load_time > 0") + pg_sub_query.append("pages.load_time IS NOT NULL") + pg_query = f"""\ + SELECT COALESCE(AVG(pages.load_time) ,0) AS value + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return row + + +def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + results = row + results["chart"] = get_performance_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond + return results + + +def get_performance_avg_page_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) + location_constraints = [] + location_constraints_vals = {} + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="pages", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s") + pg_query = f"""WITH pages AS(SELECT pages.load_time, timestamp + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} AND pages.load_time>0 AND pages.load_time IS NOT NULL + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + ) + SELECT generated_timestamp AS timestamp, + COALESCE(AVG(pages.load_time),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT pages.load_time + FROM pages + WHERE {" AND ".join(pg_sub_query_chart)} + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + ) AS pages ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **location_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() + return rows + + +def __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("resources.duration > 0") + pg_sub_query.append("resources.type= %(type)s") + pg_query = f"""\ + SELECT COALESCE(AVG(resources.duration),0) AS value + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + + row = cur.fetchone() + return row + + +def get_application_activity_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + results = row + results["chart"] = get_performance_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond + return results + + +def get_performance_avg_request_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) + request_constraints = [] + request_constraints_vals = {} + + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="resources", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s") + + pg_query = f"""WITH resources AS(SELECT resources.duration, resources.timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + AND resources.type = 'fetch' AND resources.duration>0 + {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} + ) + SELECT generated_timestamp AS timestamp, + COALESCE(AVG(resources.duration),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT resources.duration + FROM resources + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **request_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() + + return rows + + +def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + results["chart"] = __get_page_metrics_avg_dom_content_load_start_chart(cur, project_id, startTimestamp, + endTimestamp, **args) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond + return results + + +def __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("pages.timestamp>=%(startTimestamp)s") + pg_sub_query.append("pages.timestamp<%(endTimestamp)s") + pg_sub_query.append("pages.dom_content_loaded_time > 0") + pg_query = f"""SELECT COALESCE(AVG(NULLIF(pages.dom_content_loaded_time, 0)), 0) AS value + FROM (SELECT pages.dom_content_loaded_time + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + ) AS pages;""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return row + + +def __get_page_metrics_avg_dom_content_load_start_chart(cur, project_id, startTimestamp, endTimestamp, density=19, + **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, + chart=False, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False, + chart=True, data=args, main_table="pages", time_column="timestamp", + duration=False) + pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s") + pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s") + pg_sub_query_subset.append("pages.dom_content_loaded_time > 0") + + pg_query = f"""WITH pages AS(SELECT pages.dom_content_loaded_time, pages.timestamp + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_subset)} + ) + SELECT generated_timestamp AS timestamp, + COALESCE(AVG(pages.dom_content_loaded_time),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT pages.dom_content_loaded_time + FROM pages + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS pages ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **__get_constraint_values(args)})) + rows = cur.fetchall() + return rows + + +def get_page_metrics_avg_first_contentful_pixel(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + rows = __get_page_metrics_avg_first_contentful_pixel(cur, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_page_metrics_avg_first_contentful_pixel(cur, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond + return results + + +def __get_page_metrics_avg_first_contentful_pixel(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("pages.timestamp>=%(startTimestamp)s") + pg_sub_query.append("pages.timestamp<%(endTimestamp)s") + pg_sub_query.append("pages.first_contentful_paint_time > 0") + pg_query = f"""SELECT COALESCE(AVG(NULLIF(pages.first_contentful_paint_time, 0)), 0) AS value + FROM (SELECT pages.first_contentful_paint_time + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + ) AS pages;""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + return rows + + +def get_user_activity_avg_visited_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_user_activity_avg_visited_pages(cur, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_user_activity_avg_visited_pages(cur, project_id, startTimestamp, endTimestamp, **args) + + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.count + return results + + +def __get_user_activity_avg_visited_pages(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + pg_query = f"""\ + SELECT COALESCE(CEIL(AVG(NULLIF(sessions.pages_count,0))),0) AS value + FROM public.sessions + WHERE {" AND ".join(pg_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return row + + +def get_user_activity_avg_session_duration(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args) + + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond + return results + + +def __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + pg_query = f"""\ + SELECT COALESCE(AVG(NULLIF(sessions.duration,0)),0) AS value + FROM public.sessions + WHERE {" AND ".join(pg_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return row + + +def get_top_metrics_avg_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(AVG(pages.response_time), 0) AS value + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND pages.timestamp >= %(startTimestamp)s + AND pages.timestamp < %(endTimestamp)s + AND pages.response_time > 0;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond + return helper.dict_to_camel_case(row) + + +def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(AVG(pages.first_paint_time), 0) AS value + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND pages.timestamp >= %(startTimestamp)s + AND pages.timestamp < %(endTimestamp)s + AND pages.first_paint_time > 0;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond + return helper.dict_to_camel_case(row) + + +def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, density=19, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + pg_sub_query_chart.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(AVG(pages.dom_content_loaded_time), 0) AS value + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND pages.timestamp >= %(startTimestamp)s + AND pages.timestamp < %(endTimestamp)s + AND pages.dom_content_loaded_time > 0;""" + params = {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(AVG(NULLIF(pages.dom_content_loaded_time,0)),0) AS value + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT dom_content_loaded_time + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS pages ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp ASC;""" + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + row["chart"] = helper.list_to_camel_case(rows), + row["unit"] = schemas.TemplatePredefinedUnits.millisecond + return helper.dict_to_camel_case(row) + + +def get_top_metrics_avg_till_first_bit(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(AVG(pages.ttfb), 0) AS value + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND pages.timestamp >= %(startTimestamp)s + AND pages.timestamp < %(endTimestamp)s + AND pages.ttfb > 0;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond + return helper.dict_to_camel_case(row) + + +def get_top_metrics_avg_time_to_interactive(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(AVG(pages.time_to_interactive), 0) AS value + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND pages.timestamp >= %(startTimestamp)s + AND pages.timestamp < %(endTimestamp)s + AND pages.time_to_interactive > 0;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.millisecond + return helper.dict_to_camel_case(row) + + +def get_top_metrics_count_requests(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COUNT(pages.session_id) AS value + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + row["unit"] = schemas.TemplatePredefinedUnits.count + return helper.dict_to_camel_case(row) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py new file mode 100644 index 000000000..a66324532 --- /dev/null +++ b/api/chalicelib/core/dashboards2.py @@ -0,0 +1,309 @@ +import json + +import schemas +from chalicelib.core import custom_metrics, dashboard +from chalicelib.utils import helper +from chalicelib.utils import pg_client +from chalicelib.utils.TimeUTC import TimeUTC + +CATEGORY_DESCRIPTION = { + 'overview': 'lorem ipsum', + 'custom': 'lorem cusipsum', + 'errors': 'lorem erripsum', + 'performance': 'lorem perfipsum', + 'resources': 'lorem resipsum' +} + + +def get_templates(project_id, user_id): + with pg_client.PostgresClient() as cur: + pg_query = cur.mogrify(f"""SELECT category, jsonb_agg(metrics ORDER BY name) AS widgets + FROM (SELECT * , default_config AS config + FROM metrics LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(metric_series.* ORDER BY index), '[]'::jsonb) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + WHERE deleted_at IS NULL + AND (project_id ISNULL OR (project_id = %(project_id)s AND (is_public OR user_id= %(userId)s))) + ) AS metrics + GROUP BY category + ORDER BY category;""", {"project_id": project_id, "userId": user_id}) + cur.execute(pg_query) + rows = cur.fetchall() + for r in rows: + r["description"] = CATEGORY_DESCRIPTION.get(r["category"], "") + for w in r["widgets"]: + w["created_at"] = TimeUTC.datetime_to_timestamp(w["created_at"]) + w["edited_at"] = TimeUTC.datetime_to_timestamp(w["edited_at"]) + return helper.list_to_camel_case(rows) + + +def create_dashboard(project_id, user_id, data: schemas.CreateDashboardSchema): + with pg_client.PostgresClient() as cur: + pg_query = f"""INSERT INTO dashboards(project_id, user_id, name, is_public, is_pinned) + VALUES(%(projectId)s, %(userId)s, %(name)s, %(is_public)s, %(is_pinned)s) + RETURNING *""" + params = {"userId": user_id, "projectId": project_id, **data.dict()} + if data.metrics is not None and len(data.metrics) > 0: + pg_query = f"""WITH dash AS ({pg_query}) + INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) + VALUES {",".join([f"((SELECT dashboard_id FROM dash),%(metric_id_{i})s, %(userId)s, (SELECT default_config FROM metrics WHERE metric_id=%(metric_id_{i})s)||%(config_{i})s)" for i in range(len(data.metrics))])} + RETURNING (SELECT dashboard_id FROM dash)""" + for i, m in enumerate(data.metrics): + params[f"metric_id_{i}"] = m + # params[f"config_{i}"] = schemas.AddWidgetToDashboardPayloadSchema.schema() \ + # .get("properties", {}).get("config", {}).get("default", {}) + # params[f"config_{i}"]["position"] = i + # params[f"config_{i}"] = json.dumps(params[f"config_{i}"]) + params[f"config_{i}"] = json.dumps({"position": i}) + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + if row is None: + return {"errors": ["something went wrong while creating the dashboard"]} + return {"data": get_dashboard(project_id=project_id, user_id=user_id, dashboard_id=row["dashboard_id"])} + + +def get_dashboards(project_id, user_id): + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT * + FROM dashboards + WHERE deleted_at ISNULL + AND project_id = %(projectId)s + AND (user_id = %(userId)s OR is_public);""" + params = {"userId": user_id, "projectId": project_id} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + return helper.list_to_camel_case(rows) + + +def get_dashboard(project_id, user_id, dashboard_id): + with pg_client.PostgresClient() as cur: + pg_query = """SELECT dashboards.*, all_metric_widgets.widgets AS widgets + FROM dashboards + LEFT JOIN LATERAL (SELECT COALESCE(JSONB_AGG(raw_metrics), '[]') AS widgets + FROM (SELECT dashboard_widgets.*, metrics.*, metric_series.series + FROM metrics + INNER JOIN dashboard_widgets USING (metric_id) + LEFT JOIN LATERAL (SELECT JSONB_AGG(metric_series.* ORDER BY index) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + WHERE dashboard_widgets.dashboard_id = dashboards.dashboard_id + AND metrics.deleted_at ISNULL + AND (metrics.project_id = %(projectId)s OR metrics.project_id ISNULL)) AS raw_metrics + ) AS all_metric_widgets ON (TRUE) + WHERE dashboards.deleted_at ISNULL + AND dashboards.project_id = %(projectId)s + AND dashboard_id = %(dashboard_id)s + AND (dashboards.user_id = %(userId)s OR is_public);""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id} + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + if row is not None: + for w in row["widgets"]: + row["created_at"] = TimeUTC.datetime_to_timestamp(w["created_at"]) + row["edited_at"] = TimeUTC.datetime_to_timestamp(w["edited_at"]) + return helper.dict_to_camel_case(row) + + +def delete_dashboard(project_id, user_id, dashboard_id): + with pg_client.PostgresClient() as cur: + pg_query = """UPDATE dashboards + SET deleted_at = timezone('utc'::text, now()) + WHERE dashboards.project_id = %(projectId)s + AND dashboard_id = %(dashboard_id)s + AND (dashboards.user_id = %(userId)s OR is_public);""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id} + cur.execute(cur.mogrify(pg_query, params)) + return {"data": {"success": True}} + + +def update_dashboard(project_id, user_id, dashboard_id, data: schemas.EditDashboardSchema): + with pg_client.PostgresClient() as cur: + pg_query = f"""UPDATE dashboards + SET name = %(name)s + {", is_public = %(is_public)s" if data.is_public is not None else ""} + {", is_pinned = %(is_pinned)s" if data.is_pinned is not None else ""} + WHERE dashboards.project_id = %(projectId)s + AND dashboard_id = %(dashboard_id)s + AND (dashboards.user_id = %(userId)s OR is_public)""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, **data.dict()} + if data.metrics is not None and len(data.metrics) > 0: + pg_query = f"""WITH dash AS ({pg_query}) + INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) + VALUES {",".join([f"(%(dashboard_id)s, %(metric_id_{i})s, %(userId)s, (SELECT default_config FROM metrics WHERE metric_id=%(metric_id_{i})s)||%(config_{i})s)" for i in range(len(data.metrics))])};""" + for i, m in enumerate(data.metrics): + params[f"metric_id_{i}"] = m + # params[f"config_{i}"] = schemas.AddWidgetToDashboardPayloadSchema.schema() \ + # .get("properties", {}).get("config", {}).get("default", {}) + # params[f"config_{i}"]["position"] = i + # params[f"config_{i}"] = json.dumps(params[f"config_{i}"]) + params[f"config_{i}"] = json.dumps({"position": i}) + + cur.execute(cur.mogrify(pg_query, params)) + + return get_dashboard(project_id=project_id, user_id=user_id, dashboard_id=dashboard_id) + + +def get_widget(project_id, user_id, dashboard_id, widget_id): + with pg_client.PostgresClient() as cur: + pg_query = """SELECT metrics.*, metric_series.series + FROM dashboard_widgets + INNER JOIN dashboards USING (dashboard_id) + INNER JOIN metrics USING (metric_id) + LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(metric_series.* ORDER BY index), '[]'::jsonb) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + WHERE dashboard_id = %(dashboard_id)s + AND widget_id = %(widget_id)s + AND (dashboards.is_public OR dashboards.user_id = %(userId)s) + AND dashboards.deleted_at IS NULL + AND metrics.deleted_at ISNULL + AND (metrics.project_id = %(projectId)s OR metrics.project_id ISNULL) + AND (metrics.is_public OR metrics.user_id = %(userId)s);""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, "widget_id": widget_id} + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def add_widget(project_id, user_id, dashboard_id, data: schemas.AddWidgetToDashboardPayloadSchema): + with pg_client.PostgresClient() as cur: + pg_query = """INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) + SELECT %(dashboard_id)s AS dashboard_id, %(metric_id)s AS metric_id, + %(userId)s AS user_id, (SELECT default_config FROM metrics WHERE metric_id=%(metric_id)s)||%(config)s::jsonb AS config + WHERE EXISTS(SELECT 1 FROM dashboards + WHERE dashboards.deleted_at ISNULL AND dashboards.project_id = %(projectId)s + AND dashboard_id = %(dashboard_id)s + AND (dashboards.user_id = %(userId)s OR is_public)) + RETURNING *;""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, **data.dict()} + params["config"] = json.dumps(data.config) + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def update_widget(project_id, user_id, dashboard_id, widget_id, data: schemas.UpdateWidgetPayloadSchema): + with pg_client.PostgresClient() as cur: + pg_query = """UPDATE dashboard_widgets + SET config= %(config)s + WHERE dashboard_id=%(dashboard_id)s AND widget_id=%(widget_id)s + RETURNING *;""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, + "widget_id": widget_id, **data.dict()} + params["config"] = json.dumps(data.config) + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def remove_widget(project_id, user_id, dashboard_id, widget_id): + with pg_client.PostgresClient() as cur: + pg_query = """DELETE FROM dashboard_widgets + WHERE dashboard_id=%(dashboard_id)s AND widget_id=%(widget_id)s;""" + params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, "widget_id": widget_id} + cur.execute(cur.mogrify(pg_query, params)) + return {"data": {"success": True}} + + +def pin_dashboard(project_id, user_id, dashboard_id): + with pg_client.PostgresClient() as cur: + pg_query = """UPDATE dashboards + SET is_pinned = FALSE + WHERE project_id=%(project_id)s; + UPDATE dashboards + SET is_pinned = True + WHERE dashboard_id=%(dashboard_id)s AND project_id=%(project_id)s AND deleted_at ISNULL + RETURNING *;""" + params = {"userId": user_id, "project_id": project_id, "dashboard_id": dashboard_id} + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def create_metric_add_widget(project_id, user_id, dashboard_id, data: schemas.CreateCustomMetricsSchema): + metric_id = custom_metrics.create(project_id=project_id, user_id=user_id, data=data, dashboard=True) + return add_widget(project_id=project_id, user_id=user_id, dashboard_id=dashboard_id, + data=schemas.AddWidgetToDashboardPayloadSchema(metricId=metric_id)) + + +PREDEFINED = {schemas.TemplatePredefinedKeys.count_sessions: dashboard.get_processed_sessions, + schemas.TemplatePredefinedKeys.avg_image_load_time: dashboard.get_application_activity_avg_image_load_time, + schemas.TemplatePredefinedKeys.avg_page_load_time: dashboard.get_application_activity_avg_page_load_time, + schemas.TemplatePredefinedKeys.avg_request_load_time: dashboard.get_application_activity_avg_request_load_time, + schemas.TemplatePredefinedKeys.avg_dom_content_load_start: dashboard.get_page_metrics_avg_dom_content_load_start, + schemas.TemplatePredefinedKeys.avg_first_contentful_pixel: dashboard.get_page_metrics_avg_first_contentful_pixel, + schemas.TemplatePredefinedKeys.avg_visited_pages: dashboard.get_user_activity_avg_visited_pages, + schemas.TemplatePredefinedKeys.avg_session_duration: dashboard.get_user_activity_avg_session_duration, + schemas.TemplatePredefinedKeys.avg_pages_dom_buildtime: dashboard.get_pages_dom_build_time, + schemas.TemplatePredefinedKeys.avg_pages_response_time: dashboard.get_pages_response_time, + schemas.TemplatePredefinedKeys.avg_response_time: dashboard.get_top_metrics_avg_response_time, + schemas.TemplatePredefinedKeys.avg_first_paint: dashboard.get_top_metrics_avg_first_paint, + schemas.TemplatePredefinedKeys.avg_dom_content_loaded: dashboard.get_top_metrics_avg_dom_content_loaded, + schemas.TemplatePredefinedKeys.avg_till_first_bit: dashboard.get_top_metrics_avg_till_first_bit, + schemas.TemplatePredefinedKeys.avg_time_to_interactive: dashboard.get_top_metrics_avg_time_to_interactive, + schemas.TemplatePredefinedKeys.count_requests: dashboard.get_top_metrics_count_requests, + schemas.TemplatePredefinedKeys.avg_time_to_render: dashboard.get_time_to_render, + schemas.TemplatePredefinedKeys.avg_used_js_heap_size: dashboard.get_memory_consumption, + schemas.TemplatePredefinedKeys.avg_cpu: dashboard.get_avg_cpu, + schemas.TemplatePredefinedKeys.avg_fps: dashboard.get_avg_fps, + schemas.TemplatePredefinedKeys.impacted_sessions_by_js_errors: dashboard.get_impacted_sessions_by_js_errors, + schemas.TemplatePredefinedKeys.domains_errors_4xx: dashboard.get_domains_errors_4xx, + schemas.TemplatePredefinedKeys.domains_errors_5xx: dashboard.get_domains_errors_5xx, + schemas.TemplatePredefinedKeys.errors_per_domains: dashboard.get_errors_per_domains, + schemas.TemplatePredefinedKeys.calls_errors: dashboard.get_calls_errors, + schemas.TemplatePredefinedKeys.errors_by_type: dashboard.get_errors_per_type, + schemas.TemplatePredefinedKeys.errors_by_origin: dashboard.get_resources_by_party, + schemas.TemplatePredefinedKeys.speed_index_by_location: dashboard.get_speed_index_location, + schemas.TemplatePredefinedKeys.slowest_domains: dashboard.get_slowest_domains, + schemas.TemplatePredefinedKeys.sessions_per_browser: dashboard.get_sessions_per_browser, + schemas.TemplatePredefinedKeys.time_to_render: dashboard.get_time_to_render, + schemas.TemplatePredefinedKeys.impacted_sessions_by_slow_pages: dashboard.get_impacted_sessions_by_slow_pages, + schemas.TemplatePredefinedKeys.memory_consumption: dashboard.get_memory_consumption, + schemas.TemplatePredefinedKeys.cpu_load: dashboard.get_avg_cpu, + schemas.TemplatePredefinedKeys.frame_rate: dashboard.get_avg_fps, + schemas.TemplatePredefinedKeys.crashes: dashboard.get_crashes, + schemas.TemplatePredefinedKeys.resources_vs_visually_complete: dashboard.get_resources_vs_visually_complete, + schemas.TemplatePredefinedKeys.pages_dom_buildtime: dashboard.get_pages_dom_build_time, + schemas.TemplatePredefinedKeys.pages_response_time: dashboard.get_pages_response_time, + schemas.TemplatePredefinedKeys.pages_response_time_distribution: dashboard.get_pages_response_time_distribution, + schemas.TemplatePredefinedKeys.missing_resources: dashboard.get_missing_resources_trend, + schemas.TemplatePredefinedKeys.slowest_resources: dashboard.get_slowest_resources, + schemas.TemplatePredefinedKeys.resources_fetch_time: dashboard.get_resources_loading_time, + schemas.TemplatePredefinedKeys.resource_type_vs_response_end: dashboard.resource_type_vs_response_end, + schemas.TemplatePredefinedKeys.resources_count_by_type: dashboard.get_resources_count_by_type, + } + + +def get_predefined_metric(key: schemas.TemplatePredefinedKeys, project_id: int, data: dict): + return PREDEFINED.get(key, lambda *args: None)(project_id=project_id, **data) + + +def make_chart_metrics(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema): + raw_metric = custom_metrics.get_with_template(metric_id=metric_id, project_id=project_id, user_id=user_id, + include_dashboard=False) + if raw_metric is None: + return None + metric = schemas.CustomMetricAndTemplate = schemas.CustomMetricAndTemplate.parse_obj(raw_metric) + if metric.is_template: + return get_predefined_metric(key=metric.predefined_key, project_id=project_id, data=data.dict()) + else: + return custom_metrics.make_chart(project_id=project_id, user_id=user_id, metric_id=metric_id, data=data, + metric=raw_metric) + + +def make_chart_widget(dashboard_id, project_id, user_id, widget_id, data: schemas.CustomMetricChartPayloadSchema): + raw_metric = get_widget(widget_id=widget_id, project_id=project_id, user_id=user_id, dashboard_id=dashboard_id) + if raw_metric is None: + return None + metric = schemas.CustomMetricAndTemplate = schemas.CustomMetricAndTemplate.parse_obj(raw_metric) + if metric.is_template: + return get_predefined_metric(key=metric.predefined_key, project_id=project_id, data=data.dict()) + else: + return custom_metrics.make_chart(project_id=project_id, user_id=user_id, metric_id=raw_metric["metricId"], + data=data, metric=raw_metric) diff --git a/api/chalicelib/core/insights.py b/api/chalicelib/core/insights.py index 08adfd3ca..5b3894606 100644 --- a/api/chalicelib/core/insights.py +++ b/api/chalicelib/core/insights.py @@ -1,11 +1,8 @@ import schemas -from chalicelib.core import sessions_metas +from chalicelib.core.dashboard import __get_constraints, __get_constraint_values from chalicelib.utils import helper, dev from chalicelib.utils import pg_client from chalicelib.utils.TimeUTC import TimeUTC -from chalicelib.utils.metrics_helper import __get_step_size -import math -from chalicelib.core.dashboard import __get_constraints, __get_constraint_values def __transform_journey(rows): @@ -930,4 +927,4 @@ def search(text, feature_type, project_id, platform=None): rows = cur.fetchall() else: return [] - return [helper.dict_to_camel_case(row) for row in rows] \ No newline at end of file + return [helper.dict_to_camel_case(row) for row in rows] diff --git a/api/chalicelib/core/integration_jira_cloud.py b/api/chalicelib/core/integration_jira_cloud.py index ea9c6c24e..7d8c956cf 100644 --- a/api/chalicelib/core/integration_jira_cloud.py +++ b/api/chalicelib/core/integration_jira_cloud.py @@ -15,10 +15,17 @@ class JIRAIntegration(integration_base.BaseIntegration): # TODO: enable super-constructor when OAuth is done # super(JIRAIntegration, self).__init__(jwt, user_id, JIRACloudIntegrationProxy) self._user_id = user_id - i = self.get() - if i is None: + self.integration = self.get() + if self.integration is None: return - self.issue_handler = JIRACloudIntegrationIssue(token=i["token"], username=i["username"], url=i["url"]) + self.integration["valid"] = True + try: + self.issue_handler = JIRACloudIntegrationIssue(token=self.integration["token"], + username=self.integration["username"], + url=self.integration["url"]) + except Exception as e: + self.issue_handler = None + self.integration["valid"] = False @property def provider(self): @@ -37,10 +44,10 @@ class JIRAIntegration(integration_base.BaseIntegration): return helper.dict_to_camel_case(cur.fetchone()) def get_obfuscated(self): - integration = self.get() - if integration is None: + if self.integration is None: return None - integration["token"] = obfuscate_string(integration["token"]) + integration = dict(self.integration) + integration["token"] = obfuscate_string(self.integration["token"]) integration["provider"] = self.provider.lower() return integration @@ -90,14 +97,13 @@ class JIRAIntegration(integration_base.BaseIntegration): return {"state": "success"} def add_edit(self, data): - s = self.get() - if s is not None: + if self.integration is not None: return self.update( changes={ "username": data["username"], "token": data["token"] \ if data.get("token") and len(data["token"]) > 0 and data["token"].find("***") == -1 \ - else s["token"], + else self.integration["token"], "url": data["url"] }, obfuscate=True diff --git a/api/chalicelib/core/integrations_manager.py b/api/chalicelib/core/integrations_manager.py index fca271870..ef63a7d96 100644 --- a/api/chalicelib/core/integrations_manager.py +++ b/api/chalicelib/core/integrations_manager.py @@ -36,7 +36,10 @@ def get_integration(tenant_id, user_id, tool=None): if tool not in SUPPORTED_TOOLS: return {"errors": [f"issue tracking tool not supported yet, available: {SUPPORTED_TOOLS}"]}, None if tool == integration_jira_cloud.PROVIDER: - return None, integration_jira_cloud.JIRAIntegration(tenant_id=tenant_id, user_id=user_id) + integration = integration_jira_cloud.JIRAIntegration(tenant_id=tenant_id, user_id=user_id) + if integration.integration is not None and not integration.integration.get("valid", True): + return {"errors": ["JIRA: connexion issue/unauthorized"]}, integration + return None, integration elif tool == integration_github.PROVIDER: return None, integration_github.GitHubIntegration(tenant_id=tenant_id, user_id=user_id) return {"errors": ["lost integration"]}, None diff --git a/api/chalicelib/core/projects.py b/api/chalicelib/core/projects.py index c5ae912aa..e4ac36ad8 100644 --- a/api/chalicelib/core/projects.py +++ b/api/chalicelib/core/projects.py @@ -57,7 +57,7 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st cur.execute(f"""\ SELECT - s.project_id, s.name, s.project_key + s.project_id, s.name, s.project_key, s.save_request_payloads {',s.gdpr' if gdpr else ''} {',COALESCE((SELECT TRUE FROM public.sessions WHERE sessions.project_id = s.project_id LIMIT 1), FALSE) AS recorded' if recorded else ''} {',stack_integrations.count>0 AS stack_integrations' if stack_integrations else ''} @@ -65,27 +65,26 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st FROM public.projects AS s {'LEFT JOIN LATERAL (SELECT COUNT(*) AS count FROM public.integrations WHERE s.project_id = integrations.project_id LIMIT 1) AS stack_integrations ON TRUE' if stack_integrations else ''} WHERE s.deleted_at IS NULL - ORDER BY s.project_id;""" - ) + ORDER BY s.project_id;""") rows = cur.fetchall() if recording_state: project_ids = [f'({r["project_id"]})' for r in rows] - query = f"""SELECT projects.project_id, COALESCE(MAX(start_ts), 0) AS last - FROM (VALUES {",".join(project_ids)}) AS projects(project_id) - LEFT JOIN sessions USING (project_id) - GROUP BY project_id;""" - cur.execute( - query=query - ) + query = cur.mogrify(f"""SELECT projects.project_id, COALESCE(MAX(start_ts), 0) AS last + FROM (VALUES {",".join(project_ids)}) AS projects(project_id) + LEFT JOIN sessions USING (project_id) + WHERE sessions.start_ts >= %(startDate)s AND sessions.start_ts <= %(endDate)s + GROUP BY project_id;""", + {"startDate": TimeUTC.now(delta_days=-3), "endDate": TimeUTC.now(delta_days=1)}) + + cur.execute(query=query) status = cur.fetchall() for r in rows: + r["status"] = "red" for s in status: if s["project_id"] == r["project_id"]: - if s["last"] < TimeUTC.now(-2): - r["status"] = "red" - elif s["last"] < TimeUTC.now(-1): + if TimeUTC.now(-2) <= s["last"] < TimeUTC.now(-1): r["status"] = "yellow" - else: + elif s["last"] >= TimeUTC.now(-1): r["status"] = "green" break @@ -109,7 +108,8 @@ def get_project(tenant_id, project_id, include_last_session=False, include_gdpr= SELECT s.project_id, s.project_key, - s.name + s.name, + s.save_request_payloads {",(SELECT max(ss.start_ts) FROM public.sessions AS ss WHERE ss.project_id = %(project_id)s) AS last_recorded_session_at" if include_last_session else ""} {',s.gdpr' if include_gdpr else ''} {tracker_query} @@ -244,7 +244,8 @@ def get_project_key(project_id): where project_id =%(project_id)s AND deleted_at ISNULL;""", {"project_id": project_id}) ) - return cur.fetchone()["project_key"] + project = cur.fetchone() + return project["project_key"] if project is not None else None def get_capture_status(project_id): @@ -280,3 +281,13 @@ def update_capture_status(project_id, changes): ) return changes + + +def get_projects_ids(tenant_id): + with pg_client.PostgresClient() as cur: + cur.execute(f"""SELECT s.project_id + FROM public.projects AS s + WHERE s.deleted_at IS NULL + ORDER BY s.project_id;""") + rows = cur.fetchall() + return [r["project_id"] for r in rows] diff --git a/api/chalicelib/core/resources.py b/api/chalicelib/core/resources.py index 6a7e395f8..d85e56b6f 100644 --- a/api/chalicelib/core/resources.py +++ b/api/chalicelib/core/resources.py @@ -1,23 +1,23 @@ from chalicelib.utils import helper, pg_client -def get_by_session_id(session_id): +def get_by_session_id(session_id, project_id): with pg_client.PostgresClient() as cur: ch_query = """\ SELECT timestamp AS datetime, url, type, - duration, + resources.duration AS duration, ttfb, header_size, encoded_body_size, decoded_body_size, success, COALESCE(status, CASE WHEN success THEN 200 END) AS status - FROM events.resources - WHERE session_id = %(session_id)s;""" - params = {"session_id": session_id} + FROM events.resources INNER JOIN sessions USING (session_id) + WHERE session_id = %(session_id)s AND project_id= %(project_id)s;""" + params = {"session_id": session_id, "project_id": project_id} cur.execute(cur.mogrify(ch_query, params)) rows = cur.fetchall() return helper.list_to_camel_case(rows) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 1903cc08b..c0fdf57bd 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -39,7 +39,8 @@ def __group_metadata(session, project_metadata): return meta -def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_viewed=False, group_metadata=False): +def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_viewed=False, group_metadata=False, + live=True): with pg_client.PostgresClient() as cur: extra_query = [] if include_fav_viewed: @@ -93,13 +94,13 @@ def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_ data['userEvents'] = events.get_customs_by_sessionId2_pg(project_id=project_id, session_id=session_id) data['mobsUrl'] = sessions_mobs.get_web(sessionId=session_id) - data['resources'] = resources.get_by_session_id(session_id=session_id) + data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id) data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['issues'] = issues.get_by_session_id(session_id=session_id) - data['live'] = assist.is_live(project_id=project_id, - session_id=session_id, - project_key=data["projectKey"]) + data['live'] = live and assist.is_live(project_id=project_id, + session_id=session_id, + project_key=data["projectKey"]) data["inDB"] = True return data else: @@ -233,20 +234,19 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, e data.order = "DESC" sort = 'session_id' if data.sort is not None and data.sort != "session_id": - sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) - else: - sort = 'session_id' + # sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) + sort = helper.key_to_snake_case(data.sort) meta_keys = metadata.get(project_id=project_id) main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, COALESCE(JSONB_AGG(full_sessions) FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions - FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY issue_score DESC, {sort} {data.order}, session_id desc) AS rn + FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY {sort} {data.order}, issue_score DESC) AS rn FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} {query_part} ORDER BY s.session_id desc) AS filtred_sessions - ORDER BY issue_score DESC, {sort} {data.order}) AS full_sessions;""", + ORDER BY {sort} {data.order}, issue_score DESC) AS full_sessions;""", full_args) # print("--------------------") # print(main_query) @@ -280,9 +280,9 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, e for i, s in enumerate(sessions): sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ if sessions[i][f'metadata_{k["index"]}'] is not None} - if not data.group_by_user and data.sort is not None and data.sort != "session_id": - sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data.sort)], - reverse=data.order.upper() == "DESC") + # if not data.group_by_user and data.sort is not None and data.sort != "session_id": + # sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data.sort)], + # reverse=data.order.upper() == "DESC") return { 'total': total, 'sessions': helper.list_to_camel_case(sessions) diff --git a/api/chalicelib/core/telemetry.py b/api/chalicelib/core/telemetry.py index 48f403f57..28eb97f73 100644 --- a/api/chalicelib/core/telemetry.py +++ b/api/chalicelib/core/telemetry.py @@ -30,7 +30,7 @@ def compute(): RETURNING *,(SELECT email FROM public.users WHERE role='owner' LIMIT 1);""" ) data = cur.fetchone() - requests.post('https://parrot.asayer.io/os/telemetry', json={"stats": [process_data(data)]}) + requests.post('https://api.openreplay.com/os/telemetry', json={"stats": [process_data(data)]}) def new_client(): @@ -40,4 +40,4 @@ def new_client(): (SELECT email FROM public.users WHERE role='owner' LIMIT 1) AS email FROM public.tenants;""") data = cur.fetchone() - requests.post('https://parrot.asayer.io/os/signup', json=process_data(data)) + requests.post('https://api.openreplay.com/os/signup', json=process_data(data)) diff --git a/api/chalicelib/core/users.py b/api/chalicelib/core/users.py index b4ac0f869..ceada34f8 100644 --- a/api/chalicelib/core/users.py +++ b/api/chalicelib/core/users.py @@ -571,7 +571,6 @@ def auth_exists(user_id, tenant_id, jwt_iat, jwt_aud): ) -@dev.timed def authenticate(email, password, for_change_password=False, for_plugin=False): with pg_client.PostgresClient() as cur: query = cur.mogrify( diff --git a/api/chalicelib/utils/TimeUTC.py b/api/chalicelib/utils/TimeUTC.py index bac7a027f..d399e1651 100644 --- a/api/chalicelib/utils/TimeUTC.py +++ b/api/chalicelib/utils/TimeUTC.py @@ -88,13 +88,18 @@ class TimeUTC: return datetime.utcfromtimestamp(ts // 1000).strftime(fmt) @staticmethod - def human_to_timestamp(ts, pattern): + def human_to_timestamp(ts, pattern="%Y-%m-%dT%H:%M:%S.%f"): return int(datetime.strptime(ts, pattern).timestamp() * 1000) @staticmethod def datetime_to_timestamp(date): if date is None: return None + if isinstance(date, str): + fp = date.find(".") + if fp > 0: + date += '0' * (6 - len(date[fp + 1:])) + date = datetime.fromisoformat(date) return int(datetime.timestamp(date) * 1000) @staticmethod diff --git a/api/chalicelib/utils/jira_client.py b/api/chalicelib/utils/jira_client.py index d3b637373..b1734660c 100644 --- a/api/chalicelib/utils/jira_client.py +++ b/api/chalicelib/utils/jira_client.py @@ -5,22 +5,24 @@ import requests from jira import JIRA from jira.exceptions import JIRAError from requests.auth import HTTPBasicAuth +from starlette import status +from starlette.exceptions import HTTPException fields = "id, summary, description, creator, reporter, created, assignee, status, updated, comment, issuetype, labels" class JiraManager: - # retries = 5 retries = 0 def __init__(self, url, username, password, project_id=None): self._config = {"JIRA_PROJECT_ID": project_id, "JIRA_URL": url, "JIRA_USERNAME": username, "JIRA_PASSWORD": password} try: - self._jira = JIRA({'server': url}, basic_auth=(username, password), logging=True, max_retries=1) + self._jira = JIRA(url, basic_auth=(username, password), logging=True, max_retries=1) except Exception as e: print("!!! JIRA AUTH ERROR") print(e) + raise e def set_jira_project_id(self, project_id): self._config["JIRA_PROJECT_ID"] = project_id @@ -33,8 +35,8 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_projects() - print(f"=>Error {e.text}") - raise e + print(f"=>Exception {e.text}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") projects_dict_list = [] for project in projects: projects_dict_list.append(self.__parser_project_info(project)) @@ -49,8 +51,8 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_project() - print(f"=>Error {e.text}") - raise e + print(f"=>Exception {e.text}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") return self.__parser_project_info(project) def get_issues(self, sql: str, offset: int = 0): @@ -65,8 +67,8 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_issues(sql, offset) - print(f"=>Error {e.text}") - raise e + print(f"=>Exception {e.text}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") issue_dict_list = [] for issue in issues: @@ -85,8 +87,8 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_issue(issue_id) - print(f"=>Error {e.text}") - raise e + print(f"=>Exception {e.text}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") return self.__parser_issue_info(issue) def get_issue_v3(self, issue_id: str): @@ -105,8 +107,8 @@ class JiraManager: if self.retries > 0: time.sleep(1) return self.get_issue_v3(issue_id) - print(f"=>Error {e}") - raise e + print(f"=>Exception {e}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: get issue error") return self.__parser_issue_info(issue.json()) def create_issue(self, issue_dict): @@ -119,8 +121,8 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.create_issue(issue_dict) - print(f"=>Error {e.text}") - raise e + print(f"=>Exception {e.text}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def close_issue(self, issue): try: @@ -131,8 +133,8 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.close_issue(issue) - print(f"=>Error {e.text}") - raise e + print(f"=>Exception {e.text}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def assign_issue(self, issue_id, account_id) -> bool: try: @@ -142,8 +144,8 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.assign_issue(issue_id, account_id) - print(f"=>Error {e.text}") - raise e + print(f"=>Exception {e.text}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def add_comment(self, issue_id: str, comment: str): try: @@ -153,8 +155,8 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.add_comment(issue_id, comment) - print(f"=>Error {e.text}") - raise e + print(f"=>Exception {e.text}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") return self.__parser_comment_info(comment) def add_comment_v3(self, issue_id: str, comment: str): @@ -190,8 +192,8 @@ class JiraManager: if self.retries > 0: time.sleep(1) return self.add_comment_v3(issue_id, comment) - print(f"=>Error {e}") - raise e + print(f"=>Exception {e}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: comment error") return self.__parser_comment_info(comment_response.json()) def get_comments(self, issueKey): @@ -206,8 +208,8 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_comments(issueKey) - print(f"=>Error {e.text}") - raise e + print(f"=>Exception {e.text}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") def get_meta(self): meta = {} @@ -217,14 +219,16 @@ class JiraManager: def get_assignable_users(self): try: - users = self._jira.search_assignable_users_for_issues('', project=self._config['JIRA_PROJECT_ID']) + users = self._jira.search_assignable_users_for_issues(project=self._config['JIRA_PROJECT_ID'], query="*") except JIRAError as e: self.retries -= 1 if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_assignable_users() - print(f"=>Error {e.text}") - raise e + print(f"=>Exception {e.text}") + if e.status_code == 401: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="JIRA: 401 Unauthorized") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") users_dict = [] for user in users: users_dict.append({ @@ -244,8 +248,8 @@ class JiraManager: if (e.status_code // 100) == 4 and self.retries > 0: time.sleep(1) return self.get_issue_types() - print(f"=>Error {e.text}") - raise e + print(f"=>Exception {e.text}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"JIRA: {e.text}") types_dict = [] for type in types: if not type.subtask and not type.name.lower() == "epic": diff --git a/api/chalicelib/utils/pg_client.py b/api/chalicelib/utils/pg_client.py index 6e4118689..3d60dda5c 100644 --- a/api/chalicelib/utils/pg_client.py +++ b/api/chalicelib/utils/pg_client.py @@ -1,3 +1,4 @@ +import time from threading import Semaphore import psycopg2 @@ -9,7 +10,8 @@ _PG_CONFIG = {"host": config("pg_host"), "database": config("pg_dbname"), "user": config("pg_user"), "password": config("pg_password"), - "port": config("pg_port", cast=int)} + "port": config("pg_port", cast=int), + "application_name": config("APP_NAME", default="PY")} PG_CONFIG = dict(_PG_CONFIG) if config("pg_timeout", cast=int, default=0) > 0: PG_CONFIG["options"] = f"-c statement_timeout={config('pg_timeout', cast=int) * 1000}" @@ -36,9 +38,14 @@ class ORThreadedConnectionPool(psycopg2.pool.ThreadedConnectionPool): postgreSQL_pool: ORThreadedConnectionPool = None +RETRY_MAX = config("PG_RETRY_MAX", cast=int, default=50) +RETRY_INTERVAL = config("PG_RETRY_INTERVAL", cast=int, default=2) +RETRY = 0 + def make_pool(): global postgreSQL_pool + global RETRY if postgreSQL_pool is not None: try: postgreSQL_pool.closeall() @@ -50,7 +57,13 @@ def make_pool(): print("Connection pool created successfully") except (Exception, psycopg2.DatabaseError) as error: print("Error while connecting to PostgreSQL", error) - raise error + if RETRY < RETRY_MAX: + RETRY += 1 + print(f"waiting for {RETRY_INTERVAL}s before retry n°{RETRY}") + time.sleep(RETRY_INTERVAL) + make_pool() + else: + raise error make_pool() @@ -64,6 +77,8 @@ class PostgresClient: def __init__(self, long_query=False): self.long_query = long_query if long_query: + long_config = dict(_PG_CONFIG) + long_config["application_name"] += "-LONG" self.connection = psycopg2.connect(**_PG_CONFIG) else: self.connection = postgreSQL_pool.getconn() diff --git a/api/entrypoint.sh b/api/entrypoint.sh index a092737be..fe5912f0f 100755 --- a/api/entrypoint.sh +++ b/api/entrypoint.sh @@ -1,2 +1,5 @@ #!/bin/bash +cd sourcemap-reader +nohup npm start &> /tmp/sourcemap-reader.log & +cd .. uvicorn app:app --host 0.0.0.0 --reload diff --git a/api/requirements.txt b/api/requirements.txt index 4af962f4f..198b535dd 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -4,11 +4,11 @@ boto3==1.16.1 pyjwt==1.7.1 psycopg2-binary==2.8.6 elasticsearch==7.9.1 -jira==2.0.0 +jira==3.1.1 -fastapi==0.74.1 +fastapi==0.75.0 uvicorn[standard]==0.17.5 python-decouple==3.6 pydantic[email]==1.8.2 diff --git a/api/routers/app/__init__.py b/api/routers/app/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/api/routers/base.py b/api/routers/base.py index ff7fe165f..5c665b2d1 100644 --- a/api/routers/base.py +++ b/api/routers/base.py @@ -2,11 +2,13 @@ from fastapi import APIRouter, Depends from auth.auth_apikey import APIKeyAuth from auth.auth_jwt import JWTAuth +from auth.auth_project import ProjectAuthorizer from or_dependencies import ORRoute def get_routers() -> (APIRouter, APIRouter, APIRouter): public_app = APIRouter(route_class=ORRoute) - app = APIRouter(dependencies=[Depends(JWTAuth())], route_class=ORRoute) - app_apikey = APIRouter(dependencies=[Depends(APIKeyAuth())], route_class=ORRoute) + app = APIRouter(dependencies=[Depends(JWTAuth()), Depends(ProjectAuthorizer("projectId"))], route_class=ORRoute) + app_apikey = APIRouter(dependencies=[Depends(APIKeyAuth()), Depends(ProjectAuthorizer("projectKey"))], + route_class=ORRoute) return public_app, app, app_apikey diff --git a/api/routers/core.py b/api/routers/core.py index 97a749429..813577b88 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -1,7 +1,7 @@ from typing import Union from decouple import config -from fastapi import Depends, Body +from fastapi import Depends, Body, BackgroundTasks import schemas from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assignments, projects, \ @@ -21,8 +21,10 @@ from routers.base import get_routers public_app, app, app_apikey = get_routers() +@app.get('/{projectId}/sessions/{sessionId}', tags=["sessions"]) @app.get('/{projectId}/sessions2/{sessionId}', tags=["sessions"]) -def get_session2(projectId: int, sessionId: Union[int, str], context: schemas.CurrentContext = Depends(OR_context)): +def get_session2(projectId: int, sessionId: Union[int, str], background_tasks: BackgroundTasks, + context: schemas.CurrentContext = Depends(OR_context)): if isinstance(sessionId, str): return {"errors": ["session not found"]} data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, user_id=context.user_id, @@ -30,12 +32,14 @@ def get_session2(projectId: int, sessionId: Union[int, str], context: schemas.Cu if data is None: return {"errors": ["session not found"]} if data.get("inDB"): - sessions_favorite_viewed.view_session(project_id=projectId, user_id=context.user_id, session_id=sessionId) + background_tasks.add_task(sessions_favorite_viewed.view_session, project_id=projectId, user_id=context.user_id, + session_id=sessionId) return { 'data': data } +@app.get('/{projectId}/sessions/{sessionId}/favorite', tags=["sessions"]) @app.get('/{projectId}/sessions2/{sessionId}/favorite', tags=["sessions"]) def add_remove_favorite_session2(projectId: int, sessionId: int, context: schemas.CurrentContext = Depends(OR_context)): @@ -44,6 +48,7 @@ def add_remove_favorite_session2(projectId: int, sessionId: int, session_id=sessionId)} +@app.get('/{projectId}/sessions/{sessionId}/assign', tags=["sessions"]) @app.get('/{projectId}/sessions2/{sessionId}/assign', tags=["sessions"]) def assign_session(projectId: int, sessionId, context: schemas.CurrentContext = Depends(OR_context)): data = sessions_assignments.get_by_session(project_id=projectId, session_id=sessionId, @@ -56,6 +61,7 @@ def assign_session(projectId: int, sessionId, context: schemas.CurrentContext = } +@app.get('/{projectId}/sessions/{sessionId}/errors/{errorId}/sourcemaps', tags=["sessions", "sourcemaps"]) @app.get('/{projectId}/sessions2/{sessionId}/errors/{errorId}/sourcemaps', tags=["sessions", "sourcemaps"]) def get_error_trace(projectId: int, sessionId: int, errorId: str, context: schemas.CurrentContext = Depends(OR_context)): @@ -67,6 +73,7 @@ def get_error_trace(projectId: int, sessionId: int, errorId: str, } +@app.get('/{projectId}/sessions/{sessionId}/assign/{issueId}', tags=["sessions", "issueTracking"]) @app.get('/{projectId}/sessions2/{sessionId}/assign/{issueId}', tags=["sessions", "issueTracking"]) def assign_session(projectId: int, sessionId: int, issueId: str, context: schemas.CurrentContext = Depends(OR_context)): @@ -79,6 +86,8 @@ def assign_session(projectId: int, sessionId: int, issueId: str, } +@app.post('/{projectId}/sessions/{sessionId}/assign/{issueId}/comment', tags=["sessions", "issueTracking"]) +@app.put('/{projectId}/sessions/{sessionId}/assign/{issueId}/comment', tags=["sessions", "issueTracking"]) @app.post('/{projectId}/sessions2/{sessionId}/assign/{issueId}/comment', tags=["sessions", "issueTracking"]) @app.put('/{projectId}/sessions2/{sessionId}/assign/{issueId}/comment', tags=["sessions", "issueTracking"]) def comment_assignment(projectId: int, sessionId: int, issueId: str, data: schemas.CommentAssignmentSchema = Body(...), @@ -387,7 +396,7 @@ def delete_sumologic(projectId: int, context: schemas.CurrentContext = Depends(O def get_integration_status(context: schemas.CurrentContext = Depends(OR_context)): error, integration = integrations_manager.get_integration(tenant_id=context.tenant_id, user_id=context.user_id) - if error is not None: + if error is not None and integration is None: return {"data": {}} return {"data": integration.get_obfuscated()} @@ -399,7 +408,7 @@ def add_edit_jira_cloud(data: schemas.JiraGithubSchema = Body(...), error, integration = integrations_manager.get_integration(tool=integration_jira_cloud.PROVIDER, tenant_id=context.tenant_id, user_id=context.user_id) - if error is not None: + if error is not None and integration is None: return error data.provider = integration_jira_cloud.PROVIDER return {"data": integration.add_edit(data=data.dict())} @@ -422,7 +431,7 @@ def add_edit_github(data: schemas.JiraGithubSchema = Body(...), def delete_default_issue_tracking_tool(context: schemas.CurrentContext = Depends(OR_context)): error, integration = integrations_manager.get_integration(tenant_id=context.tenant_id, user_id=context.user_id) - if error is not None: + if error is not None and integration is None: return error return {"data": integration.delete()} @@ -825,6 +834,21 @@ def sessions_live(projectId: int, userId: str = None, context: schemas.CurrentCo return {'data': data} +@app.get('/{projectId}/assist/sessions/{sessionId}', tags=["assist"]) +def get_live_session(projectId: int, sessionId: str, background_tasks: BackgroundTasks, + context: schemas.CurrentContext = Depends(OR_context)): + data = assist.get_live_session_by_id(project_id=projectId, session_id=sessionId) + if data is None: + data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, + user_id=context.user_id, include_fav_viewed=True, group_metadata=True, live=False) + if data is None: + return {"errors": ["session not found"]} + if data.get("inDB"): + background_tasks.add_task(sessions_favorite_viewed.view_session, project_id=projectId, + user_id=context.user_id, session_id=sessionId) + return {'data': data} + + @app.post('/{projectId}/heatmaps/url', tags=["heatmaps"]) def get_heatmaps_by_url(projectId: int, data: schemas.GetHeatmapPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): @@ -889,12 +913,14 @@ def errors_stats(projectId: int, startTimestamp: int, endTimestamp: int, @app.get('/{projectId}/errors/{errorId}', tags=['errors']) -def errors_get_details(projectId: int, errorId: str, density24: int = 24, density30: int = 30, +def errors_get_details(projectId: int, errorId: str, background_tasks: BackgroundTasks, density24: int = 24, + density30: int = 30, context: schemas.CurrentContext = Depends(OR_context)): data = errors.get_details(project_id=projectId, user_id=context.user_id, error_id=errorId, **{"density24": density24, "density30": density30}) if data.get("data") is not None: - errors_favorite_viewed.viewed_error(project_id=projectId, user_id=context.user_id, error_id=errorId) + background_tasks.add_task(errors_favorite_viewed.viewed_error, project_id=projectId, user_id=context.user_id, + error_id=errorId) return data @@ -1065,78 +1091,6 @@ def change_client_password(data: schemas.EditUserPasswordSchema = Body(...), user_id=context.user_id) -@app.post('/{projectId}/custom_metrics/try', tags=["customMetrics"]) -@app.put('/{projectId}/custom_metrics/try', tags=["customMetrics"]) -def try_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.merged_live(project_id=projectId, data=data)} - - -@app.post('/{projectId}/custom_metrics', tags=["customMetrics"]) -@app.put('/{projectId}/custom_metrics', tags=["customMetrics"]) -def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - return custom_metrics.create(project_id=projectId, user_id=context.user_id, data=data) - - -@app.get('/{projectId}/custom_metrics', tags=["customMetrics"]) -def get_custom_metrics(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.get_all(project_id=projectId, user_id=context.user_id)} - - -@app.get('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) -def get_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): - data = custom_metrics.get(project_id=projectId, user_id=context.user_id, metric_id=metric_id) - if data is None: - return {"errors": ["custom metric not found"]} - return {"data": data} - - -@app.post('/{projectId}/custom_metrics/{metric_id}/sessions', tags=["customMetrics"]) -def get_custom_metric_sessions(projectId: int, metric_id: int, - data: schemas.CustomMetricSessionsPayloadSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - data = custom_metrics.get_sessions(project_id=projectId, user_id=context.user_id, metric_id=metric_id, data=data) - if data is None: - return {"errors": ["custom metric not found"]} - return {"data": data} - - -@app.post('/{projectId}/custom_metrics/{metric_id}/chart', tags=["customMetrics"]) -def get_custom_metric_chart(projectId: int, metric_id: int, data: schemas.CustomMetricChartPayloadSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - data = custom_metrics.make_chart(project_id=projectId, user_id=context.user_id, metric_id=metric_id, - data=data) - if data is None: - return {"errors": ["custom metric not found"]} - return {"data": data} - - -@app.post('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) -@app.put('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) -def update_custom_metric(projectId: int, metric_id: int, data: schemas.UpdateCustomMetricsSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - data = custom_metrics.update(project_id=projectId, user_id=context.user_id, metric_id=metric_id, data=data) - if data is None: - return {"errors": ["custom metric not found"]} - return {"data": data} - - -@app.post('/{projectId}/custom_metrics/{metric_id}/status', tags=["customMetrics"]) -@app.put('/{projectId}/custom_metrics/{metric_id}/status', tags=["customMetrics"]) -def update_custom_metric_state(projectId: int, metric_id: int, - data: schemas.UpdateCustomMetricsStatusSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - return { - "data": custom_metrics.change_state(project_id=projectId, user_id=context.user_id, metric_id=metric_id, - status=data.active)} - - -@app.delete('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) -def delete_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.delete(project_id=projectId, user_id=context.user_id, metric_id=metric_id)} - - @app.post('/{projectId}/saved_search', tags=["savedSearch"]) @app.put('/{projectId}/saved_search', tags=["savedSearch"]) def add_saved_search(projectId: int, data: schemas.SavedSearchSchema = Body(...), diff --git a/api/routers/subs/dashboard.py b/api/routers/subs/dashboard.py index 169893693..e2d4ba268 100644 --- a/api/routers/subs/dashboard.py +++ b/api/routers/subs/dashboard.py @@ -325,22 +325,73 @@ def get_dashboard_resources_count_by_type(projectId: int, data: schemas.MetricPa @app.post('/{projectId}/dashboard/overview', tags=["dashboard", "metrics"]) @app.get('/{projectId}/dashboard/overview', tags=["dashboard", "metrics"]) def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body(...)): - return {"data": [ - *helper.explode_widget(key="count_sessions", - data=dashboard.get_processed_sessions(project_id=projectId, **data.dict())), + results = [ + {"key": "count_sessions", + "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, *helper.explode_widget(data={**dashboard.get_application_activity(project_id=projectId, **data.dict()), "chart": dashboard.get_performance(project_id=projectId, **data.dict()) .get("chart", [])}), *helper.explode_widget(data=dashboard.get_page_metrics(project_id=projectId, **data.dict())), *helper.explode_widget(data=dashboard.get_user_activity(project_id=projectId, **data.dict())), - *helper.explode_widget(data=dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict()), - key="avg_pages_dom_buildtime"), - *helper.explode_widget(data=dashboard.get_pages_response_time(project_id=projectId, **data.dict()), - key="avg_pages_response_time"), + {"key": "avg_pages_dom_buildtime", + "data": dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict())}, + {"key": "avg_pages_response_time", + "data": dashboard.get_pages_response_time(project_id=projectId, **data.dict()) + }, *helper.explode_widget(dashboard.get_top_metrics(project_id=projectId, **data.dict())), - *helper.explode_widget(data=dashboard.get_time_to_render(project_id=projectId, **data.dict()), - key="avg_time_to_render"), - *helper.explode_widget(dashboard.get_memory_consumption(project_id=projectId, **data.dict())), - *helper.explode_widget(dashboard.get_avg_cpu(project_id=projectId, **data.dict())), - *helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **data.dict())), - ]} + {"key": "avg_time_to_render", "data": dashboard.get_time_to_render(project_id=projectId, **data.dict())}, + {"key": "avg_used_js_heap_size", "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, + {"key": "avg_cpu", "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_fps, "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} + ] + results = sorted(results, key=lambda r: r["key"]) + return {"data": results} + + +@app.post('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) +@app.get('/{projectId}/dashboard/overview2', tags=["dashboard", "metrics"]) +def get_dashboard_group(projectId: int, data: schemas.MetricPayloadSchema = Body(...)): + results = [ + {"key": schemas.TemplatePredefinedKeys.count_sessions, + "data": dashboard.get_processed_sessions(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_image_load_time, + "data": dashboard.get_application_activity_avg_image_load_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_page_load_time, + "data": dashboard.get_application_activity_avg_page_load_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_request_load_time, + "data": dashboard.get_application_activity_avg_request_load_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_dom_content_load_start, + "data": dashboard.get_page_metrics_avg_dom_content_load_start(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_first_contentful_pixel, + "data": dashboard.get_page_metrics_avg_first_contentful_pixel(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_visited_pages, + "data": dashboard.get_user_activity_avg_visited_pages(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_session_duration, + "data": dashboard.get_user_activity_avg_session_duration(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_pages_dom_buildtime, + "data": dashboard.get_pages_dom_build_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_pages_response_time, + "data": dashboard.get_pages_response_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_response_time, + "data": dashboard.get_top_metrics_avg_response_time(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_first_paint, + "data": dashboard.get_top_metrics_avg_first_paint(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_dom_content_loaded, + "data": dashboard.get_top_metrics_avg_dom_content_loaded(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_till_first_bit, + "data": dashboard.get_top_metrics_avg_till_first_bit(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_time_to_interactive, + "data": dashboard.get_top_metrics_avg_time_to_interactive(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.count_requests, + "data": dashboard.get_top_metrics_count_requests(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_time_to_render, + "data": dashboard.get_time_to_render(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_used_js_heap_size, + "data": dashboard.get_memory_consumption(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_cpu, + "data": dashboard.get_avg_cpu(project_id=projectId, **data.dict())}, + {"key": schemas.TemplatePredefinedKeys.avg_fps, + "data": dashboard.get_avg_fps(project_id=projectId, **data.dict())} + ] + results = sorted(results, key=lambda r: r["key"]) + return {"data": results} diff --git a/api/routers/subs/metrics.py b/api/routers/subs/metrics.py new file mode 100644 index 000000000..0a806b146 --- /dev/null +++ b/api/routers/subs/metrics.py @@ -0,0 +1,181 @@ +from fastapi import Body, Depends + +import schemas +from chalicelib.core import dashboards2, custom_metrics +from or_dependencies import OR_context +from routers.base import get_routers + +public_app, app, app_apikey = get_routers() + + +@app.post('/{projectId}/dashboards', tags=["dashboard"]) +@app.put('/{projectId}/dashboards', tags=["dashboard"]) +def create_dashboards(projectId: int, data: schemas.CreateDashboardSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return dashboards2.create_dashboard(project_id=projectId, user_id=context.user_id, data=data) + + +@app.get('/{projectId}/dashboards', tags=["dashboard"]) +def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.get_dashboards(project_id=projectId, user_id=context.user_id)} + + +@app.get('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) +def get_dashboard(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): + data = dashboards2.get_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId) + if data is None: + return {"errors": ["dashboard not found"]} + return {"data": data} + + +@app.post('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) +@app.put('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) +def update_dashboard(projectId: int, dashboardId: int, data: schemas.EditDashboardSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.update_dashboard(project_id=projectId, user_id=context.user_id, + dashboard_id=dashboardId, data=data)} + + +@app.delete('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"]) +def delete_dashboard(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): + return dashboards2.delete_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId) + + +@app.get('/{projectId}/dashboards/{dashboardId}/pin', tags=["dashboard"]) +def pin_dashboard(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.pin_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId)} + + +@app.post('/{projectId}/dashboards/{dashboardId}/widgets', tags=["dashboard"]) +@app.put('/{projectId}/dashboards/{dashboardId}/widgets', tags=["dashboard"]) +def add_widget_to_dashboard(projectId: int, dashboardId: int, + data: schemas.AddWidgetToDashboardPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.add_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, + data=data)} + + +@app.post('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard"]) +@app.put('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard"]) +def create_metric_and_add_to_dashboard(projectId: int, dashboardId: int, + data: schemas.CreateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.create_metric_add_widget(project_id=projectId, user_id=context.user_id, + dashboard_id=dashboardId, data=data)} + + +@app.post('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) +@app.put('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) +def update_widget_in_dashboard(projectId: int, dashboardId: int, widgetId: int, + data: schemas.UpdateWidgetPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return dashboards2.update_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, + widget_id=widgetId, data=data) + + +@app.delete('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"]) +def remove_widget_from_dashboard(projectId: int, dashboardId: int, widgetId: int, + context: schemas.CurrentContext = Depends(OR_context)): + return dashboards2.remove_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, + widget_id=widgetId) + + +@app.post('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}/chart', tags=["dashboard"]) +def get_widget_chart(projectId: int, dashboardId: int, widgetId: int, + data: schemas.CustomMetricChartPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + data = dashboards2.make_chart_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId, + widget_id=widgetId, data=data) + if data is None: + return {"errors": ["widget not found"]} + return {"data": data} + + +@app.get('/{projectId}/metrics/templates', tags=["dashboard"]) +def get_templates(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": dashboards2.get_templates(project_id=projectId, user_id=context.user_id)} + + +@app.post('/{projectId}/metrics/try', tags=["dashboard"]) +@app.put('/{projectId}/metrics/try', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics/try', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics/try', tags=["customMetrics"]) +def try_custom_metric(projectId: int, data: schemas.TryCustomMetricsPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.merged_live(project_id=projectId, data=data)} + + +@app.post('/{projectId}/metrics', tags=["dashboard"]) +@app.put('/{projectId}/metrics', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics', tags=["customMetrics"]) +def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return custom_metrics.create(project_id=projectId, user_id=context.user_id, data=data) + + +@app.get('/{projectId}/metrics', tags=["dashboard"]) +@app.get('/{projectId}/custom_metrics', tags=["customMetrics"]) +def get_custom_metrics(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.get_all(project_id=projectId, user_id=context.user_id)} + + +@app.get('/{projectId}/metrics/{metric_id}', tags=["dashboard"]) +@app.get('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def get_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): + data = custom_metrics.get(project_id=projectId, user_id=context.user_id, metric_id=metric_id) + if data is None: + return {"errors": ["custom metric not found"]} + return {"data": data} + + +@app.post('/{projectId}/metrics/{metric_id}/sessions', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics/{metric_id}/sessions', tags=["customMetrics"]) +def get_custom_metric_sessions(projectId: int, metric_id: int, + data: schemas.CustomMetricSessionsPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + data = custom_metrics.get_sessions(project_id=projectId, user_id=context.user_id, metric_id=metric_id, data=data) + if data is None: + return {"errors": ["custom metric not found"]} + return {"data": data} + + +@app.post('/{projectId}/metrics/{metric_id}/chart', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics/{metric_id}/chart', tags=["customMetrics"]) +def get_custom_metric_chart(projectId: int, metric_id: int, data: schemas.CustomMetricChartPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + data = dashboards2.make_chart_metrics(project_id=projectId, user_id=context.user_id, metric_id=metric_id, + data=data) + if data is None: + return {"errors": ["custom metric not found"]} + return {"data": data} + + +@app.post('/{projectId}/metrics/{metric_id}', tags=["dashboard"]) +@app.put('/{projectId}/metrics/{metric_id}', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def update_custom_metric(projectId: int, metric_id: int, data: schemas.UpdateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + data = custom_metrics.update(project_id=projectId, user_id=context.user_id, metric_id=metric_id, data=data) + if data is None: + return {"errors": ["custom metric not found"]} + return {"data": data} + + +@app.post('/{projectId}/metrics/{metric_id}/status', tags=["dashboard"]) +@app.put('/{projectId}/metrics/{metric_id}/status', tags=["dashboard"]) +@app.post('/{projectId}/custom_metrics/{metric_id}/status', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics/{metric_id}/status', tags=["customMetrics"]) +def update_custom_metric_state(projectId: int, metric_id: int, + data: schemas.UpdateCustomMetricsStatusSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return { + "data": custom_metrics.change_state(project_id=projectId, user_id=context.user_id, metric_id=metric_id, + status=data.active)} + + +@app.delete('/{projectId}/metrics/{metric_id}', tags=["dashboard"]) +@app.delete('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def delete_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.delete(project_id=projectId, user_id=context.user_id, metric_id=metric_id)} diff --git a/api/routers/app/v1_api.py b/api/routers/subs/v1_api.py similarity index 100% rename from api/routers/app/v1_api.py rename to api/routers/subs/v1_api.py diff --git a/api/schemas.py b/api/schemas.py index 77cb78c05..b01553cc8 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -776,6 +776,7 @@ class CustomMetricCreateSeriesSchema(BaseModel): class MetricTimeseriesViewType(str, Enum): line_chart = "lineChart" progress = "progress" + area_chart = "areaChart" class MetricTableViewType(str, Enum): @@ -803,8 +804,8 @@ class TimeseriesMetricOfType(str, Enum): class CustomMetricSessionsPayloadSchema(FlatSessionsSearch): - startDate: int = Field(TimeUTC.now(-7)) - endDate: int = Field(TimeUTC.now()) + startTimestamp: int = Field(TimeUTC.now(-7)) + endTimestamp: int = Field(TimeUTC.now()) class Config: alias_generator = attribute_to_camel_case @@ -817,10 +818,10 @@ class CustomMetricChartPayloadSchema(CustomMetricSessionsPayloadSchema): alias_generator = attribute_to_camel_case -class CreateCustomMetricsSchema(CustomMetricChartPayloadSchema): +class TryCustomMetricsPayloadSchema(CustomMetricChartPayloadSchema): name: str = Field(...) - series: List[CustomMetricCreateSeriesSchema] = Field(..., min_items=1) - is_public: bool = Field(default=True, const=True) + series: List[CustomMetricCreateSeriesSchema] = Field(...) + is_public: bool = Field(default=True) view_type: Union[MetricTimeseriesViewType, MetricTableViewType] = Field(MetricTimeseriesViewType.line_chart) metric_type: MetricType = Field(MetricType.timeseries) metric_of: Union[TableMetricOfType, TimeseriesMetricOfType] = Field(TableMetricOfType.user_id) @@ -858,6 +859,10 @@ class CreateCustomMetricsSchema(CustomMetricChartPayloadSchema): alias_generator = attribute_to_camel_case +class CreateCustomMetricsSchema(TryCustomMetricsPayloadSchema): + series: List[CustomMetricCreateSeriesSchema] = Field(..., min_items=1) + + class CustomMetricUpdateSeriesSchema(CustomMetricCreateSeriesSchema): series_id: Optional[int] = Field(None) @@ -875,3 +880,99 @@ class UpdateCustomMetricsStatusSchema(BaseModel): class SavedSearchSchema(FunnelSchema): filter: FlatSessionsSearchPayloadSchema = Field([]) + + +class CreateDashboardSchema(BaseModel): + name: str = Field(..., min_length=1) + is_public: bool = Field(default=False) + is_pinned: bool = Field(default=False) + metrics: Optional[List[int]] = Field(default=[]) + + class Config: + alias_generator = attribute_to_camel_case + + +class EditDashboardSchema(CreateDashboardSchema): + is_public: Optional[bool] = Field(default=None) + is_pinned: Optional[bool] = Field(default=None) + + +class UpdateWidgetPayloadSchema(BaseModel): + config: dict = Field(default={}) + + class Config: + alias_generator = attribute_to_camel_case + + +class AddWidgetToDashboardPayloadSchema(UpdateWidgetPayloadSchema): + metric_id: int = Field(...) + + class Config: + alias_generator = attribute_to_camel_case + + +# these values should match the keys in metrics table +class TemplatePredefinedKeys(str, Enum): + count_sessions = "count_sessions" + avg_request_load_time = "avg_request_load_time" + avg_page_load_time = "avg_page_load_time" + avg_image_load_time = "avg_image_load_time" + avg_dom_content_load_start = "avg_dom_content_load_start" + avg_first_contentful_pixel = "avg_first_contentful_pixel" + avg_visited_pages = "avg_visited_pages" + avg_session_duration = "avg_session_duration" + avg_pages_dom_buildtime = "avg_pages_dom_buildtime" + avg_pages_response_time = "avg_pages_response_time" + avg_response_time = "avg_response_time" + avg_first_paint = "avg_first_paint" + avg_dom_content_loaded = "avg_dom_content_loaded" + avg_till_first_bit = "avg_till_first_byte" + avg_time_to_interactive = "avg_time_to_interactive" + count_requests = "count_requests" + avg_time_to_render = "avg_time_to_render" + avg_used_js_heap_size = "avg_used_js_heap_size" + avg_cpu = "avg_cpu" + avg_fps = "avg_fps" + impacted_sessions_by_js_errors = "impacted_sessions_by_js_errors" + domains_errors_4xx = "domains_errors_4xx" + domains_errors_5xx = "domains_errors_5xx" + errors_per_domains = "errors_per_domains" + calls_errors = "calls_errors" + errors_by_type = "errors_per_type" + errors_by_origin = "resources_by_party" + speed_index_by_location = "speed_location" + slowest_domains = "slowest_domains" + sessions_per_browser = "sessions_per_browser" + time_to_render = "time_to_render" + impacted_sessions_by_slow_pages = "impacted_sessions_by_slow_pages" + memory_consumption = "memory_consumption" + cpu_load = "cpu" + frame_rate = "fps" + crashes = "crashes" + resources_vs_visually_complete = "resources_vs_visually_complete" + pages_dom_buildtime = "pages_dom_buildtime" + pages_response_time = "pages_response_time" + pages_response_time_distribution = "pages_response_time_distribution" + missing_resources = "missing_resources" + slowest_resources = "slowest_resources" + resources_fetch_time = "resources_loading_time" + resource_type_vs_response_end = "resource_type_vs_response_end" + resources_count_by_type = "resources_count_by_type" + + +class TemplatePredefinedUnits(str, Enum): + millisecond = "ms" + minute = "min" + memory = "mb" + frame = "f/s" + percentage = "%" + count = "count" + + +class CustomMetricAndTemplate(BaseModel): + is_template: bool = Field(...) + project_id: Optional[int] = Field(...) + predefined_key: Optional[TemplatePredefinedKeys] = Field(...) + + class Config: + alias_generator = attribute_to_camel_case diff --git a/backend/Dockerfile b/backend/Dockerfile index 5cefd4cb4..b7a494f86 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13-alpine3.10 AS prepare +FROM golang:1.18-alpine3.15 AS prepare RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash @@ -27,6 +27,7 @@ ENV TZ=UTC \ HTTP_PORT=80 \ BEACON_SIZE_LIMIT=7000000 \ KAFKA_USE_SSL=true \ + KAFKA_MAX_POLL_INTERVAL_MS=400000 \ REDIS_STREAMS_MAX_LEN=3000 \ TOPIC_RAW_WEB=raw \ TOPIC_RAW_IOS=raw-ios \ diff --git a/backend/go.mod b/backend/go.mod index ab98ca444..6588529a8 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,14 +1,12 @@ module openreplay/backend -go 1.13 +go 1.18 require ( cloud.google.com/go/logging v1.4.2 github.com/ClickHouse/clickhouse-go v1.4.3 - github.com/Masterminds/squirrel v1.5.0 github.com/aws/aws-sdk-go v1.35.23 github.com/btcsuite/btcutil v1.0.2 - github.com/confluentinc/confluent-kafka-go v1.7.0 // indirect github.com/elastic/go-elasticsearch/v7 v7.13.1 github.com/go-redis/redis v6.15.9+incompatible github.com/google/uuid v1.1.2 @@ -16,14 +14,47 @@ require ( github.com/jackc/pgconn v1.6.0 github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 github.com/jackc/pgx/v4 v4.6.0 - github.com/klauspost/compress v1.11.9 // indirect github.com/klauspost/pgzip v1.2.5 - github.com/lib/pq v1.2.0 github.com/oschwald/maxminddb-golang v1.7.0 github.com/pkg/errors v0.9.1 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce github.com/ua-parser/uap-go v0.0.0-20200325213135-e1c09f13e2fe + golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 google.golang.org/api v0.50.0 gopkg.in/confluentinc/confluent-kafka-go.v1 v1.7.0 - +) + +require ( + cloud.google.com/go v0.84.0 // indirect + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect + github.com/confluentinc/confluent-kafka-go v1.7.0 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.0.2 // indirect + github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 // indirect + github.com/jackc/pgtype v1.3.0 // indirect + github.com/jackc/puddle v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/klauspost/compress v1.11.9 // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/mod v0.4.2 // indirect + golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/tools v0.1.4 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84 // indirect + google.golang.org/grpc v1.38.0 // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index 8d538a0b4..607936204 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -46,8 +46,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/Masterminds/squirrel v1.5.0 h1:JukIZisrUXadA9pl3rMkjhiamxiB0cXiu+HGp/Y8cY8= -github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/aws/aws-sdk-go v1.35.23 h1:SCP0d0XvyJTDmfnHEQPvBaYi3kea1VNUo7uQmkVgFts= github.com/aws/aws-sdk-go v1.35.23/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= @@ -75,8 +73,8 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/confluentinc/confluent-kafka-go v1.5.2 h1:l+qt+a0Okmq0Bdr1P55IX4fiwFJyg0lZQmfHkAFkv7E= -github.com/confluentinc/confluent-kafka-go v1.5.2/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg= +github.com/confluentinc/confluent-kafka-go v1.7.0 h1:tXh3LWb2Ne0WiU3ng4h5qiGA9XV61rz46w60O+cq8bM= +github.com/confluentinc/confluent-kafka-go v1.7.0/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -93,7 +91,6 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -135,7 +132,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -152,11 +148,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -184,7 +178,6 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -203,7 +196,6 @@ github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye47 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= @@ -219,7 +211,6 @@ github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCM github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.3.0 h1:l8JvKrby3RI7Kg3bYEeU9TA4vqC38QDpFCfcrC7KuN0= github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik= -github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= @@ -254,10 +245,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= @@ -682,8 +669,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.5.2 h1:g0WBLy6fobNUU8W/e9zx6I0Yl79Ya+BDW1NwzAlTiiQ= -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.5.2/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY= +gopkg.in/confluentinc/confluent-kafka-go.v1 v1.7.0 h1:+RlmciBLDd/XwM1iudiG3HtCg45purnsOxEoY/+JZdQ= +gopkg.in/confluentinc/confluent-kafka-go.v1 v1.7.0/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/backend/pkg/db/cache/messages_common.go b/backend/pkg/db/cache/messages-common.go similarity index 66% rename from backend/pkg/db/cache/messages_common.go rename to backend/pkg/db/cache/messages-common.go index 3983982fe..8ca7b2f85 100644 --- a/backend/pkg/db/cache/messages_common.go +++ b/backend/pkg/db/cache/messages-common.go @@ -28,30 +28,6 @@ func (c *PGCache) InsertIssueEvent(sessionID uint64, crash *IssueEvent) error { return c.Conn.InsertIssueEvent(sessionID, session.ProjectID, crash) } -func (c *PGCache) InsertUserID(sessionID uint64, userID *IOSUserID) error { - if err := c.Conn.InsertIOSUserID(sessionID, userID); err != nil { - return err - } - session, err := c.GetSession(sessionID) - if err != nil { - return err - } - session.UserID = &userID.Value - return nil -} - -func (c *PGCache) InsertUserAnonymousID(sessionID uint64, userAnonymousID *IOSUserAnonymousID) error { - if err := c.Conn.InsertIOSUserAnonymousID(sessionID, userAnonymousID); err != nil { - return err - } - session, err := c.GetSession(sessionID) - if err != nil { - return err - } - session.UserAnonymousID = &userAnonymousID.Value - return nil -} - func (c *PGCache) InsertMetadata(sessionID uint64, metadata *Metadata) error { session, err := c.GetSession(sessionID) if err != nil { diff --git a/backend/pkg/db/cache/messages_ios.go b/backend/pkg/db/cache/messages-ios.go similarity index 100% rename from backend/pkg/db/cache/messages_ios.go rename to backend/pkg/db/cache/messages-ios.go diff --git a/backend/pkg/db/cache/messages_web.go b/backend/pkg/db/cache/messages-web.go similarity index 100% rename from backend/pkg/db/cache/messages_web.go rename to backend/pkg/db/cache/messages-web.go diff --git a/backend/pkg/db/cache/pg_cache.go b/backend/pkg/db/cache/pg-cache.go similarity index 100% rename from backend/pkg/db/cache/pg_cache.go rename to backend/pkg/db/cache/pg-cache.go diff --git a/backend/pkg/db/postgres/alert.go b/backend/pkg/db/postgres/alert.go deleted file mode 100644 index 964977bd3..000000000 --- a/backend/pkg/db/postgres/alert.go +++ /dev/null @@ -1,228 +0,0 @@ -package postgres - -import ( - "database/sql" - "errors" - "fmt" - sq "github.com/Masterminds/squirrel" - "log" - "strconv" - "time" -) - -type TimeString sql.NullString -type query struct { - Left string `db:"query.left" json:"left"` - Operator string `db:"query.operator" json:"operator"` - Right float64 `db:"query.right" json:"right"` -} -type options struct { - RenotifyInterval int64 `db:"options.renotifyInterval" json:"renotifyInterval"` - LastNotification int64 `db:"options.lastNotification" json:"lastNotification;omitempty"` - CurrentPeriod int64 `db:"options.currentPeriod" json:"currentPeriod"` - PreviousPeriod int64 `db:"options.previousPeriod" json:"previousPeriod;omitempty"` - Message []map[string]string `db:"options.message" json:"message;omitempty"` - Change string `db:"options.change" json:"change;omitempty"` -} -type Alert struct { - AlertID uint32 `db:"alert_id" json:"alert_id"` - ProjectID uint32 `db:"project_id" json:"project_id"` - Name string `db:"name" json:"name"` - Description sql.NullString `db:"description" json:"description"` - Active bool `db:"active" json:"active"` - DetectionMethod string `db:"detection_method" json:"detection_method"` - Query query `db:"query" json:"query"` - DeletedAt *int64 `db:"deleted_at" json:"deleted_at"` - CreatedAt *int64 `db:"created_at" json:"created_at"` - Options options `db:"options" json:"options"` - TenantId uint32 `db:"tenant_id" json:"tenant_id"` -} - -func (pg *Conn) IterateAlerts(iter func(alert *Alert, err error)) error { - rows, err := pg.query(` - SELECT - alerts.alert_id, - alerts.project_id, - alerts.name, - alerts.description, - alerts.active, - alerts.detection_method, - alerts.query, - CAST(EXTRACT(epoch FROM alerts.deleted_at) * 1000 AS BIGINT) AS deleted_at, - CAST(EXTRACT(epoch FROM alerts.created_at) * 1000 AS BIGINT) AS created_at, - alerts.options, - 0 AS tenant_id - FROM public.alerts - WHERE alerts.active AND alerts.deleted_at ISNULL; - `) - if err != nil { - return err - } - defer rows.Close() - for rows.Next() { - a := new(Alert) - if err = rows.Scan( - &a.AlertID, - &a.ProjectID, - &a.Name, - &a.Description, - &a.Active, - &a.DetectionMethod, - &a.Query, - &a.DeletedAt, - &a.CreatedAt, - &a.Options, - &a.TenantId, - ); err != nil { - iter(nil, err) - continue - } - iter(a, nil) - } - - if err = rows.Err(); err != nil { - return err - } - return nil -} - -func (pg *Conn) SaveLastNotification(allIds []uint32) error { - var paramrefs string - for _, v := range allIds { - paramrefs += strconv.Itoa(int(v)) + `,` - } - paramrefs = paramrefs[:len(paramrefs)-1] // remove last "," - q := "UPDATE public.Alerts SET options = options||'{\"lastNotification\":" + strconv.Itoa(int(time.Now().Unix()*1000)) + "}'::jsonb WHERE alert_id IN (" + paramrefs + ");" - //log.Println(q) - log.Println("Updating PG") - return pg.exec(q) -} - -type columnDefinition struct { - table string - formula string - condition string - group string -} - -var LeftToDb = map[string]columnDefinition{ - "performance.dom_content_loaded.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "COALESCE(AVG(NULLIF(dom_content_loaded_time ,0)),0)"}, - "performance.first_meaningful_paint.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "COALESCE(AVG(NULLIF(first_contentful_paint_time,0)),0)"}, - "performance.page_load_time.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(load_time ,0))"}, - "performance.dom_build_time.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(dom_building_time,0))"}, - "performance.speed_index.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(speed_index,0))"}, - "performance.page_response_time.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(response_time,0))"}, - "performance.ttfb.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(first_paint_time,0))"}, - "performance.time_to_render.average": {table: "events.pages INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(visually_complete,0))"}, - "performance.image_load_time.average": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(resources.duration,0))", condition: "type='img'"}, - "performance.request_load_time.average": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(resources.duration,0))", condition: "type='fetch'"}, - "resources.load_time.average": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "AVG(NULLIF(resources.duration,0))"}, - "resources.missing.count": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "COUNT(DISTINCT url_hostpath)", condition: "success= FALSE"}, - "errors.4xx_5xx.count": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "COUNT(session_id)", condition: "status/100!=2"}, - "errors.4xx.count": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "COUNT(session_id)", condition: "status/100=4"}, - "errors.5xx.count": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "COUNT(session_id)", condition: "status/100=5"}, - "errors.javascript.impacted_sessions.count": {table: "events.resources INNER JOIN public.sessions USING(session_id)", formula: "COUNT(DISTINCT session_id)", condition: "success= FALSE AND type='script'"}, - "performance.crashes.count": {table: "(SELECT *, start_ts AS timestamp FROM public.sessions WHERE errors_count > 0) AS sessions", formula: "COUNT(DISTINCT session_id)", condition: "errors_count > 0"}, - "errors.javascript.count": {table: "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", formula: "COUNT(DISTINCT session_id)", condition: "source='js_exception'"}, - "errors.backend.count": {table: "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", formula: "COUNT(DISTINCT session_id)", condition: "source!='js_exception'"}, -} - -//This is the frequency of execution for each threshold -var TimeInterval = map[int64]int64{ - 15: 3, - 30: 5, - 60: 10, - 120: 20, - 240: 30, - 1440: 60, -} - -func (a *Alert) CanCheck() bool { - now := time.Now().Unix() * 1000 - var repetitionBase int64 - - if repetitionBase = a.Options.CurrentPeriod; a.DetectionMethod == "change" && a.Options.CurrentPeriod > a.Options.PreviousPeriod { - repetitionBase = a.Options.PreviousPeriod - } - - if _, ok := TimeInterval[repetitionBase]; !ok { - log.Printf("repetitionBase: %d NOT FOUND", repetitionBase) - return false - } - return a.DeletedAt == nil && a.Active && - (a.Options.RenotifyInterval <= 0 || - a.Options.LastNotification <= 0 || - ((now - a.Options.LastNotification) > a.Options.RenotifyInterval*60*1000)) && - ((now-*a.CreatedAt)%(TimeInterval[repetitionBase]*60*1000)) < 60*1000 -} - -func (a *Alert) Build() (sq.SelectBuilder, error) { - colDef, ok := LeftToDb[a.Query.Left] - if !ok { - return sq.Select(), errors.New(fmt.Sprintf("!! unsupported metric '%s' from alert: %d:%s\n", a.Query.Left, a.AlertID, a.Name)) - } - - subQ := sq. - Select(colDef.formula + " AS value"). - From(colDef.table). - Where(sq.And{sq.Expr("project_id = $1 ", a.ProjectID), - sq.Expr(colDef.condition)}) - q := sq.Select(fmt.Sprint("value, coalesce(value,0)", a.Query.Operator, a.Query.Right, " AS valid")) - if len(colDef.group) > 0 { - subQ = subQ.Column(colDef.group + " AS group_value") - subQ = subQ.GroupBy(colDef.group) - q = q.Column("group_value") - } - - if a.DetectionMethod == "threshold" { - q = q.FromSelect(subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod*60)), "stat") - } else if a.DetectionMethod == "change" { - if a.Options.Change == "change" { - if len(colDef.group) == 0 { - sub1, args1, _ := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod*60)).ToSql() - sub2, args2, _ := subQ.Where( - sq.And{ - sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod*60), - sq.Expr("timestamp>=$4 ", time.Now().Unix()-2*a.Options.CurrentPeriod*60), - }).ToSql() - sub1, _, _ = sq.Expr("SELECT ((" + sub1 + ")-(" + sub2 + ")) AS value").ToSql() - q = q.JoinClause("FROM ("+sub1+") AS stat", append(args1, args2...)...) - } else { - subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod*60)) - sub2, args2, _ := subQ.Where( - sq.And{ - sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod*60), - sq.Expr("timestamp>=$4 ", time.Now().Unix()-2*a.Options.CurrentPeriod*60), - }).ToSql() - sub1 := sq.Select("group_value", "(stat1.value-stat2.value) AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) - q = q.FromSelect(sub1, "stat") - } - } else if a.Options.Change == "percent" { - if len(colDef.group) == 0 { - sub1, args1, _ := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod*60)).ToSql() - sub2, args2, _ := subQ.Where( - sq.And{ - sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod*60), - sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod*60-a.Options.CurrentPeriod*60), - }).ToSql() - sub1, _, _ = sq.Expr("SELECT ((" + sub1 + ")/(" + sub2 + ")-1)*100 AS value").ToSql() - q = q.JoinClause("FROM ("+sub1+") AS stat", append(args1, args2...)...) - } else { - subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod*60)) - sub2, args2, _ := subQ.Where( - sq.And{ - sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod*60), - sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod*60-a.Options.CurrentPeriod*60), - }).ToSql() - sub1 := sq.Select("group_value", "(stat1.value/stat2.value-1)*100 AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) - q = q.FromSelect(sub1, "stat") - } - } else { - return q, errors.New("unsupported change method") - } - - } else { - return q, errors.New("unsupported detection method") - } - return q, nil -} \ No newline at end of file diff --git a/backend/pkg/db/postgres/listener.go b/backend/pkg/db/postgres/listener.go index f90d83485..ef99c2c59 100644 --- a/backend/pkg/db/postgres/listener.go +++ b/backend/pkg/db/postgres/listener.go @@ -11,7 +11,6 @@ import ( type Listener struct { conn *pgx.Conn Integrations chan *Integration - Alerts chan *Alert Errors chan error } @@ -32,23 +31,6 @@ func NewIntegrationsListener(url string) (*Listener, error) { return listener, nil } -func NewAlertsListener(url string) (*Listener, error) { - conn, err := pgx.Connect(context.Background(), url) - if err != nil { - return nil, err - } - listener := &Listener{ - conn: conn, - Errors: make(chan error), - } - listener.Alerts = make(chan *Alert, 50) - if _, err := conn.Exec(context.Background(), "LISTEN alert"); err != nil { - return nil, err - } - go listener.listen() - return listener, nil -} - func (listener *Listener) listen() { for { notification, err := listener.conn.WaitForNotification(context.Background()) @@ -64,13 +46,6 @@ func (listener *Listener) listen() { } else { listener.Integrations <- integrationP } - case "alert": - alertP := new(Alert) - if err := json.Unmarshal([]byte(notification.Payload), alertP); err != nil { - listener.Errors <- fmt.Errorf("%v | Payload: %v", err, notification.Payload) - } else { - listener.Alerts <- alertP - } } } } diff --git a/backend/pkg/db/postgres/messages_common.go b/backend/pkg/db/postgres/messages-common.go similarity index 100% rename from backend/pkg/db/postgres/messages_common.go rename to backend/pkg/db/postgres/messages-common.go diff --git a/backend/pkg/db/postgres/messages_ios.go b/backend/pkg/db/postgres/messages-ios.go similarity index 100% rename from backend/pkg/db/postgres/messages_ios.go rename to backend/pkg/db/postgres/messages-ios.go diff --git a/backend/pkg/db/postgres/messages_web_stats.go b/backend/pkg/db/postgres/messages-web-stats.go similarity index 97% rename from backend/pkg/db/postgres/messages_web_stats.go rename to backend/pkg/db/postgres/messages-web-stats.go index 933442b0b..27a6272e2 100644 --- a/backend/pkg/db/postgres/messages_web_stats.go +++ b/backend/pkg/db/postgres/messages-web-stats.go @@ -35,7 +35,7 @@ func (conn *Conn) InsertWebStatsPerformance(sessionID uint64, p *PerformanceTrac } func (conn *Conn) InsertWebStatsResourceEvent(sessionID uint64, e *ResourceEvent) error { - host, _, err := url.GetURLParts(e.URL) + host, _, _, err := url.GetURLParts(e.URL) if err != nil { return err } diff --git a/backend/pkg/db/postgres/messages_web.go b/backend/pkg/db/postgres/messages-web.go similarity index 87% rename from backend/pkg/db/postgres/messages_web.go rename to backend/pkg/db/postgres/messages-web.go index 0d822cddd..eb4bca364 100644 --- a/backend/pkg/db/postgres/messages_web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -55,7 +55,7 @@ func (conn *Conn) InsertWebUserAnonymousID(sessionID uint64, userAnonymousID *Us // TODO: fix column "dom_content_loaded_event_end" of relation "pages" func (conn *Conn) InsertWebPageEvent(sessionID uint64, e *PageEvent) error { - host, path, err := url.GetURLParts(e.URL) + host, path, query, err := url.GetURLParts(e.URL) if err != nil { return err } @@ -64,20 +64,27 @@ func (conn *Conn) InsertWebPageEvent(sessionID uint64, e *PageEvent) error { return err } defer tx.rollback() + // base_path is depricated if err := tx.exec(` INSERT INTO events.pages ( - session_id, message_id, timestamp, referrer, base_referrer, host, path, base_path, + session_id, message_id, timestamp, referrer, base_referrer, host, path, query, dom_content_loaded_time, load_time, response_end, first_paint_time, first_contentful_paint_time, speed_index, visually_complete, time_to_interactive, - response_time, dom_building_time + response_time, dom_building_time, + base_path ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, + $1, $2, $3, + $4, $5, + $6, $7, $8, NULLIF($9, 0), NULLIF($10, 0), NULLIF($11, 0), NULLIF($12, 0), NULLIF($13, 0), NULLIF($14, 0), NULLIF($15, 0), NULLIF($16, 0), - NULLIF($17, 0), NULLIF($18, 0) + NULLIF($17, 0), NULLIF($18, 0), + '', ) `, - sessionID, e.MessageID, e.Timestamp, e.Referrer, url.DiscardURLQuery(e.Referrer), host, path, url.DiscardURLQuery(path), + sessionID, e.MessageID, e.Timestamp, + e.Referrer, url.DiscardURLQuery(e.Referrer), + host, path, query, e.DomContentLoadedEventEnd, e.LoadEventEnd, e.ResponseEnd, e.FirstPaint, e.FirstContentfulPaint, e.SpeedIndex, e.VisuallyComplete, e.TimeToInteractive, calcResponseTime(e), calcDomBuildingTime(e), @@ -109,7 +116,7 @@ func (conn *Conn) InsertWebClickEvent(sessionID uint64, e *ClickEvent) error { INSERT INTO events.clicks (session_id, message_id, timestamp, label, selector, url) (SELECT - $1, $2, $3, NULLIF($4, ''), $5, host || base_path + $1, $2, $3, NULLIF($4, ''), $5, host || path FROM events.pages WHERE session_id = $1 AND timestamp <= $3 ORDER BY timestamp DESC LIMIT 1 ) @@ -210,20 +217,27 @@ func (conn *Conn) InsertWebFetchEvent(sessionID uint64, savePayload bool, e *Fet request = &e.Request response = &e.Response } - conn.insertAutocompleteValue(sessionID, "REQUEST", url.DiscardURLQuery(e.URL)) + host, path, query, err := url.GetURLParts(e.URL) + conn.insertAutocompleteValue(sessionID, "REQUEST", path) + if err != nil { + return err + } return conn.batchQueue(sessionID, ` INSERT INTO events_common.requests ( - session_id, timestamp, - seq_index, url, duration, success, - request_body, response_body, status_code, method + session_id, timestamp, seq_index, + url, host, path, query, + request_body, response_body, status_code, method, + duration, success, ) VALUES ( - $1, $2, - $3, $4, $5, $6, - $7, $8, $9::smallint, NULLIF($10, '')::http_method + $1, $2, $3, + $4, $5, $6, $7 + $8, $9, $10::smallint, NULLIF($11, '')::http_method, + $12, $13 ) ON CONFLICT DO NOTHING`, - sessionID, e.Timestamp, - getSqIdx(e.MessageID), e.URL, e.Duration, e.Status < 400, + sessionID, e.Timestamp, getSqIdx(e.MessageID), + e.URL, host, path, query, request, response, e.Status, url.EnsureMethod(e.Method), + e.Duration, e.Status < 400, ) } diff --git a/backend/pkg/db/postgres/unstarted_session.go b/backend/pkg/db/postgres/unstarted-session.go similarity index 100% rename from backend/pkg/db/postgres/unstarted_session.go rename to backend/pkg/db/postgres/unstarted-session.go diff --git a/backend/pkg/env/worker_id.go b/backend/pkg/env/worker-id.go similarity index 100% rename from backend/pkg/env/worker_id.go rename to backend/pkg/env/worker-id.go diff --git a/backend/pkg/messages/get_timestamp.go b/backend/pkg/messages/get-timestamp.go similarity index 100% rename from backend/pkg/messages/get_timestamp.go rename to backend/pkg/messages/get-timestamp.go diff --git a/backend/pkg/messages/legacy_message_transform.go b/backend/pkg/messages/legacy-message-transform.go similarity index 100% rename from backend/pkg/messages/legacy_message_transform.go rename to backend/pkg/messages/legacy-message-transform.go diff --git a/backend/pkg/messages/read_message.go b/backend/pkg/messages/read-message.go similarity index 100% rename from backend/pkg/messages/read_message.go rename to backend/pkg/messages/read-message.go diff --git a/backend/pkg/pprof/pprof.go b/backend/pkg/pprof/pprof.go new file mode 100644 index 000000000..a05080178 --- /dev/null +++ b/backend/pkg/pprof/pprof.go @@ -0,0 +1,13 @@ +package pprof + +import ( + "log" + "net/http" + _ "net/http/pprof" +) + +func StartProfilingServer() { + go func() { + log.Println(http.ListenAndServe("localhost:6060", nil)) + }() +} diff --git a/backend/pkg/queue/import.go b/backend/pkg/queue/import.go index 2bca9c8fd..623d301ca 100644 --- a/backend/pkg/queue/import.go +++ b/backend/pkg/queue/import.go @@ -1,15 +1,14 @@ package queue import ( - "openreplay/backend/pkg/redisstream" "openreplay/backend/pkg/queue/types" + "openreplay/backend/pkg/redisstream" ) -func NewConsumer(group string, topics []string, handler types.MessageHandler) types.Consumer { +func NewConsumer(group string, topics []string, handler types.MessageHandler, _ bool) types.Consumer { return redisstream.NewConsumer(group, topics, handler) } func NewProducer() types.Producer { return redisstream.NewProducer() } - diff --git a/backend/pkg/queue/messages.go b/backend/pkg/queue/messages.go index eca4a4d49..0ab184ee6 100644 --- a/backend/pkg/queue/messages.go +++ b/backend/pkg/queue/messages.go @@ -7,13 +7,12 @@ import ( "openreplay/backend/pkg/queue/types" ) - -func NewMessageConsumer(group string, topics []string, handler types.DecodedMessageHandler) types.Consumer { +func NewMessageConsumer(group string, topics []string, handler types.DecodedMessageHandler, autoCommit bool) types.Consumer { return NewConsumer(group, topics, func(sessionID uint64, value []byte, meta *types.Meta) { if err := messages.ReadBatch(value, func(msg messages.Message) { handler(sessionID, msg, meta) }); err != nil { log.Printf("Decode error: %v\n", err) } - }) + }, autoCommit) } diff --git a/backend/pkg/queue/types/types.go b/backend/pkg/queue/types/types.go index b671323d0..600babe25 100644 --- a/backend/pkg/queue/types/types.go +++ b/backend/pkg/queue/types/types.go @@ -6,26 +6,22 @@ import ( type Consumer interface { ConsumeNext() error - DisableAutoCommit() Commit() error CommitBack(gap int64) error Close() } - type Producer interface { Produce(topic string, key uint64, value []byte) error Close(timeout int) Flush(timeout int) } - type Meta struct { - ID uint64 - Topic string + ID uint64 + Topic string Timestamp int64 } type MessageHandler func(uint64, []byte, *Meta) type DecodedMessageHandler func(uint64, messages.Message, *Meta) - diff --git a/backend/pkg/redisstream/consumer.go b/backend/pkg/redisstream/consumer.go index 164ee9236..d32972981 100644 --- a/backend/pkg/redisstream/consumer.go +++ b/backend/pkg/redisstream/consumer.go @@ -1,24 +1,22 @@ package redisstream import ( + "log" "net" + "sort" "strconv" "strings" - "log" - "sort" "time" - "github.com/pkg/errors" _redis "github.com/go-redis/redis" + "github.com/pkg/errors" "openreplay/backend/pkg/queue/types" ) - - -type idsInfo struct{ - id []string - ts []int64 +type idsInfo struct { + id []string + ts []int64 } type streamPendingIDsMap map[string]*idsInfo @@ -41,26 +39,25 @@ func NewConsumer(group string, streams []string, messageHandler types.MessageHan } } - idsPending := make(streamPendingIDsMap) streamsCount := len(streams) for i := 0; i < streamsCount; i++ { - // ">" is for never-delivered messages. - // Otherwise - never acknoledged only + // ">" is for never-delivered messages. + // Otherwise - never acknoledged only // TODO: understand why in case of "0" it eats 100% cpu - streams = append(streams, ">") - + streams = append(streams, ">") + idsPending[streams[i]] = new(idsInfo) } return &Consumer{ - redis: redis, + redis: redis, messageHandler: messageHandler, - streams: streams, - group: group, - autoCommit: true, - idsPending: idsPending, + streams: streams, + group: group, + autoCommit: true, + idsPending: idsPending, } } @@ -106,9 +103,9 @@ func (c *Consumer) ConsumeNext() error { return errors.New("Too many messages per ms in redis") } c.messageHandler(sessionID, []byte(valueString), &types.Meta{ - Topic: r.Stream, + Topic: r.Stream, Timestamp: int64(ts), - ID: ts << 13 | (idx & 0x1FFF), // Max: 4096 messages/ms for 69 years + ID: ts<<13 | (idx & 0x1FFF), // Max: 4096 messages/ms for 69 years }) if c.autoCommit { if err = c.redis.XAck(r.Stream, c.group, m.ID).Err(); err != nil { @@ -119,7 +116,7 @@ func (c *Consumer) ConsumeNext() error { c.idsPending[r.Stream].id = append(c.idsPending[r.Stream].id, m.ID) c.idsPending[r.Stream].ts = append(c.idsPending[r.Stream].ts, int64(ts)) } - + } } return nil @@ -158,13 +155,9 @@ func (c *Consumer) CommitBack(gap int64) error { c.idsPending[stream].id = idsInfo.id[maxI:] c.idsPending[stream].ts = idsInfo.ts[maxI:] } - return nil -} - -func (c *Consumer) DisableAutoCommit() { - //c.autoCommit = false + return nil } func (c *Consumer) Close() { // noop -} \ No newline at end of file +} diff --git a/backend/pkg/token/tokenizer.go b/backend/pkg/token/tokenizer.go index 3f1069a63..f61e1f145 100644 --- a/backend/pkg/token/tokenizer.go +++ b/backend/pkg/token/tokenizer.go @@ -22,8 +22,8 @@ func NewTokenizer(secret string) *Tokenizer { } type TokenData struct { - ID uint64 - ExpTime int64 + ID uint64 + ExpTime int64 } func (tokenizer *Tokenizer) sign(body string) []byte { @@ -33,7 +33,7 @@ func (tokenizer *Tokenizer) sign(body string) []byte { } func (tokenizer *Tokenizer) Compose(d TokenData) string { - body := strconv.FormatUint(d.ID, 36) + + body := strconv.FormatUint(d.ID, 36) + "." + strconv.FormatInt(d.ExpTime, 36) sign := base58.Encode(tokenizer.sign(body)) return body + "." + sign @@ -58,8 +58,8 @@ func (tokenizer *Tokenizer) Parse(token string) (*TokenData, error) { if err != nil { return nil, err } - if expTime <= time.Now().UnixNano()/1e6 { - return &TokenData{id,expTime}, EXPIRED + if expTime <= time.Now().UnixMilli() { + return &TokenData{id, expTime}, EXPIRED } - return &TokenData{id,expTime}, nil + return &TokenData{id, expTime}, nil } diff --git a/backend/pkg/url/assets/url.go b/backend/pkg/url/assets/url.go index 1fe717531..b55921149 100644 --- a/backend/pkg/url/assets/url.go +++ b/backend/pkg/url/assets/url.go @@ -5,11 +5,18 @@ import ( "path/filepath" "strconv" "strings" + "time" + + "openreplay/backend/pkg/flakeid" ) func getSessionKey(sessionID uint64) string { - // Based on timestamp, changes once per week. Check pkg/flakeid for understanding sessionID - return strconv.FormatUint(sessionID>>50, 10) + return strconv.FormatUint( + uint64(time.UnixMilli( + int64(flakeid.ExtractTimestamp(sessionID)), + ).Weekday()), + 10, + ) } func ResolveURL(baseurl string, rawurl string) string { diff --git a/backend/pkg/url/url.go b/backend/pkg/url/url.go index b9181774d..48cd0ef8d 100644 --- a/backend/pkg/url/url.go +++ b/backend/pkg/url/url.go @@ -1,18 +1,19 @@ package url import ( - "strings" _url "net/url" + "strings" ) func DiscardURLQuery(url string) string { return strings.Split(url, "?")[0] -} +} -func GetURLParts(rawURL string) (string, string, error) { +func GetURLParts(rawURL string) (string, string, string, error) { u, err := _url.Parse(rawURL) if err != nil { - return "", "", err + return "", "", "", err } - return u.Host, u.RequestURI(), nil -} \ No newline at end of file + // u.Scheme ? + return u.Host, u.RawPath, u.RawQuery, nil +} diff --git a/backend/pkg/utime/utime.go b/backend/pkg/utime/utime.go deleted file mode 100644 index e3b5a2751..000000000 --- a/backend/pkg/utime/utime.go +++ /dev/null @@ -1,11 +0,0 @@ -package utime - -import "time" - -func CurrentTimestamp() int64 { - return time.Now().UnixNano() / 1e6 -} - -func ToMilliseconds(t time.Time) int64 { - return t.UnixNano() / 1e6 -} diff --git a/backend/services/assets/cacher/cacher.go b/backend/services/assets/cacher/cacher.go index 59b09449f..70ea31928 100644 --- a/backend/services/assets/cacher/cacher.go +++ b/backend/services/assets/cacher/cacher.go @@ -1,31 +1,31 @@ package cacher import ( + "crypto/tls" "fmt" "io" "io/ioutil" "mime" "net/http" - "crypto/tls" "path/filepath" "strings" "time" "github.com/pkg/errors" - - "openreplay/backend/pkg/url/assets" + "openreplay/backend/pkg/storage" + "openreplay/backend/pkg/url/assets" ) const MAX_CACHE_DEPTH = 5 type cacher struct { - timeoutMap *timeoutMap // Concurrency implemented - s3 *storage.S3 // AWS Docs: "These clients are safe to use concurrently." - httpClient *http.Client // Docs: "Clients are safe for concurrent use by multiple goroutines." - rewriter *assets.Rewriter // Read only - Errors chan error - sizeLimit int + timeoutMap *timeoutMap // Concurrency implemented + s3 *storage.S3 // AWS Docs: "These clients are safe to use concurrently." + httpClient *http.Client // Docs: "Clients are safe for concurrent use by multiple goroutines." + rewriter *assets.Rewriter // Read only + Errors chan error + sizeLimit int } func NewCacher(region string, bucket string, origin string, sizeLimit int) *cacher { @@ -36,26 +36,26 @@ func NewCacher(region string, bucket string, origin string, sizeLimit int) *cach httpClient: &http.Client{ Timeout: time.Duration(6) * time.Second, Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, }, - rewriter: rewriter, - Errors: make(chan error), - sizeLimit: sizeLimit, + rewriter: rewriter, + Errors: make(chan error), + sizeLimit: sizeLimit, } } func (c *cacher) cacheURL(requestURL string, sessionID uint64, depth byte, context string, isJS bool) { - if c.timeoutMap.contains(requestURL) { - return - } - c.timeoutMap.add(requestURL) var cachePath string - if (isJS) { + if isJS { cachePath = assets.GetCachePathForJS(requestURL) } else { cachePath = assets.GetCachePathForAssets(sessionID, requestURL) } + if c.timeoutMap.contains(cachePath) { + return + } + c.timeoutMap.add(cachePath) if c.s3.Exists(cachePath) { return } @@ -94,20 +94,19 @@ func (c *cacher) cacheURL(requestURL string, sessionID uint64, depth byte, conte if isCSS { strData = c.rewriter.RewriteCSS(sessionID, requestURL, strData) // TODO: one method for reqrite and return list } - - // TODO: implement in streams + + // TODO: implement in streams err = c.s3.Upload(strings.NewReader(strData), cachePath, contentType, false) if err != nil { c.Errors <- errors.Wrap(err, context) return } - c.timeoutMap.add(requestURL) if isCSS { if depth > 0 { for _, extractedURL := range assets.ExtractURLsFromCSS(string(data)) { - if fullURL, cachable := assets.GetFullCachableURL(requestURL, extractedURL); cachable { - go c.cacheURL(fullURL, sessionID, depth-1, context + "\n -> " + fullURL, false) + if fullURL, cachable := assets.GetFullCachableURL(requestURL, extractedURL); cachable { + go c.cacheURL(fullURL, sessionID, depth-1, context+"\n -> "+fullURL, false) } } if err != nil { diff --git a/backend/services/assets/cacher/timeoutMap.go b/backend/services/assets/cacher/timeoutMap.go index 36fc4ee4d..5a8e31424 100644 --- a/backend/services/assets/cacher/timeoutMap.go +++ b/backend/services/assets/cacher/timeoutMap.go @@ -5,30 +5,30 @@ import ( "time" ) -const MAX_STORAGE_TIME = 18 * time.Hour +const MAX_STORAGE_TIME = 24 * time.Hour // If problem with cache contention (>=4 core) look at sync.Map type timeoutMap struct { mx sync.RWMutex - m map[string]time.Time + m map[string]time.Time } func newTimeoutMap() *timeoutMap { return &timeoutMap{ m: make(map[string]time.Time), } -} +} func (tm *timeoutMap) add(key string) { tm.mx.Lock() - defer tm.mx.Unlock() + defer tm.mx.Unlock() tm.m[key] = time.Now() } func (tm *timeoutMap) contains(key string) bool { tm.mx.RLock() - defer tm.mx.RUnlock() + defer tm.mx.RUnlock() _, ok := tm.m[key] return ok } @@ -36,7 +36,7 @@ func (tm *timeoutMap) contains(key string) bool { func (tm *timeoutMap) deleteOutdated() { now := time.Now() tm.mx.Lock() - defer tm.mx.Unlock() + defer tm.mx.Unlock() for key, t := range tm.m { if now.Sub(t) > MAX_STORAGE_TIME { delete(tm.m, key) diff --git a/backend/services/assets/main.go b/backend/services/assets/main.go index 450dfc83c..664dc5b09 100644 --- a/backend/services/assets/main.go +++ b/backend/services/assets/main.go @@ -18,7 +18,7 @@ import ( func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - GROUP_CACHE := env.String("GROUP_CACHE") + GROUP_CACHE := env.String("GROUP_CACHE") TOPIC_CACHE := env.String("TOPIC_CACHE") cacher := cacher.NewCacher( @@ -29,10 +29,10 @@ func main() { ) consumer := queue.NewMessageConsumer( - GROUP_CACHE, - []string{ TOPIC_CACHE }, + GROUP_CACHE, + []string{TOPIC_CACHE}, func(sessionID uint64, message messages.Message, e *types.Meta) { - switch msg := message.(type) { + switch msg := message.(type) { case *messages.AssetCache: cacher.CacheURL(sessionID, msg.URL) case *messages.ErrorEvent: @@ -47,17 +47,17 @@ func main() { for _, source := range sourceList { cacher.CacheJSFile(source) } - } + } }, + true, ) - tick := time.Tick(20 * time.Minute) sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - log.Printf("Cacher service started\n") + log.Printf("Cacher service started\n") for { select { case sig := <-sigchan: @@ -74,4 +74,4 @@ func main() { } } } -} \ No newline at end of file +} diff --git a/backend/services/db/main.go b/backend/services/db/main.go index d6190a4f0..2ad6e4aa8 100644 --- a/backend/services/db/main.go +++ b/backend/services/db/main.go @@ -74,8 +74,8 @@ func main() { } }) }, + false, ) - consumer.DisableAutoCommit() sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) diff --git a/backend/services/ender/builder/builder.go b/backend/services/ender/builder/builder.go index e36bdcbe3..9c2067985 100644 --- a/backend/services/ender/builder/builder.go +++ b/backend/services/ender/builder/builder.go @@ -110,11 +110,11 @@ func (b *builder) buildInputEvent() { func (b *builder) handleMessage(message Message, messageID uint64) { timestamp := GetTimestamp(message) - if b.timestamp <= timestamp { // unnecessary? TODO: test and remove + if b.timestamp < timestamp { // unnecessary? TODO: test and remove b.timestamp = timestamp } - b.lastProcessedTimestamp = time.Now().UnixNano() / 1e6 + b.lastProcessedTimestamp = time.Now().UnixMilli() // Might happen before the first timestamp. switch msg := message.(type) { diff --git a/backend/services/ender/main.go b/backend/services/ender/main.go index e8d739f0e..f2430f3a0 100644 --- a/backend/services/ender/main.go +++ b/backend/services/ender/main.go @@ -8,12 +8,12 @@ import ( "os/signal" "syscall" - "openreplay/backend/pkg/intervals" "openreplay/backend/pkg/env" + "openreplay/backend/pkg/intervals" + logger "openreplay/backend/pkg/log" "openreplay/backend/pkg/messages" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/queue/types" - logger "openreplay/backend/pkg/log" "openreplay/backend/services/ender/builder" ) @@ -29,24 +29,24 @@ func main() { producer := queue.NewProducer() consumer := queue.NewMessageConsumer( - GROUP_EVENTS, - []string{ + GROUP_EVENTS, + []string{ env.String("TOPIC_RAW_WEB"), env.String("TOPIC_RAW_IOS"), - }, + }, func(sessionID uint64, msg messages.Message, meta *types.Meta) { statsLogger.HandleAndLog(sessionID, meta) builderMap.HandleMessage(sessionID, msg, msg.Meta().Index) }, + false, ) - consumer.DisableAutoCommit() - + tick := time.Tick(intervals.EVENTS_COMMIT_INTERVAL * time.Millisecond) sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - log.Printf("Ender service started\n") + log.Printf("Ender service started\n") for { select { case sig := <-sigchan: @@ -55,8 +55,8 @@ func main() { consumer.CommitBack(intervals.EVENTS_BACK_COMMIT_GAP) consumer.Close() os.Exit(0) - case <- tick: - builderMap.IterateReadyMessages(time.Now().UnixNano()/1e6, func(sessionID uint64, readyMsg messages.Message) { + case <-tick: + builderMap.IterateReadyMessages(time.Now().UnixMilli(), func(sessionID uint64, readyMsg messages.Message) { producer.Produce(TOPIC_TRIGGER, sessionID, messages.Encode(readyMsg)) }) // TODO: why exactly do we need Flush here and not in any other place? @@ -69,4 +69,3 @@ func main() { } } } - diff --git a/backend/services/http/handlers_depricated.go b/backend/services/http/handlers-depricated.go similarity index 100% rename from backend/services/http/handlers_depricated.go rename to backend/services/http/handlers-depricated.go diff --git a/backend/services/http/handlers_ios.go b/backend/services/http/handlers-ios.go similarity index 68% rename from backend/services/http/handlers_ios.go rename to backend/services/http/handlers-ios.go index 6c3f945bd..8116980e1 100644 --- a/backend/services/http/handlers_ios.go +++ b/backend/services/http/handlers-ios.go @@ -2,55 +2,55 @@ package main import ( "encoding/json" - "net/http" "errors" - "time" - "math/rand" - "strconv" "log" + "math/rand" + "net/http" + "strconv" + "time" "openreplay/backend/pkg/db/postgres" - "openreplay/backend/pkg/token" . "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/token" ) -const FILES_SIZE_LIMIT int64 = 1e7 // 10Mb +const FILES_SIZE_LIMIT int64 = 1e7 // 10Mb func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { type request struct { - Token string `json:"token"` - ProjectKey *string `json:"projectKey"` - TrackerVersion string `json:"trackerVersion"` - RevID string `json:"revID"` - UserUUID *string `json:"userUUID"` + Token string `json:"token"` + ProjectKey *string `json:"projectKey"` + TrackerVersion string `json:"trackerVersion"` + RevID string `json:"revID"` + UserUUID *string `json:"userUUID"` //UserOS string `json"userOS"` //hardcoded 'MacOS' - UserOSVersion string `json:"userOSVersion"` - UserDevice string `json:"userDevice"` - Timestamp uint64 `json:"timestamp"` + UserOSVersion string `json:"userOSVersion"` + UserDevice string `json:"userDevice"` + Timestamp uint64 `json:"timestamp"` // UserDeviceType uint 0:phone 1:pad 2:tv 3:carPlay 5:mac // “performances”:{ - // “activeProcessorCount”:8, - // “isLowPowerModeEnabled”:0, - // “orientation”:0, - // “systemUptime”:585430, - // “batteryState”:0, - // “thermalState”:0, - // “batteryLevel”:0, - // “processorCount”:8, - // “physicalMemory”:17179869184 - // }, + // “activeProcessorCount”:8, + // “isLowPowerModeEnabled”:0, + // “orientation”:0, + // “systemUptime”:585430, + // “batteryState”:0, + // “thermalState”:0, + // “batteryLevel”:0, + // “processorCount”:8, + // “physicalMemory”:17179869184 + // }, } type response struct { - Token string `json:"token"` - ImagesHashList []string `json:"imagesHashList"` - UserUUID string `json:"userUUID"` - BeaconSizeLimit int64 `json:"beaconSizeLimit"` - SessionID string `json:"sessionID"` + Token string `json:"token"` + ImagesHashList []string `json:"imagesHashList"` + UserUUID string `json:"userUUID"` + BeaconSizeLimit int64 `json:"beaconSizeLimit"` + SessionID string `json:"sessionID"` } startTime := time.Now() req := &request{} body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) - //defer body.Close() + defer body.Close() if err := json.NewDecoder(body).Decode(req); err != nil { responseWithError(w, http.StatusBadRequest, err) return @@ -85,29 +85,29 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { responseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } - sessionID, err := flaker.Compose(uint64(startTime.UnixNano() / 1e6)) + sessionID, err := flaker.Compose(uint64(startTime.UnixMilli())) if err != nil { responseWithError(w, http.StatusInternalServerError, err) return } // TODO: if EXPIRED => send message for two sessions association expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) - tokenData = &token.TokenData{sessionID, expTime.UnixNano() / 1e6} + tokenData = &token.TokenData{sessionID, expTime.UnixMilli()} country := geoIP.ExtractISOCodeFromHTTPRequest(r) // The difference with web is mostly here: producer.Produce(TOPIC_RAW_IOS, tokenData.ID, Encode(&IOSSessionStart{ - Timestamp: req.Timestamp, - ProjectID: uint64(p.ProjectID), - TrackerVersion: req.TrackerVersion, - RevID: req.RevID, - UserUUID: userUUID, - UserOS: "IOS", - UserOSVersion: req.UserOSVersion, - UserDevice: MapIOSDevice(req.UserDevice), - UserDeviceType: GetIOSDeviceType(req.UserDevice), - UserCountry: country, + Timestamp: req.Timestamp, + ProjectID: uint64(p.ProjectID), + TrackerVersion: req.TrackerVersion, + RevID: req.RevID, + UserUUID: userUUID, + UserOS: "IOS", + UserOSVersion: req.UserOSVersion, + UserDevice: MapIOSDevice(req.UserDevice), + UserDeviceType: GetIOSDeviceType(req.UserDevice), + UserCountry: country, })) } @@ -119,14 +119,13 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { responseWithJSON(w, &response{ // ImagesHashList: imagesHashList, - Token: tokenizer.Compose(*tokenData), - UserUUID: userUUID, - SessionID: strconv.FormatUint(tokenData.ID, 10), + Token: tokenizer.Compose(*tokenData), + UserUUID: userUUID, + SessionID: strconv.FormatUint(tokenData.ID, 10), BeaconSizeLimit: BEACON_SIZE_LIMIT, }) } - func pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { sessionData, err := tokenizer.ParseFromHTTPRequest(r) if err != nil { @@ -136,8 +135,6 @@ func pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { pushMessages(w, r, sessionData.ID, TOPIC_RAW_IOS) } - - func pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { sessionData, err := tokenizer.ParseFromHTTPRequest(r) if err != nil && err != token.EXPIRED { @@ -145,10 +142,9 @@ func pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { return } // Check timestamps here? - pushMessages(w, r, sessionData.ID,TOPIC_RAW_IOS) + pushMessages(w, r, sessionData.ID, TOPIC_RAW_IOS) } - func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { log.Printf("recieved imagerequest") @@ -159,16 +155,16 @@ func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { } r.Body = http.MaxBytesReader(w, r.Body, FILES_SIZE_LIMIT) - // defer r.Body.Close() + defer r.Body.Close() err = r.ParseMultipartForm(1e6) // ~1Mb if err == http.ErrNotMultipart || err == http.ErrMissingBoundary { responseWithError(w, http.StatusUnsupportedMediaType, err) - // } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB + // } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB } else if err != nil { responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging } - if (r.MultipartForm == nil) { + if r.MultipartForm == nil { responseWithError(w, http.StatusInternalServerError, errors.New("Multipart not parsed")) } @@ -177,7 +173,7 @@ func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { return } - prefix := r.MultipartForm.Value["projectKey"][0] + "/" + strconv.FormatUint(sessionData.ID, 10) + "/" + prefix := r.MultipartForm.Value["projectKey"][0] + "/" + strconv.FormatUint(sessionData.ID, 10) + "/" for _, fileHeaderList := range r.MultipartForm.File { for _, fileHeader := range fileHeaderList { @@ -187,7 +183,7 @@ func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { } key := prefix + fileHeader.Filename log.Printf("Uploading image... %v", key) - go func() { //TODO: mime type from header + go func() { //TODO: mime type from header if err := s3.Upload(file, key, "image/jpeg", false); err != nil { log.Printf("Upload ios screen error. %v", err) } diff --git a/backend/services/http/handlers_web.go b/backend/services/http/handlers-web.go similarity index 91% rename from backend/services/http/handlers_web.go rename to backend/services/http/handlers-web.go index 09d2511d8..dcbd33720 100644 --- a/backend/services/http/handlers_web.go +++ b/backend/services/http/handlers-web.go @@ -11,8 +11,8 @@ import ( "time" "openreplay/backend/pkg/db/postgres" - "openreplay/backend/pkg/token" . "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/token" ) func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { @@ -30,18 +30,18 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { UserID string `json:"userID"` } type response struct { - Timestamp int64 `json:"timestamp"` - Delay int64 `json:"delay"` - Token string `json:"token"` - UserUUID string `json:"userUUID"` - SessionID string `json:"sessionID"` - BeaconSizeLimit int64 `json:"beaconSizeLimit"` + Timestamp int64 `json:"timestamp"` + Delay int64 `json:"delay"` + Token string `json:"token"` + UserUUID string `json:"userUUID"` + SessionID string `json:"sessionID"` + BeaconSizeLimit int64 `json:"beaconSizeLimit"` } startTime := time.Now() req := &request{} body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) // what if Body == nil?? // use r.ContentLength to return specific error? - //defer body.Close() + defer body.Close() if err := json.NewDecoder(body).Decode(req); err != nil { responseWithError(w, http.StatusBadRequest, err) return @@ -76,14 +76,14 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { responseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } - sessionID, err := flaker.Compose(uint64(startTime.UnixNano() / 1e6)) + sessionID, err := flaker.Compose(uint64(startTime.UnixMilli())) if err != nil { responseWithError(w, http.StatusInternalServerError, err) return } // TODO: if EXPIRED => send message for two sessions association expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) - tokenData = &token.TokenData{sessionID, expTime.UnixNano() / 1e6} + tokenData = &token.TokenData{sessionID, expTime.UnixMilli()} country := geoIP.ExtractISOCodeFromHTTPRequest(r) producer.Produce(TOPIC_RAW_WEB, tokenData.ID, Encode(&SessionStart{ @@ -102,17 +102,17 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { UserCountry: country, UserDeviceMemorySize: req.DeviceMemory, UserDeviceHeapSize: req.JsHeapSizeLimit, - UserID: req.UserID, + UserID: req.UserID, })) } //delayDuration := time.Now().Sub(startTime) responseWithJSON(w, &response{ - //Timestamp: startTime.UnixNano() / 1e6, - //Delay: delayDuration.Nanoseconds() / 1e6, - Token: tokenizer.Compose(*tokenData), - UserUUID: userUUID, - SessionID: strconv.FormatUint(tokenData.ID, 10), + //Timestamp: startTime.UnixMilli(), + //Delay: delayDuration.Milliseconds(), + Token: tokenizer.Compose(*tokenData), + UserUUID: userUUID, + SessionID: strconv.FormatUint(tokenData.ID, 10), BeaconSizeLimit: BEACON_SIZE_LIMIT, }) } @@ -124,7 +124,7 @@ func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { return } body := http.MaxBytesReader(w, r.Body, BEACON_SIZE_LIMIT) - //defer body.Close() + defer body.Close() buf, err := ioutil.ReadAll(body) if err != nil { responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging @@ -248,4 +248,4 @@ func notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { log.Printf("Unable to insert Unstarted Session: %v\n", err) } w.WriteHeader(http.StatusOK) -} \ No newline at end of file +} diff --git a/backend/services/http/handlers.go b/backend/services/http/handlers.go index e45e84e64..dd73925af 100644 --- a/backend/services/http/handlers.go +++ b/backend/services/http/handlers.go @@ -9,11 +9,11 @@ import ( gzip "github.com/klauspost/pgzip" ) -const JSON_SIZE_LIMIT int64 = 1e3 // 1Kb +const JSON_SIZE_LIMIT int64 = 1e3 // 1Kb func pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64, topicName string) { body := http.MaxBytesReader(w, r.Body, BEACON_SIZE_LIMIT) - //defer body.Close() + defer body.Close() var reader io.ReadCloser var err error switch r.Header.Get("Content-Encoding") { diff --git a/backend/services/http/ios-device.go b/backend/services/http/ios-device.go new file mode 100644 index 000000000..bec1f3b36 --- /dev/null +++ b/backend/services/http/ios-device.go @@ -0,0 +1,138 @@ +package main + +import ( + "strings" +) + +func MapIOSDevice(identifier string) string { + switch identifier { + case "iPod5,1": + return "iPod touch (5th generation)" + case "iPod7,1": + return "iPod touch (6th generation)" + case "iPod9,1": + return "iPod touch (7th generation)" + case "iPhone3,1", "iPhone3,2", "iPhone3,3": + return "iPhone 4" + case "iPhone4,1": + return "iPhone 4s" + case "iPhone5,1", "iPhone5,2": + return "iPhone 5" + case "iPhone5,3", "iPhone5,4": + return "iPhone 5c" + case "iPhone6,1", "iPhone6,2": + return "iPhone 5s" + case "iPhone7,2": + return "iPhone 6" + case "iPhone7,1": + return "iPhone 6 Plus" + case "iPhone8,1": + return "iPhone 6s" + case "iPhone8,2": + return "iPhone 6s Plus" + case "iPhone8,4": + return "iPhone SE" + case "iPhone9,1", "iPhone9,3": + return "iPhone 7" + case "iPhone9,2", "iPhone9,4": + return "iPhone 7 Plus" + case "iPhone10,1", "iPhone10,4": + return "iPhone 8" + case "iPhone10,2", "iPhone10,5": + return "iPhone 8 Plus" + case "iPhone10,3", "iPhone10,6": + return "iPhone X" + case "iPhone11,2": + return "iPhone XS" + case "iPhone11,4", "iPhone11,6": + return "iPhone XS Max" + case "iPhone11,8": + return "iPhone XR" + case "iPhone12,1": + return "iPhone 11" + case "iPhone12,3": + return "iPhone 11 Pro" + case "iPhone12,5": + return "iPhone 11 Pro Max" + case "iPhone12,8": + return "iPhone SE (2nd generation)" + case "iPhone13,1": + return "iPhone 12 mini" + case "iPhone13,2": + return "iPhone 12" + case "iPhone13,3": + return "iPhone 12 Pro" + case "iPhone13,4": + return "iPhone 12 Pro Max" + case "iPad2,1", "iPad2,2", "iPad2,3", "iPad2,4": + return "iPad 2" + case "iPad3,1", "iPad3,2", "iPad3,3": + return "iPad (3rd generation)" + case "iPad3,4", "iPad3,5", "iPad3,6": + return "iPad (4th generation)" + case "iPad6,11", "iPad6,12": + return "iPad (5th generation)" + case "iPad7,5", "iPad7,6": + return "iPad (6th generation)" + case "iPad7,11", "iPad7,12": + return "iPad (7th generation)" + case "iPad11,6", "iPad11,7": + return "iPad (8th generation)" + case "iPad4,1", "iPad4,2", "iPad4,3": + return "iPad Air" + case "iPad5,3", "iPad5,4": + return "iPad Air 2" + case "iPad11,3", "iPad11,4": + return "iPad Air (3rd generation)" + case "iPad13,1", "iPad13,2": + return "iPad Air (4th generation)" + case "iPad2,5", "iPad2,6", "iPad2,7": + return "iPad mini" + case "iPad4,4", "iPad4,5", "iPad4,6": + return "iPad mini 2" + case "iPad4,7", "iPad4,8", "iPad4,9": + return "iPad mini 3" + case "iPad5,1", "iPad5,2": + return "iPad mini 4" + case "iPad11,1", "iPad11,2": + return "iPad mini (5th generation)" + case "iPad6,3", "iPad6,4": + return "iPad Pro (9.7-inch)" + case "iPad7,3", "iPad7,4": + return "iPad Pro (10.5-inch)" + case "iPad8,1", "iPad8,2", "iPad8,3", "iPad8,4": + return "iPad Pro (11-inch) (1st generation)" + case "iPad8,9", "iPad8,10": + return "iPad Pro (11-inch) (2nd generation)" + case "iPad6,7", "iPad6,8": + return "iPad Pro (12.9-inch) (1st generation)" + case "iPad7,1", "iPad7,2": + return "iPad Pro (12.9-inch) (2nd generation)" + case "iPad8,5", "iPad8,6", "iPad8,7", "iPad8,8": + return "iPad Pro (12.9-inch) (3rd generation)" + case "iPad8,11", "iPad8,12": + return "iPad Pro (12.9-inch) (4th generation)" + case "AppleTV5,3": + return "Apple TV" + case "AppleTV6,2": + return "Apple TV 4K" + case "AudioAccessory1,1": + return "HomePod" + case "AudioAccessory5,1": + return "HomePod mini" + case "i386", "x86_64": + return "Simulator" + default: + return identifier + } +} + +func GetIOSDeviceType(identifier string) string { + if strings.Contains(identifier, "iPhone") { + return "mobile" //"phone" + } + if strings.Contains(identifier, "iPad") { + return "tablet" + } + return "other" +} diff --git a/backend/services/http/ios_device.go b/backend/services/http/ios_device.go deleted file mode 100644 index 2c3474157..000000000 --- a/backend/services/http/ios_device.go +++ /dev/null @@ -1,79 +0,0 @@ -package main - -import ( - "strings" -) - -func MapIOSDevice(identifier string) string { - switch identifier { - case "iPod5,1": return "iPod touch (5th generation)" - case "iPod7,1": return "iPod touch (6th generation)" - case "iPod9,1": return "iPod touch (7th generation)" - case "iPhone3,1", "iPhone3,2", "iPhone3,3": return "iPhone 4" - case "iPhone4,1": return "iPhone 4s" - case "iPhone5,1", "iPhone5,2": return "iPhone 5" - case "iPhone5,3", "iPhone5,4": return "iPhone 5c" - case "iPhone6,1", "iPhone6,2": return "iPhone 5s" - case "iPhone7,2": return "iPhone 6" - case "iPhone7,1": return "iPhone 6 Plus" - case "iPhone8,1": return "iPhone 6s" - case "iPhone8,2": return "iPhone 6s Plus" - case "iPhone8,4": return "iPhone SE" - case "iPhone9,1", "iPhone9,3": return "iPhone 7" - case "iPhone9,2", "iPhone9,4": return "iPhone 7 Plus" - case "iPhone10,1", "iPhone10,4": return "iPhone 8" - case "iPhone10,2", "iPhone10,5": return "iPhone 8 Plus" - case "iPhone10,3", "iPhone10,6": return "iPhone X" - case "iPhone11,2": return "iPhone XS" - case "iPhone11,4", "iPhone11,6": return "iPhone XS Max" - case "iPhone11,8": return "iPhone XR" - case "iPhone12,1": return "iPhone 11" - case "iPhone12,3": return "iPhone 11 Pro" - case "iPhone12,5": return "iPhone 11 Pro Max" - case "iPhone12,8": return "iPhone SE (2nd generation)" - case "iPhone13,1": return "iPhone 12 mini" - case "iPhone13,2": return "iPhone 12" - case "iPhone13,3": return "iPhone 12 Pro" - case "iPhone13,4": return "iPhone 12 Pro Max" - case "iPad2,1", "iPad2,2", "iPad2,3", "iPad2,4":return "iPad 2" - case "iPad3,1", "iPad3,2", "iPad3,3": return "iPad (3rd generation)" - case "iPad3,4", "iPad3,5", "iPad3,6": return "iPad (4th generation)" - case "iPad6,11", "iPad6,12": return "iPad (5th generation)" - case "iPad7,5", "iPad7,6": return "iPad (6th generation)" - case "iPad7,11", "iPad7,12": return "iPad (7th generation)" - case "iPad11,6", "iPad11,7": return "iPad (8th generation)" - case "iPad4,1", "iPad4,2", "iPad4,3": return "iPad Air" - case "iPad5,3", "iPad5,4": return "iPad Air 2" - case "iPad11,3", "iPad11,4": return "iPad Air (3rd generation)" - case "iPad13,1", "iPad13,2": return "iPad Air (4th generation)" - case "iPad2,5", "iPad2,6", "iPad2,7": return "iPad mini" - case "iPad4,4", "iPad4,5", "iPad4,6": return "iPad mini 2" - case "iPad4,7", "iPad4,8", "iPad4,9": return "iPad mini 3" - case "iPad5,1", "iPad5,2": return "iPad mini 4" - case "iPad11,1", "iPad11,2": return "iPad mini (5th generation)" - case "iPad6,3", "iPad6,4": return "iPad Pro (9.7-inch)" - case "iPad7,3", "iPad7,4": return "iPad Pro (10.5-inch)" - case "iPad8,1", "iPad8,2", "iPad8,3", "iPad8,4":return "iPad Pro (11-inch) (1st generation)" - case "iPad8,9", "iPad8,10": return "iPad Pro (11-inch) (2nd generation)" - case "iPad6,7", "iPad6,8": return "iPad Pro (12.9-inch) (1st generation)" - case "iPad7,1", "iPad7,2": return "iPad Pro (12.9-inch) (2nd generation)" - case "iPad8,5", "iPad8,6", "iPad8,7", "iPad8,8":return "iPad Pro (12.9-inch) (3rd generation)" - case "iPad8,11", "iPad8,12": return "iPad Pro (12.9-inch) (4th generation)" - case "AppleTV5,3": return "Apple TV" - case "AppleTV6,2": return "Apple TV 4K" - case "AudioAccessory1,1": return "HomePod" - case "AudioAccessory5,1": return "HomePod mini" - case "i386", "x86_64": return "Simulator" - default: return identifier - } -} - -func GetIOSDeviceType(identifier string) string { - if strings.Contains(identifier, "iPhone") { - return "mobile" //"phone" - } - if strings.Contains(identifier, "iPad") { - return "tablet" - } - return "other" -} \ No newline at end of file diff --git a/backend/services/http/main.go b/backend/services/http/main.go index 8ed8b6d95..1f3bc93b3 100644 --- a/backend/services/http/main.go +++ b/backend/services/http/main.go @@ -10,19 +10,19 @@ import ( "golang.org/x/net/http2" - + "openreplay/backend/pkg/db/cache" + "openreplay/backend/pkg/db/postgres" "openreplay/backend/pkg/env" "openreplay/backend/pkg/flakeid" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/queue/types" "openreplay/backend/pkg/storage" - "openreplay/backend/pkg/db/postgres" - "openreplay/backend/pkg/db/cache" - "openreplay/backend/pkg/url/assets" "openreplay/backend/pkg/token" + "openreplay/backend/pkg/url/assets" "openreplay/backend/services/http/geoip" "openreplay/backend/services/http/uaparser" + "openreplay/backend/pkg/pprof" ) var rewriter *assets.Rewriter @@ -38,12 +38,14 @@ var TOPIC_RAW_WEB string var TOPIC_RAW_IOS string var TOPIC_CACHE string var TOPIC_TRIGGER string + //var TOPIC_ANALYTICS string var CACHE_ASSESTS bool var BEACON_SIZE_LIMIT int64 func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) + pprof.StartProfilingServer() producer = queue.NewProducer() defer producer.Close(15000) @@ -53,7 +55,7 @@ func main() { TOPIC_TRIGGER = env.String("TOPIC_TRIGGER") //TOPIC_ANALYTICS = env.String("TOPIC_ANALYTICS") rewriter = assets.NewRewriter(env.String("ASSETS_ORIGIN")) - pgconn = cache.NewPGCache(postgres.NewConn(env.String("POSTGRES_STRING")), 1000 * 60 * 20) + pgconn = cache.NewPGCache(postgres.NewConn(env.String("POSTGRES_STRING")), 1000*60*20) defer pgconn.Close() s3 = storage.NewS3(env.String("AWS_REGION"), env.String("S3_BUCKET_IOS_IMAGES")) tokenizer = token.NewTokenizer(env.String("TOKEN_SECRET")) @@ -70,7 +72,7 @@ func main() { Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // TODO: agree with specification - w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Methods", "POST") w.Header().Set("Access-Control-Allow-Headers", "Content-Type,Authorization") if r.Method == http.MethodOptions { @@ -79,13 +81,12 @@ func main() { return } - log.Printf("Request: %v - %v ", r.Method, r.URL.Path) - + log.Printf("Request: %v - %v ", r.Method, r.URL.Path) switch r.URL.Path { case "/": w.WriteHeader(http.StatusOK) - case "/v1/web/not-started": + case "/v1/web/not-started": switch r.Method { case http.MethodPost: notStartedHandlerWeb(w, r) diff --git a/backend/services/http/project_id.go b/backend/services/http/project_id.go deleted file mode 100644 index 059576fe8..000000000 --- a/backend/services/http/project_id.go +++ /dev/null @@ -1,12 +0,0 @@ -package main - -func decodeProjectID(projectID uint64) uint64 { - if projectID < 0x10000000000000 || projectID >= 0x20000000000000 { - return 0 - } - projectID = (projectID - 0x10000000000000) * 4212451012670231 & 0xfffffffffffff - if projectID > 0xffffffff { - return 0 - } - return projectID -} diff --git a/backend/services/integrations/integration/bugsnag.go b/backend/services/integrations/integration/bugsnag.go index 7c31db3cb..118cdb84d 100644 --- a/backend/services/integrations/integration/bugsnag.go +++ b/backend/services/integrations/integration/bugsnag.go @@ -1,15 +1,14 @@ package integration import ( + "encoding/json" "fmt" + "io" + "io/ioutil" "net/http" - "encoding/json" "net/url" "time" - "io" - "io/ioutil" - "openreplay/backend/pkg/utime" "openreplay/backend/pkg/messages" ) @@ -18,15 +17,14 @@ import ( */ type bugsnag struct { - BugsnagProjectId string // `json:"bugsnag_project_id"` + BugsnagProjectId string // `json:"bugsnag_project_id"` AuthorizationToken string // `json:"auth_token"` } - type bugsnagEvent struct { MetaData struct { SpecialInfo struct { - AsayerSessionId uint64 `json:"asayerSessionId,string"` + AsayerSessionId uint64 `json:"asayerSessionId,string"` OpenReplaySessionToken string `json:"openReplaySessionToken"` } `json:"special_info"` } `json:"metaData"` @@ -38,7 +36,7 @@ type bugsnagEvent struct { func (b *bugsnag) Request(c *client) error { sinceTs := c.getLastMessageTimestamp() + 1000 // From next second - sinceFormatted := time.Unix(0, int64(sinceTs*1e6)).Format(time.RFC3339) + sinceFormatted := time.UnixMilli(int64(sinceTs)).Format(time.RFC3339) requestURL := fmt.Sprintf("https://api.bugsnag.com/projects/%v/events", b.BugsnagProjectId) req, err := http.NewRequest("GET", requestURL, nil) if err != nil { @@ -47,10 +45,10 @@ func (b *bugsnag) Request(c *client) error { q := req.URL.Query() // q.Add("per_page", "100") // Up to a maximum of 30. Default: 30 // q.Add("sort", "timestamp") // Default: timestamp (timestamp == ReceivedAt ??) - q.Add("direction", "asc") // Default: desc + q.Add("direction", "asc") // Default: desc q.Add("full_reports", "true") // Default: false - q.Add("filters[event.since][][type]", "eq") - q.Add("filters[event.since][][value]", sinceFormatted) // seems like inclusively + q.Add("filters[event.since][][type]", "eq") + q.Add("filters[event.since][][value]", sinceFormatted) // seems like inclusively req.URL.RawQuery = q.Encode() authToken := "token " + b.AuthorizationToken @@ -85,7 +83,7 @@ func (b *bugsnag) Request(c *client) error { } sessionID := e.MetaData.SpecialInfo.AsayerSessionId token := e.MetaData.SpecialInfo.OpenReplaySessionToken - if sessionID == 0 && token == "" { + if sessionID == 0 && token == "" { // c.errChan <- "No AsayerSessionId found. | Message: %v", e continue } @@ -94,16 +92,16 @@ func (b *bugsnag) Request(c *client) error { c.errChan <- err continue } - timestamp := uint64(utime.ToMilliseconds(parsedTime)) + timestamp := uint64(parsedTime.UnixMilli()) c.setLastMessageTimestamp(timestamp) c.evChan <- &SessionErrorEvent{ SessionID: sessionID, - Token: token, + Token: token, RawErrorEvent: &messages.RawErrorEvent{ - Source: "bugsnag", + Source: "bugsnag", Timestamp: timestamp, - Name: e.Exceptions[0].Message, - Payload: string(jsonEvent), + Name: e.Exceptions[0].Message, + Payload: string(jsonEvent), }, } } diff --git a/backend/services/integrations/integration/client.go b/backend/services/integrations/integration/client.go index 2abf9913d..315bfe4e9 100644 --- a/backend/services/integrations/integration/client.go +++ b/backend/services/integrations/integration/client.go @@ -5,10 +5,10 @@ import ( "fmt" "log" "sync" + "time" "openreplay/backend/pkg/db/postgres" "openreplay/backend/pkg/messages" - "openreplay/backend/pkg/utime" ) const MAX_ATTEMPTS_IN_A_ROW = 4 @@ -20,10 +20,10 @@ type requester interface { } type requestData struct { - LastMessageTimestamp uint64 // `json:"lastMessageTimestamp, string"` - LastMessageId string + LastMessageTimestamp uint64 // `json:"lastMessageTimestamp, string"` + LastMessageId string UnsuccessfullAttemptsCount int - LastAttemptTimestamp int64 + LastAttemptTimestamp int64 } type client struct { @@ -31,19 +31,19 @@ type client struct { requester integration *postgres.Integration // TODO: timeout ? - mux sync.Mutex + mux sync.Mutex updateChan chan<- postgres.Integration - evChan chan<- *SessionErrorEvent - errChan chan<- error + evChan chan<- *SessionErrorEvent + errChan chan<- error } type SessionErrorEvent struct { SessionID uint64 - Token string + Token string *messages.RawErrorEvent } -type ClientMap map[ string ]*client +type ClientMap map[string]*client func NewClient(i *postgres.Integration, updateChan chan<- postgres.Integration, evChan chan<- *SessionErrorEvent, errChan chan<- error) (*client, error) { c := new(client) @@ -60,15 +60,14 @@ func NewClient(i *postgres.Integration, updateChan chan<- postgres.Integration, // TODO: RequestData manager if c.requestData.LastMessageTimestamp == 0 { // ? - c.requestData.LastMessageTimestamp = uint64(utime.CurrentTimestamp() - 24*60*60*1000) + c.requestData.LastMessageTimestamp = uint64(time.Now().Add(-time.Hour * 24).UnixMilli()) } return c, nil } - // from outside -func (c* client) Update(i *postgres.Integration) error { +func (c *client) Update(i *postgres.Integration) error { c.mux.Lock() defer c.mux.Unlock() var r requester @@ -111,8 +110,8 @@ func (c *client) getLastMessageTimestamp() uint64 { } func (c *client) setLastMessageId(timestamp uint64, id string) { //if timestamp >= c.requestData.LastMessageTimestamp { - c.requestData.LastMessageId = id - c.requestData.LastMessageTimestamp = timestamp + c.requestData.LastMessageId = id + c.requestData.LastMessageTimestamp = timestamp //} } func (c *client) getLastMessageId() string { @@ -128,18 +127,18 @@ func (c *client) Request() { c.mux.Lock() defer c.mux.Unlock() if c.requestData.UnsuccessfullAttemptsCount >= MAX_ATTEMPTS || - (c.requestData.UnsuccessfullAttemptsCount >= MAX_ATTEMPTS_IN_A_ROW && - utime.CurrentTimestamp() - c.requestData.LastAttemptTimestamp < ATTEMPTS_INTERVAL) { + (c.requestData.UnsuccessfullAttemptsCount >= MAX_ATTEMPTS_IN_A_ROW && + time.Now().UnixMilli()-c.requestData.LastAttemptTimestamp < ATTEMPTS_INTERVAL) { return } - c.requestData.LastAttemptTimestamp = utime.CurrentTimestamp() + c.requestData.LastAttemptTimestamp = time.Now().UnixMilli() err := c.requester.Request(c) if err != nil { log.Println("ERRROR L139") log.Println(err) c.handleError(err) - c.requestData.UnsuccessfullAttemptsCount++; + c.requestData.UnsuccessfullAttemptsCount++ } else { c.requestData.UnsuccessfullAttemptsCount = 0 } @@ -152,5 +151,3 @@ func (c *client) Request() { c.integration.RequestData = rd c.updateChan <- *c.integration } - - diff --git a/backend/services/integrations/integration/datadog.go b/backend/services/integrations/integration/datadog.go index eb7b5daee..096c3b822 100644 --- a/backend/services/integrations/integration/datadog.go +++ b/backend/services/integrations/integration/datadog.go @@ -1,38 +1,37 @@ package integration import ( - "fmt" - "net/http" - "encoding/json" "bytes" - "time" + "encoding/json" + "fmt" "io" - "io/ioutil" + "io/ioutil" + "net/http" + "time" - "openreplay/backend/pkg/utime" "openreplay/backend/pkg/messages" ) -/* +/* We collect Logs. Datadog also has Events */ type datadog struct { - ApplicationKey string //`json:"application_key"` - ApiKey string //`json:"api_key"` + ApplicationKey string //`json:"application_key"` + ApiKey string //`json:"api_key"` } type datadogResponce struct { - Logs []json.RawMessage + Logs []json.RawMessage NextLogId *string - Status string + Status string } type datadogLog struct { Content struct { - Timestamp string - Message string + Timestamp string + Message string Attributes struct { Error struct { // Not sure about this Message string @@ -48,10 +47,10 @@ func (d *datadog) makeRequest(nextLogId *string, fromTs uint64, toTs uint64) (*h d.ApplicationKey, ) startAt := "null" - if nextLogId != nil && *nextLogId != "" { + if nextLogId != nil && *nextLogId != "" { startAt = *nextLogId } - // Query: status:error/info/warning? + // Query: status:error/info/warning? // openReplaySessionToken instead of asayer_session_id jsonBody := fmt.Sprintf(`{ "limit": 1000, @@ -72,8 +71,8 @@ func (d *datadog) makeRequest(nextLogId *string, fromTs uint64, toTs uint64) (*h } func (d *datadog) Request(c *client) error { - fromTs := c.getLastMessageTimestamp() + 1 // From next millisecond - toTs := uint64(utime.CurrentTimestamp()) + fromTs := c.getLastMessageTimestamp() + 1 // From next millisecond + toTs := uint64(time.Now().UnixMilli()) var nextLogId *string for { req, err := d.makeRequest(nextLogId, fromTs, toTs) @@ -111,16 +110,16 @@ func (d *datadog) Request(c *client) error { c.errChan <- err continue } - timestamp := uint64(utime.ToMilliseconds(parsedTime)) + timestamp := uint64(parsedTime.UnixMilli()) c.setLastMessageTimestamp(timestamp) c.evChan <- &SessionErrorEvent{ //SessionID: sessionID, Token: token, RawErrorEvent: &messages.RawErrorEvent{ - Source: "datadog", + Source: "datadog", Timestamp: timestamp, - Name: ddLog.Content.Attributes.Error.Message, - Payload: string(jsonLog), + Name: ddLog.Content.Attributes.Error.Message, + Payload: string(jsonLog), }, } } @@ -129,4 +128,4 @@ func (d *datadog) Request(c *client) error { return nil } } -} \ No newline at end of file +} diff --git a/backend/services/integrations/integration/elasticsearch.go b/backend/services/integrations/integration/elasticsearch.go index 14480e0b8..dd6f5d5f9 100644 --- a/backend/services/integrations/integration/elasticsearch.go +++ b/backend/services/integrations/integration/elasticsearch.go @@ -12,7 +12,6 @@ import ( "time" "openreplay/backend/pkg/messages" - "openreplay/backend/pkg/utime" ) type elasticsearch struct { @@ -164,7 +163,7 @@ func (es *elasticsearch) Request(c *client) error { c.errChan <- err continue } - timestamp := uint64(utime.ToMilliseconds(esLog.Time)) + timestamp := uint64(esLog.Time.UnixMilli()) c.setLastMessageTimestamp(timestamp) var sessionID uint64 diff --git a/backend/services/integrations/integration/newrelic.go b/backend/services/integrations/integration/newrelic.go index 937ab166d..2dce79aa5 100644 --- a/backend/services/integrations/integration/newrelic.go +++ b/backend/services/integrations/integration/newrelic.go @@ -2,25 +2,24 @@ package integration import ( "encoding/json" - "time" + "errors" "fmt" - "net/http" "io" - "io/ioutil" - "errors" + "io/ioutil" + "net/http" + "time" "openreplay/backend/pkg/messages" ) /* - We use insights-api for query. They also have Logs and Events + We use insights-api for query. They also have Logs and Events */ - // TODO: Eu/us type newrelic struct { - ApplicationId string //`json:"application_id"` - XQueryKey string //`json:"x_query_key"` + ApplicationId string //`json:"application_id"` + XQueryKey string //`json:"x_query_key"` } // TODO: Recheck @@ -34,14 +33,14 @@ type newrelicResponce struct { type newrelicEvent struct { //AsayerSessionID uint64 `json:"asayer_session_id,string"` // string/int decoder? OpenReplaySessionToken string `json:"openReplaySessionToken"` - ErrorClass string `json:"error.class"` - Timestamp uint64 `json:"timestamp"` + ErrorClass string `json:"error.class"` + Timestamp uint64 `json:"timestamp"` } func (nr *newrelic) Request(c *client) error { sinceTs := c.getLastMessageTimestamp() + 1000 // From next second // In docs - format "yyyy-mm-dd HH:MM:ss", but time.RFC3339 works fine too - sinceFormatted := time.Unix(0, int64(sinceTs*1e6)).Format(time.RFC3339) + sinceFormatted := time.UnixMilli(int64(sinceTs)).Format(time.RFC3339) // US/EU endpoint ?? requestURL := fmt.Sprintf("https://insights-api.eu.newrelic.com/v1/accounts/%v/query", nr.ApplicationId) req, err := http.NewRequest("GET", requestURL, nil) @@ -64,11 +63,10 @@ func (nr *newrelic) Request(c *client) error { } defer resp.Body.Close() - // 401 (unauthorised) if wrong XQueryKey/deploymentServer is wrong or 403 (Forbidden) if ApplicationId is wrong // 400 if Query has problems if resp.StatusCode >= 400 { - io.Copy(ioutil.Discard, resp.Body) // Read the body to free socket + io.Copy(ioutil.Discard, resp.Body) // Read the body to free socket return fmt.Errorf("Newrelic: server respond with the code %v| Request: ", resp.StatusCode, *req) } // Pagination depending on returning metadata ? @@ -92,13 +90,13 @@ func (nr *newrelic) Request(c *client) error { c.evChan <- &SessionErrorEvent{ Token: e.OpenReplaySessionToken, RawErrorEvent: &messages.RawErrorEvent{ - Source: "newrelic", + Source: "newrelic", Timestamp: e.Timestamp, - Name: e.ErrorClass, - Payload: string(jsonEvent), + Name: e.ErrorClass, + Payload: string(jsonEvent), }, } } } return nil -} \ No newline at end of file +} diff --git a/backend/services/integrations/integration/sentry.go b/backend/services/integrations/integration/sentry.go index 0330430c3..1c5bfdaad 100644 --- a/backend/services/integrations/integration/sentry.go +++ b/backend/services/integrations/integration/sentry.go @@ -1,44 +1,41 @@ package integration import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" "net/http" "net/url" - "encoding/json" - "strings" - "fmt" - "time" "strconv" - "io" - "io/ioutil" + "strings" + "time" - "openreplay/backend/pkg/utime" "openreplay/backend/pkg/messages" ) - -/* +/* They also have different stuff - Documentation says: + Documentation says: "Note: This endpoint is experimental and may be removed without notice." */ type sentry struct { OrganizationSlug string // `json:"organization_slug"` - ProjectSlug string // `json:"project_slug"` - Token string // `json:"token"` + ProjectSlug string // `json:"project_slug"` + Token string // `json:"token"` } type sentryEvent struct { Tags []struct { - Key string - Value string `json:"value"` + Key string + Value string `json:"value"` } - DateCreated string `json:"dateCreated"` // or dateReceived ? - Title string - EventID string `json:"eventID"` + DateCreated string `json:"dateCreated"` // or dateReceived ? + Title string + EventID string `json:"eventID"` } - func (sn *sentry) Request(c *client) error { requestURL := fmt.Sprintf("https://sentry.io/api/0/projects/%v/%v/events/", sn.OrganizationSlug, sn.ProjectSlug) req, err := http.NewRequest("GET", requestURL, nil) @@ -88,9 +85,9 @@ PageLoop: c.errChan <- fmt.Errorf("%v | Event: %v", err, e) continue } - timestamp := uint64(utime.ToMilliseconds(parsedTime)) + timestamp := uint64(parsedTime.UnixMilli()) // TODO: not to receive all the messages (use default integration timestamp) - if firstEvent { // TODO: reverse range? + if firstEvent { // TODO: reverse range? c.setLastMessageId(timestamp, e.EventID) firstEvent = false } @@ -117,12 +114,12 @@ PageLoop: c.evChan <- &SessionErrorEvent{ SessionID: sessionID, - Token: token, + Token: token, RawErrorEvent: &messages.RawErrorEvent{ - Source: "sentry", + Source: "sentry", Timestamp: timestamp, - Name: e.Title, - Payload: string(jsonEvent), + Name: e.Title, + Payload: string(jsonEvent), }, } } @@ -137,7 +134,7 @@ PageLoop: return fmt.Errorf("Link header format error. Got: '%v'", linkHeader) } - nextLinkInfo := pagInfo[ 1 ] + nextLinkInfo := pagInfo[1] if strings.Contains(nextLinkInfo, `results="false"`) { break } @@ -151,4 +148,4 @@ PageLoop: } } return nil -} \ No newline at end of file +} diff --git a/backend/services/integrations/integration/stackdriver.go b/backend/services/integrations/integration/stackdriver.go index bb8e3cef9..e852d5d36 100644 --- a/backend/services/integrations/integration/stackdriver.go +++ b/backend/services/integrations/integration/stackdriver.go @@ -1,22 +1,19 @@ package integration - import ( - "google.golang.org/api/option" "cloud.google.com/go/logging/logadmin" "google.golang.org/api/iterator" - - //"strconv" - "encoding/json" - "time" - "fmt" - "context" + "google.golang.org/api/option" + + //"strconv" + "context" + "encoding/json" + "fmt" + "time" - "openreplay/backend/pkg/utime" "openreplay/backend/pkg/messages" ) - // Old: asayerSessionId const SD_FILTER_QUERY = ` @@ -28,7 +25,7 @@ const SD_FILTER_QUERY = ` type stackdriver struct { ServiceAccountCredentials string // `json:"service_account_credentials"` - LogName string // `json:"log_name"` + LogName string // `json:"log_name"` } type saCreds struct { @@ -37,10 +34,10 @@ type saCreds struct { func (sd *stackdriver) Request(c *client) error { fromTs := c.getLastMessageTimestamp() + 1 // Timestamp is RFC3339Nano, so we take the next millisecond - fromFormatted := time.Unix(0, int64(fromTs *1e6)).Format(time.RFC3339Nano) + fromFormatted := time.UnixMilli(int64(fromTs)).Format(time.RFC3339Nano) ctx := context.Background() - var parsedCreds saCreds + var parsedCreds saCreds err := json.Unmarshal([]byte(sd.ServiceAccountCredentials), &parsedCreds) if err != nil { return err @@ -49,56 +46,56 @@ func (sd *stackdriver) Request(c *client) error { opt := option.WithCredentialsJSON([]byte(sd.ServiceAccountCredentials)) client, err := logadmin.NewClient(ctx, parsedCreds.ProjectId, opt) if err != nil { - return err + return err } defer client.Close() - - filter := fmt.Sprintf(SD_FILTER_QUERY, parsedCreds.ProjectId, sd.LogName, fromFormatted) - // By default, Entries are listed from oldest to newest. - /* ResourceNames(rns []string) - "projects/[PROJECT_ID]" - "organizations/[ORGANIZATION_ID]" - "billingAccounts/[BILLING_ACCOUNT_ID]" - "folders/[FOLDER_ID]" - */ - it := client.Entries(ctx, logadmin.Filter(filter)) - // TODO: Pagination: - //pager := iterator.NewPager(it, 1000, "") - //nextToken, err := pager.NextPage(&entries) - //if nextToken == "" { break } - for { - e, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - return err - } + filter := fmt.Sprintf(SD_FILTER_QUERY, parsedCreds.ProjectId, sd.LogName, fromFormatted) + // By default, Entries are listed from oldest to newest. + /* ResourceNames(rns []string) + "projects/[PROJECT_ID]" + "organizations/[ORGANIZATION_ID]" + "billingAccounts/[BILLING_ACCOUNT_ID]" + "folders/[FOLDER_ID]" + */ + it := client.Entries(ctx, logadmin.Filter(filter)) - token := e.Labels["openReplaySessionToken"] - // sessionID, err := strconv.ParseUint(strSessionID, 10, 64) - // if err != nil { - // c.errChan <- err - // continue - // } - jsonEvent, err := json.Marshal(e) - if err != nil { - c.errChan <- err - continue - } - timestamp := uint64(utime.ToMilliseconds(e.Timestamp)) - c.setLastMessageTimestamp(timestamp) - c.evChan <- &SessionErrorEvent{ + // TODO: Pagination: + //pager := iterator.NewPager(it, 1000, "") + //nextToken, err := pager.NextPage(&entries) + //if nextToken == "" { break } + for { + e, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return err + } + + token := e.Labels["openReplaySessionToken"] + // sessionID, err := strconv.ParseUint(strSessionID, 10, 64) + // if err != nil { + // c.errChan <- err + // continue + // } + jsonEvent, err := json.Marshal(e) + if err != nil { + c.errChan <- err + continue + } + timestamp := uint64(e.Timestamp.UnixMilli()) + c.setLastMessageTimestamp(timestamp) + c.evChan <- &SessionErrorEvent{ //SessionID: sessionID, Token: token, RawErrorEvent: &messages.RawErrorEvent{ - Source: "stackdriver", + Source: "stackdriver", Timestamp: timestamp, - Name: e.InsertID, // not sure about that - Payload: string(jsonEvent), + Name: e.InsertID, // not sure about that + Payload: string(jsonEvent), }, } } return nil -} \ No newline at end of file +} diff --git a/backend/services/integrations/integration/sumologic.go b/backend/services/integrations/integration/sumologic.go index 2660dd6ac..8ff39ec9e 100644 --- a/backend/services/integrations/integration/sumologic.go +++ b/backend/services/integrations/integration/sumologic.go @@ -1,20 +1,19 @@ package integration import ( - "net/http" - "time" "encoding/json" "fmt" - "strings" "io" - "io/ioutil" + "io/ioutil" + "net/http" + "strings" + "time" - "openreplay/backend/pkg/utime" "openreplay/backend/pkg/messages" ) -/* - The maximum value for limit is 10,000 messages or 100 MB in total message size, +/* + The maximum value for limit is 10,000 messages or 100 MB in total message size, which means the query may return less than 10,000 messages if you exceed the size limit. API Documentation: https://help.sumologic.com/APIs/Search-Job-API/About-the-Search-Job-API @@ -22,31 +21,30 @@ import ( const SL_LIMIT = 10000 type sumologic struct { - AccessId string // `json:"access_id"` - AccessKey string // `json:"access_key"` - cookies []*http.Cookie + AccessId string // `json:"access_id"` + AccessKey string // `json:"access_key"` + cookies []*http.Cookie } - type sumplogicJobResponce struct { Id string } type sumologicJobStatusResponce struct { - State string + State string MessageCount int //PendingErrors []string } type sumologicResponce struct { - Messages [] struct { + Messages []struct { Map json.RawMessage } } type sumologicEvent struct { Timestamp uint64 `json:"_messagetime,string"` - Raw string `json:"_raw"` + Raw string `json:"_raw"` } func (sl *sumologic) deleteJob(jobId string, errChan chan<- error) { @@ -68,10 +66,9 @@ func (sl *sumologic) deleteJob(jobId string, errChan chan<- error) { resp.Body.Close() } - func (sl *sumologic) Request(c *client) error { fromTs := c.getLastMessageTimestamp() + 1 // From next millisecond - toTs := utime.CurrentTimestamp() + toTs := time.Now().UnixMilli() requestURL := fmt.Sprintf("https://api.%vsumologic.com/api/v1/search/jobs", "eu.") // deployment server?? jsonBody := fmt.Sprintf(`{ "query": "\"openReplaySessionToken=\" AND (*error* OR *fail* OR *exception*)", @@ -132,7 +129,7 @@ func (sl *sumologic) Request(c *client) error { tick := time.Tick(5 * time.Second) for { - <- tick + <-tick resp, err = http.DefaultClient.Do(req) if err != nil { return err // TODO: retry, counter/timeout @@ -147,12 +144,12 @@ func (sl *sumologic) Request(c *client) error { } if jobStatus.State == "DONE GATHERING RESULTS" { offset := 0 - for ;offset < jobStatus.MessageCount; { + for offset < jobStatus.MessageCount { requestURL = fmt.Sprintf( - "https://api.%vsumologic.com/api/v1/search/jobs/%v/messages?offset=%v&limit=%v", - "eu.", - jobResponce.Id, - offset, + "https://api.%vsumologic.com/api/v1/search/jobs/%v/messages?offset=%v&limit=%v", + "eu.", + jobResponce.Id, + offset, SL_LIMIT, ) req, err = http.NewRequest("GET", requestURL, nil) @@ -190,17 +187,17 @@ func (sl *sumologic) Request(c *client) error { } name := e.Raw if len(name) > 20 { - name = name[:20] // not sure about that + name = name[:20] // not sure about that } c.setLastMessageTimestamp(e.Timestamp) c.evChan <- &SessionErrorEvent{ //SessionID: sessionID, Token: token, RawErrorEvent: &messages.RawErrorEvent{ - Source: "sumologic", + Source: "sumologic", Timestamp: e.Timestamp, - Name: name, - Payload: string(m.Map), //e.Raw ? + Name: name, + Payload: string(m.Map), //e.Raw ? }, } @@ -209,11 +206,11 @@ func (sl *sumologic) Request(c *client) error { } break } - if jobStatus.State != "NOT STARTED" && + if jobStatus.State != "NOT STARTED" && jobStatus.State != "GATHERING RESULTS" { // error break } } return nil -} \ No newline at end of file +} diff --git a/backend/services/sink/main.go b/backend/services/sink/main.go index 5893e93e6..a649bb6ef 100644 --- a/backend/services/sink/main.go +++ b/backend/services/sink/main.go @@ -1,8 +1,8 @@ package main import ( - "log" "encoding/binary" + "log" "time" "os" @@ -10,67 +10,64 @@ import ( "syscall" "openreplay/backend/pkg/env" + . "openreplay/backend/pkg/messages" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/queue/types" - . "openreplay/backend/pkg/messages" ) - - func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - FS_DIR := env.String("FS_DIR"); + FS_DIR := env.String("FS_DIR") if _, err := os.Stat(FS_DIR); os.IsNotExist(err) { log.Fatalf("%v doesn't exist. %v", FS_DIR, err) } writer := NewWriter(env.Uint16("FS_ULIMIT"), FS_DIR) - count := 0 + count := 0 consumer := queue.NewMessageConsumer( env.String("GROUP_SINK"), - []string{ + []string{ env.String("TOPIC_RAW_WEB"), env.String("TOPIC_RAW_IOS"), - }, - func(sessionID uint64, message Message, _ *types.Meta) { - //typeID, err := GetMessageTypeID(value) - // if err != nil { - // log.Printf("Message type decoding error: %v", err) - // return - // } - typeID := message.Meta().TypeID - if !IsReplayerType(typeID) { - return - } + }, + func(sessionID uint64, message Message, _ *types.Meta) { + //typeID, err := GetMessageTypeID(value) + // if err != nil { + // log.Printf("Message type decoding error: %v", err) + // return + // } + typeID := message.Meta().TypeID + if !IsReplayerType(typeID) { + return + } - count++ + count++ - value := message.Encode() - var data []byte - if IsIOSType(typeID) { - data = value - } else { + value := message.Encode() + var data []byte + if IsIOSType(typeID) { + data = value + } else { data = make([]byte, len(value)+8) copy(data[8:], value[:]) binary.LittleEndian.PutUint64(data[0:], message.Meta().Index) - } - if err := writer.Write(sessionID, data); err != nil { + } + if err := writer.Write(sessionID, data); err != nil { log.Printf("Writer error: %v\n", err) } - }, + }, + false, ) - consumer.DisableAutoCommit() - sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - tick := time.Tick(30 * time.Second) + tick := time.Tick(30 * time.Second) - log.Printf("Sink service started\n") + log.Printf("Sink service started\n") for { select { case sig := <-sigchan: @@ -85,7 +82,7 @@ func main() { log.Printf("%v messages during 30 sec", count) count = 0 - + consumer.Commit() default: err := consumer.ConsumeNext() @@ -96,4 +93,3 @@ func main() { } } - diff --git a/backend/services/storage/clean.go b/backend/services/storage/clean.go index 829bc8705..72f5f359c 100644 --- a/backend/services/storage/clean.go +++ b/backend/services/storage/clean.go @@ -1,23 +1,23 @@ package main import ( - "os" - "log" - "time" - "strconv" "io/ioutil" + "log" + "os" + "strconv" + "time" "openreplay/backend/pkg/flakeid" ) -const DELETE_TIMEOUT = 12 * time.Hour; +const DELETE_TIMEOUT = 48 * time.Hour func cleanDir(dirname string) { - files, err := ioutil.ReadDir(dirname) - if err != nil { - log.Printf("Cannot read file directory. %v", err) - return - } + files, err := ioutil.ReadDir(dirname) + if err != nil { + log.Printf("Cannot read file directory. %v", err) + return + } for _, f := range files { name := f.Name() @@ -27,8 +27,9 @@ func cleanDir(dirname string) { continue } ts := int64(flakeid.ExtractTimestamp(id)) - if time.Unix(ts/1000, 0).Add(DELETE_TIMEOUT).Before(time.Now()) { + if time.UnixMilli(ts).Add(DELETE_TIMEOUT).Before(time.Now()) { + // returns a error. Don't log it sinse it can be race condition between worker instances os.Remove(dirname + "/" + name) } } -} \ No newline at end of file +} diff --git a/backend/services/storage/main.go b/backend/services/storage/main.go index 5033fb845..9579fbe4f 100644 --- a/backend/services/storage/main.go +++ b/backend/services/storage/main.go @@ -2,45 +2,41 @@ package main import ( "log" - "time" "os" "strconv" + "time" "os/signal" "syscall" "openreplay/backend/pkg/env" - "openreplay/backend/pkg/storage" "openreplay/backend/pkg/messages" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/queue/types" + "openreplay/backend/pkg/storage" ) - - func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - - storageWeb := storage.NewS3(env.String("AWS_REGION_WEB"), env.String("S3_BUCKET_WEB")) - //storageIos := storage.NewS3(env.String("AWS_REGION_IOS"), env.String("S3_BUCKET_IOS")) + storage := storage.NewS3(env.String("AWS_REGION_WEB"), env.String("S3_BUCKET_WEB")) FS_DIR := env.String("FS_DIR") FS_CLEAN_HRS := env.Int("FS_CLEAN_HRS") - var uploadKey func(string, int, *storage.S3) - uploadKey = func(key string, retryCount int, s *storage.S3) { + var uploadKey func(string, int) + uploadKey = func(key string, retryCount int) { if retryCount <= 0 { - return; + return } file, err := os.Open(FS_DIR + "/" + key) defer file.Close() if err != nil { log.Printf("File error: %v; Will retry %v more time(s)\n", err, retryCount) time.AfterFunc(2*time.Minute, func() { - uploadKey(key, retryCount - 1, s) + uploadKey(key, retryCount-1) }) } else { - if err := s.Upload(gzipFile(file), key, "application/octet-stream", true); err != nil { + if err := storage.Upload(gzipFile(file), key, "application/octet-stream", true); err != nil { log.Fatalf("Storage upload error: %v\n", err) } } @@ -48,27 +44,24 @@ func main() { consumer := queue.NewMessageConsumer( env.String("GROUP_STORAGE"), - []string{ + []string{ env.String("TOPIC_TRIGGER"), - }, - func(sessionID uint64, msg messages.Message, meta *types.Meta) { - switch msg.(type) { - case *messages.SessionEnd: - uploadKey(strconv.FormatUint(sessionID, 10), 5, storageWeb) - //case *messages.IOSSessionEnd: - // uploadKey(strconv.FormatUint(sessionID, 10), 5, storageIos) - } - }, + }, + func(sessionID uint64, msg messages.Message, meta *types.Meta) { + switch msg.(type) { + case *messages.SessionEnd: + uploadKey(strconv.FormatUint(sessionID, 10), 5) + } + }, + true, ) sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + cleanTick := time.Tick(time.Duration(FS_CLEAN_HRS) * time.Hour) - cleanTick := time.Tick(time.Duration(FS_CLEAN_HRS) * time.Hour) - - - log.Printf("Storage service started\n") + log.Printf("Storage service started\n") for { select { case sig := <-sigchan: @@ -76,7 +69,7 @@ func main() { consumer.Close() os.Exit(0) case <-cleanTick: - cleanDir(FS_DIR) + go cleanDir(FS_DIR) default: err := consumer.ConsumeNext() if err != nil { @@ -85,4 +78,3 @@ func main() { } } } - diff --git a/ee/api/.env.default b/ee/api/.env.default index 28f46f273..b4b1ad291 100644 --- a/ee/api/.env.default +++ b/ee/api/.env.default @@ -46,11 +46,13 @@ pg_port=5432 pg_user=postgres pg_timeout=30 pg_minconn=45 +PG_RETRY_MAX=50 +PG_RETRY_INTERVAL=2 put_S3_TTL=20 sentryURL= sessions_bucket=mobs sessions_region=us-east-1 sourcemaps_bucket=sourcemaps -sourcemaps_reader=http://utilities-openreplay.app.svc.cluster.local:9000/sourcemaps +sourcemaps_reader=http://127.0.0.1:9000/ stage=default-ee version_number=1.0.0 diff --git a/ee/api/.gitignore b/ee/api/.gitignore index f1ff9550b..f8ff0f789 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -180,9 +180,6 @@ Pipfile /chalicelib/core/alerts.py /chalicelib/core/alerts_processor.py /chalicelib/core/announcements.py -/chalicelib/blueprints/bp_app_api.py -/chalicelib/blueprints/bp_core.py -/chalicelib/blueprints/bp_core_crons.py /chalicelib/core/collaboration_slack.py /chalicelib/core/errors_favorite_viewed.py /chalicelib/core/events.py @@ -237,7 +234,6 @@ Pipfile /chalicelib/utils/smtp.py /chalicelib/utils/strings.py /chalicelib/utils/TimeUTC.py -/chalicelib/blueprints/app/__init__.py /routers/app/__init__.py /routers/crons/__init__.py /routers/subs/__init__.py @@ -245,8 +241,8 @@ Pipfile /chalicelib/core/assist.py /auth/auth_apikey.py /auth/auth_jwt.py -/chalicelib/blueprints/subs/bp_insights.py /build.sh +/routers/base.py /routers/core.py /routers/crons/core_crons.py /routers/subs/dashboard.py @@ -257,10 +253,12 @@ Pipfile /chalicelib/core/heatmaps.py /routers/subs/insights.py /schemas.py -/chalicelib/blueprints/app/v1_api.py -/routers/app/v1_api.py /chalicelib/core/custom_metrics.py /chalicelib/core/performance_event.py /chalicelib/core/saved_search.py /app_alerts.py /build_alerts.sh +/routers/subs/metrics.py +/routers/subs/v1_api.py +/chalicelib/core/dashboards2.py +entrypoint.sh \ No newline at end of file diff --git a/ee/api/Dockerfile b/ee/api/Dockerfile index cca6e6806..aee6aecb2 100644 --- a/ee/api/Dockerfile +++ b/ee/api/Dockerfile @@ -6,6 +6,15 @@ WORKDIR /work COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env +ENV APP_NAME chalice +# Installing Nodejs +RUN apt update && apt install -y curl && \ + curl -fsSL https://deb.nodesource.com/setup_12.x | bash - && \ + apt install -y nodejs && \ + apt remove --purge -y curl && \ + rm -rf /var/lib/apt/lists/* && \ + cd sourcemap-reader && \ + npm install # Add Tini # Startup daemon diff --git a/ee/api/Dockerfile.alerts b/ee/api/Dockerfile.alerts index 9be6ebc93..230514918 100644 --- a/ee/api/Dockerfile.alerts +++ b/ee/api/Dockerfile.alerts @@ -7,6 +7,7 @@ COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env && mv app_alerts.py app.py ENV pg_minconn 2 +ENV APP_NAME alerts # Add Tini # Startup daemon diff --git a/ee/api/_clickhouse_upgrade.sh b/ee/api/_clickhouse_upgrade.sh deleted file mode 100644 index 9b656a584..000000000 --- a/ee/api/_clickhouse_upgrade.sh +++ /dev/null @@ -1,10 +0,0 @@ -sudo yum update -sudo yum install yum-utils -sudo rpm --import https://repo.clickhouse.com/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/stable/x86_64 -sudo yum update -sudo service clickhouse-server restart - - -#later mus use in clickhouse-client: -#SET allow_experimental_window_functions = 1; \ No newline at end of file diff --git a/ee/api/app.py b/ee/api/app.py index fdf7f60b8..ed2c01aa4 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -11,10 +11,10 @@ from starlette.responses import StreamingResponse, JSONResponse from chalicelib.utils import helper from chalicelib.utils import pg_client from routers import core, core_dynamic, ee, saml -from routers.app import v1_api, v1_api_ee +from routers.subs import v1_api from routers.crons import core_crons from routers.crons import core_dynamic_crons -from routers.subs import dashboard +from routers.subs import dashboard, insights, v1_api_ee app = FastAPI() @@ -65,7 +65,7 @@ app.include_router(saml.public_app) app.include_router(saml.app) app.include_router(saml.app_apikey) app.include_router(dashboard.app) -# app.include_router(insights.app) +app.include_router(insights.app) app.include_router(v1_api.app_apikey) app.include_router(v1_api_ee.app_apikey) diff --git a/ee/api/chalicelib/core/boarding.py b/ee/api/chalicelib/core/boarding.py index 6690e59f2..8a2076b58 100644 --- a/ee/api/chalicelib/core/boarding.py +++ b/ee/api/chalicelib/core/boarding.py @@ -6,41 +6,40 @@ from chalicelib.core import projects def get_state(tenant_id): - my_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) - pids = [s["projectId"] for s in my_projects] + pids = projects.get_projects_ids(tenant_id=tenant_id) with pg_client.PostgresClient() as cur: recorded = False meta = False if len(pids) > 0: cur.execute( - cur.mogrify("""\ - SELECT - COUNT(*) - FROM public.sessions AS s - where s.project_id IN %(ids)s - LIMIT 1;""", + cur.mogrify("""SELECT EXISTS(( SELECT 1 + FROM public.sessions AS s + WHERE s.project_id IN %(ids)s)) AS exists;""", {"ids": tuple(pids)}) ) - recorded = cur.fetchone()["count"] > 0 + recorded = cur.fetchone()["exists"] meta = False if recorded: cur.execute( - cur.mogrify("""SELECT SUM((SELECT COUNT(t.meta) - FROM (VALUES (p.metadata_1), (p.metadata_2), (p.metadata_3), (p.metadata_4), (p.metadata_5), - (p.metadata_6), (p.metadata_7), (p.metadata_8), (p.metadata_9), (p.metadata_10), - (sessions.user_id)) AS t(meta) - WHERE t.meta NOTNULL)) - FROM public.projects AS p - LEFT JOIN LATERAL ( SELECT 'defined' - FROM public.sessions - WHERE sessions.project_id=p.project_id AND sessions.user_id IS NOT NULL - LIMIT 1) AS sessions(user_id) ON(TRUE) - WHERE p.tenant_id = %(tenant_id)s - AND p.deleted_at ISNULL;""" + cur.mogrify("""SELECT EXISTS((SELECT 1 + FROM public.projects AS p + LEFT JOIN LATERAL ( SELECT 1 + FROM public.sessions + WHERE sessions.project_id = p.project_id + AND sessions.user_id IS NOT NULL + LIMIT 1) AS sessions(user_id) ON (TRUE) + WHERE p.tenant_id = %(tenant_id)s AND p.deleted_at ISNULL + AND ( sessions.user_id IS NOT NULL OR p.metadata_1 IS NOT NULL + OR p.metadata_2 IS NOT NULL OR p.metadata_3 IS NOT NULL + OR p.metadata_4 IS NOT NULL OR p.metadata_5 IS NOT NULL + OR p.metadata_6 IS NOT NULL OR p.metadata_7 IS NOT NULL + OR p.metadata_8 IS NOT NULL OR p.metadata_9 IS NOT NULL + OR p.metadata_10 IS NOT NULL ) + )) AS exists;""" , {"tenant_id": tenant_id})) - meta = cur.fetchone()["sum"] > 0 + meta = cur.fetchone()["exists"] return [ {"task": "Install OpenReplay", @@ -61,22 +60,18 @@ def get_state(tenant_id): def get_state_installing(tenant_id): - my_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) - pids = [s["projectId"] for s in my_projects] + pids = projects.get_projects_ids(tenant_id=tenant_id) with pg_client.PostgresClient() as cur: recorded = False if len(pids) > 0: cur.execute( - cur.mogrify("""\ - SELECT - COUNT(*) - FROM public.sessions AS s - where s.project_id IN %(ids)s - LIMIT 1;""", + cur.mogrify("""SELECT EXISTS(( SELECT 1 + FROM public.sessions AS s + WHERE s.project_id IN %(ids)s)) AS exists;""", {"ids": tuple(pids)}) ) - recorded = cur.fetchone()["count"] > 0 + recorded = cur.fetchone()["exists"] return {"task": "Install OpenReplay", "done": recorded, @@ -86,21 +81,24 @@ def get_state_installing(tenant_id): def get_state_identify_users(tenant_id): with pg_client.PostgresClient() as cur: cur.execute( - cur.mogrify("""SELECT SUM((SELECT COUNT(t.meta) - FROM (VALUES (p.metadata_1), (p.metadata_2), (p.metadata_3), (p.metadata_4), (p.metadata_5), - (p.metadata_6), (p.metadata_7), (p.metadata_8), (p.metadata_9), (p.metadata_10), - (sessions.user_id)) AS t(meta) - WHERE t.meta NOTNULL)) - FROM public.projects AS p - LEFT JOIN LATERAL ( SELECT 'defined' - FROM public.sessions - WHERE sessions.project_id=p.project_id AND sessions.user_id IS NOT NULL - LIMIT 1) AS sessions(user_id) ON(TRUE) - WHERE p.tenant_id = %(tenant_id)s - AND p.deleted_at ISNULL;""" + cur.mogrify("""SELECT EXISTS((SELECT 1 + FROM public.projects AS p + LEFT JOIN LATERAL ( SELECT 1 + FROM public.sessions + WHERE sessions.project_id = p.project_id + AND sessions.user_id IS NOT NULL + LIMIT 1) AS sessions(user_id) ON (TRUE) + WHERE p.tenant_id = %(tenant_id)s AND p.deleted_at ISNULL + AND ( sessions.user_id IS NOT NULL OR p.metadata_1 IS NOT NULL + OR p.metadata_2 IS NOT NULL OR p.metadata_3 IS NOT NULL + OR p.metadata_4 IS NOT NULL OR p.metadata_5 IS NOT NULL + OR p.metadata_6 IS NOT NULL OR p.metadata_7 IS NOT NULL + OR p.metadata_8 IS NOT NULL OR p.metadata_9 IS NOT NULL + OR p.metadata_10 IS NOT NULL ) + )) AS exists;""" , {"tenant_id": tenant_id})) - meta = cur.fetchone()["sum"] > 0 + meta = cur.fetchone()["exists"] return {"task": "Identify Users", "done": meta, diff --git a/ee/api/chalicelib/core/dashboard.py b/ee/api/chalicelib/core/dashboard.py index c5c373c78..404aef795 100644 --- a/ee/api/chalicelib/core/dashboard.py +++ b/ee/api/chalicelib/core/dashboard.py @@ -1,6 +1,7 @@ import math import random +import schemas from chalicelib.utils import pg_client from chalicelib.utils import args_transformer from chalicelib.utils import helper @@ -169,7 +170,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) ch_query = f"""\ SELECT toUnixTimestamp(toStartOfInterval(sessions.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(sessions.session_id) AS count + COUNT(sessions.session_id) AS value FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -181,19 +182,17 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) rows = ch.execute(query=ch_query, params=params) results = { - "count": sum([r["count"] for r in rows]), + "value": sum([r["value"] for r in rows]), "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, - neutral={"count": 0}) + neutral={"value": 0}) } diff = endTimestamp - startTimestamp endTimestamp = startTimestamp startTimestamp = endTimestamp - diff - ch_query = f"""\ - SELECT - COUNT(sessions.session_id) AS count + ch_query = f""" SELECT COUNT(sessions.session_id) AS count FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)};""" params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, @@ -203,7 +202,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) count = count[0]["count"] - results["countProgress"] = helper.__progress(old_val=count, new_val=results["count"]) + results["progress"] = helper.__progress(old_val=count, new_val=results["value"]) return results @@ -222,9 +221,8 @@ def get_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimesta with ch_client.ClickHouseClient() as ch: ch_query = f"""\ - SELECT - toUnixTimestamp(toStartOfInterval(errors.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(DISTINCT errors.session_id) AS count + SELECT toUnixTimestamp(toStartOfInterval(errors.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, + COUNT(DISTINCT errors.session_id) AS count FROM errors {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -304,9 +302,8 @@ def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), errors = {} for error_id in error_ids: ch_query = f"""\ - SELECT - toUnixTimestamp(toStartOfInterval(errors.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(errors.session_id) AS count + SELECT toUnixTimestamp(toStartOfInterval(errors.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, + COUNT(errors.session_id) AS count FROM errors {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -348,10 +345,8 @@ def __get_page_metrics(ch, project_id, startTimestamp, endTimestamp, **args): ch_sub_query += meta_condition # changed dom_content_loaded_event_start to dom_content_loaded_event_end ch_query = f"""\ - SELECT - COALESCE(AVG(NULLIF(pages.dom_content_loaded_event_end ,0)),0) AS avg_dom_content_load_start, --- COALESCE(AVG(NULLIF(pages.dom_content_loaded_event_start ,0)),0) AS avg_dom_content_load_start, - COALESCE(AVG(NULLIF(pages.first_contentful_paint,0)),0) AS avg_first_contentful_pixel + SELECT COALESCE(AVG(NULLIF(pages.dom_content_loaded_event_end ,0)),0) AS avg_dom_content_load_start, + COALESCE(AVG(NULLIF(pages.first_contentful_paint,0)),0) AS avg_first_contentful_pixel FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)};""" params = {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, @@ -395,8 +390,7 @@ def __get_application_activity(ch, project_id, startTimestamp, endTimestamp, **a ch_sub_query += meta_condition ch_sub_query.append("resources.type= %(type)s") ch_query = f"""\ - SELECT - AVG(NULLIF(resources.duration,0)) AS avg + SELECT AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)};""" row = ch.execute(query=ch_query, @@ -443,9 +437,8 @@ def __get_user_activity(cur, project_id, startTimestamp, endTimestamp, **args): ch_sub_query += meta_condition ch_query = f"""\ - SELECT - COALESCE(CEIL(AVG(NULLIF(sessions.pages_count,0))),0) AS avg_visited_pages, - COALESCE(AVG(NULLIF(sessions.duration,0)),0) AS avg_session_duration + SELECT COALESCE(CEIL(AVG(NULLIF(sessions.pages_count,0))),0) AS avg_visited_pages, + COALESCE(AVG(NULLIF(sessions.duration,0)),0) AS avg_session_duration FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)};""" params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, @@ -464,47 +457,56 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query.append("resources.type = 'img'") ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) ch_sub_query_chart.append("resources.type = 'img'") - ch_sub_query_chart.append("resources.url = %(url)s") + ch_sub_query_chart.append("resources.url IN %(url)s") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT resources.url, - AVG(NULLIF(resources.duration,0)) AS avg, - COUNT(resources.session_id) AS count + AVG(NULLIF(resources.duration,0)) AS avg, + COUNT(resources.session_id) AS count FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY resources.url ORDER BY avg DESC LIMIT 10;""" - - rows = ch.execute(query=ch_query, - params={"project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)}) + params = {"project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + # print(ch.client().substitute_params(ch_query, params)) + rows = ch.execute(query=ch_query, params=params) rows = [{"url": i["url"], "avgDuration": i["avg"], "sessions": i["count"]} for i in rows] - + if len(rows) == 0: + return [] urls = [row["url"] for row in rows] charts = {} + ch_query = f"""\ + SELECT url, + toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS avg + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + GROUP BY url, timestamp + ORDER BY url, timestamp;""" + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, "url": urls, **__get_constraint_values(args)} + # print(ch.client().substitute_params(ch_query, params)) + u_rows = ch.execute(query=ch_query, params=params) for url in urls: - ch_query = f"""\ - SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(resources.duration,0)) AS avg - FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, "url": url, **__get_constraint_values(args)} - r = ch.execute(query=ch_query, params=params) + sub_rows = [] + for r in u_rows: + if r["url"] == url: + sub_rows.append(r) + elif len(sub_rows) > 0: + break charts[url] = [{"timestamp": int(i["timestamp"]), "avgDuration": i["avg"]} - for i in __complete_missing_steps(rows=r, start_time=startTimestamp, + for i in __complete_missing_steps(rows=sub_rows, start_time=startTimestamp, end_time=endTimestamp, density=density, neutral={"avg": 0})] for i in range(len(rows)): rows[i] = helper.dict_to_camel_case(rows[i]) - rows[i]["chart"] = [helper.dict_to_camel_case(chart) for chart in charts[rows[i]["url"]]] + rows[i]["chart"] = helper.list_to_camel_case(charts[rows[i]["url"]]) return sorted(rows, key=lambda k: k["sessions"], reverse=True) @@ -544,9 +546,8 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(resources.duration,0)) AS avg + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} AND resources.type = 'img' @@ -558,9 +559,8 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi __complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, neutral={"avg": 0})] - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(resources.duration,0)) AS avg + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} AND resources.type = 'fetch' @@ -577,9 +577,8 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi data=args) ch_sub_query_chart += meta_condition - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(pages.load_event_end ,0)) AS avg + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(pages.load_event_end ,0)) AS avg FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} @@ -648,9 +647,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, if resource_type == "ALL" and not pages_only and not events_only: ch_sub_query.append("positionUTF8(url_hostpath,%(value)s)!=0") with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - arrayJoin(arraySlice(arrayReverseSort(arrayDistinct(groupArray(url))), 1, 5)) AS value, - type AS key + ch_query = f"""SELECT arrayJoin(arraySlice(arrayReverseSort(arrayDistinct(groupArray(url))), 1, 5)) AS value, + type AS key FROM resources WHERE {" AND ".join(ch_sub_query)} GROUP BY type @@ -685,9 +683,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, ch_sub_query.append(f"resources.type = '{__get_resource_db_type_from_type(resource_type)}'") with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - DISTINCT url_hostpath AS value, - %(resource_type)s AS key + ch_query = f"""SELECT DISTINCT url_hostpath AS value, + %(resource_type)s AS key FROM resources WHERE {" AND ".join(ch_sub_query)} LIMIT 10;""" @@ -787,34 +784,6 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, return [helper.dict_to_camel_case(row) for row in rows] -# def frustration_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1), -# endTimestamp=TimeUTC.now(), **args): -# with pg_client.PostgresClient() as cur: -# sub_q = "" -# if platform == 'mobile': -# sub_q = "AND s.user_device_type = 'mobile' AND s.project_id = %(project_id)s AND s.start_ts >= %(startTimestamp)s AND s.start_ts < %(endTimestamp)s" -# elif platform == 'desktop': -# sub_q = "AND s.user_device_type = 'desktop' AND s.project_id = %(project_id)s AND s.start_ts >= %(startTimestamp)s AND s.start_ts < %(endTimestamp)s" -# -# cur.execute(cur.mogrify(f"""\ -# SELECT s.project_id, -# s.session_id::text AS session_id, -# s.* -# FROM public.sessions AS s -# LEFT JOIN public.session_watchdogs AS sw ON s.session_id=sw.session_id -# LEFT JOIN public.watchdogs AS w ON w.watchdog_id=sw.watchdog_id -# WHERE s.project_id = %(project_id)s -# AND w.type='clickrage' -# AND s.start_ts>=%(startTimestamp)s -# AND s.start_ts<=%(endTimestamp)s -# {sub_q} -# ORDER BY s.session_id DESC -# LIMIT 5;""", -# {"project_id": project_id, "startTimestamp": startTimestamp, -# "endTimestamp": endTimestamp})) -# return helper.list_to_camel_case(cur.fetchall()) - - def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=7, **args): @@ -826,9 +795,8 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day ch_sub_query += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - resources.url_hostpath AS key, - COUNT(resources.session_id) AS doc_count + ch_query = f"""SELECT resources.url_hostpath AS key, + COUNT(resources.session_id) AS doc_count FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY url_hostpath @@ -841,10 +809,9 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day if len(rows) == 0: return [] ch_sub_query.append("resources.url_hostpath = %(value)s") - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - COUNT(resources.session_id) AS doc_count, - toUnixTimestamp(MAX(resources.datetime))*1000 AS max_datatime + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + COUNT(resources.session_id) AS doc_count, + toUnixTimestamp(MAX(resources.datetime))*1000 AS max_datatime FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY timestamp @@ -879,9 +846,8 @@ def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - resources.url_hostpath, COUNT(resources.session_id) AS doc_count + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + resources.url_hostpath, COUNT(resources.session_id) AS doc_count FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp, resources.url_hostpath @@ -935,9 +901,8 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(NULLIF(resources.duration,0)) AS avg + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -969,9 +934,8 @@ def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=- ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(pages.dom_building_time) AS avg + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(pages.dom_building_time) AS value FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -985,10 +949,10 @@ def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=- FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avg": avg, + return {"value": avg, "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, - density=density, neutral={"avg": 0})} + density=density, neutral={"value": 0})} def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1009,54 +973,50 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query_chart.append("isNotNull(resources.duration)") ch_sub_query_chart.append("resources.duration>0") with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - splitByChar('/', resources.url_hostpath)[-1] AS name, - AVG(NULLIF(resources.duration,0)) AS avg + ch_query = f"""SELECT any(url) AS url, any(type) AS type, + splitByChar('/', resources.url_hostpath)[-1] AS name, + AVG(NULLIF(resources.duration,0)) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY name ORDER BY avg DESC LIMIT 10;""" - rows = ch.execute(query=ch_query, - params={"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)}) - ch_sub_query_chart.append("endsWith(resources.url_hostpath, %(url)s)>0") + params = {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + print(ch.format(query=ch_query, params=params)) + rows = ch.execute(query=ch_query, params=params) + ch_sub_query.append(ch_sub_query_chart[-1]) results = [] + names = {f"name_{i}": r["name"] for i, r in enumerate(rows)} + ch_query = f"""SELECT splitByChar('/', resources.url_hostpath)[-1] AS name, + toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(resources.duration) AS avg + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + AND ({" OR ".join([f"endsWith(resources.url_hostpath, %(name_{i})s)>0" for i in range(len(names.keys()))])}) + GROUP BY name,timestamp + ORDER BY name,timestamp;""" + params = {"step_size": step_size, "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + **names, **__get_constraint_values(args)} + # print(ch.format(query=ch_query, params=params)) + charts = ch.execute(query=ch_query, params=params) for r in rows: - # if isinstance(r["url"], bytes): - # try: - # r["url"] = r["url"].decode("utf-8") - # except UnicodeDecodeError: - # continue - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, - AVG(resources.duration) AS avg - FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} - WHERE {" AND ".join(ch_sub_query_chart)} - GROUP BY timestamp - ORDER BY timestamp;""" - chart = ch.execute(query=ch_query, - params={"step_size": step_size, "project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "url": r["name"], **__get_constraint_values(args)}) - r["chart"] = __complete_missing_steps(rows=chart, start_time=startTimestamp, + sub_chart = [] + for c in charts: + if c["name"] == r["name"]: + cc = dict(c) + cc.pop("name") + sub_chart.append(cc) + elif len(sub_chart) > 0: + break + r["chart"] = __complete_missing_steps(rows=sub_chart, start_time=startTimestamp, end_time=endTimestamp, density=density, neutral={"avg": 0}) - ch_query = f"""SELECT url, type - FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} - WHERE {" AND ".join(ch_sub_query)} - ORDER BY duration DESC - LIMIT 1;""" - url = ch.execute(query=ch_query, - params={"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, - "url": r["name"], **__get_constraint_values(args)}) - r["url"] = url[0]["url"] - r["type"] = __get_resource_type_from_db_type(url[0]["type"]) + r["type"] = __get_resource_type_from_db_type(r["type"]) results.append(r) return results @@ -1119,7 +1079,7 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 ch_sub_query_chart.append(f"url_path = %(value)s") with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(pages.response_time) AS avg + AVG(pages.response_time) AS value FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1134,10 +1094,10 @@ def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1 FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avg": avg, + return {"value": avg, "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, - density=density, neutral={"avg": 0})} + density=density, neutral={"value": 0})} def get_pages_response_time_distribution(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1268,9 +1228,8 @@ def get_busiest_time_of_day(project_id, startTimestamp=TimeUTC.now(delta_days=-1 ch_sub_query += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - intDiv(toHour(sessions.datetime),2)*2 AS hour, - COUNT(sessions.session_id) AS count + ch_query = f"""SELECT intDiv(toHour(sessions.datetime),2)*2 AS hour, + COUNT(sessions.session_id) AS count FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY hour @@ -1320,7 +1279,7 @@ def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(pages.visually_complete) AS avg + AVG(pages.visually_complete) AS value FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1334,9 +1293,9 @@ def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avg": avg, "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, - end_time=endTimestamp, density=density, - neutral={"avg": 0})} + return {"value": avg, "chart": __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, density=density, + neutral={"value": 0})} def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1353,7 +1312,7 @@ def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(d with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(DISTINCT pages.session_id) AS count + COUNT(DISTINCT pages.session_id) AS count FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} AND (pages.response_time)>(SELECT AVG(pages.response_time) @@ -1382,7 +1341,7 @@ def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1) with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(performance.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(performance.avg_used_js_heap_size) AS avg_used_js_heap_size + AVG(performance.avg_used_js_heap_size) AS value FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1396,11 +1355,11 @@ def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1) FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avgUsedJsHeapSize": avg, + return {"value": avg, "chart": helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, - neutral={"avg_used_js_heap_size": 0}))} + neutral={"value": 0}))} def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1413,7 +1372,7 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(performance.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(performance.avg_cpu) AS avg_cpu + AVG(performance.avg_cpu) AS value FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1427,11 +1386,11 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avgCpu": avg, + return {"value": avg, "chart": helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, - neutral={"avg_cpu": 0}))} + neutral={"value": 0}))} def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), @@ -1444,7 +1403,7 @@ def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(performance.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(performance.avg_fps) AS avg_fps + AVG(performance.avg_fps) AS value FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1458,11 +1417,11 @@ def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), FROM performance {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)};""" avg = ch.execute(query=ch_query, params=params)[0]["avg"] if len(rows) > 0 else 0 - return {"avgFps": avg, + return {"value": avg, "chart": helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, - neutral={"avg_fps": 0}))} + neutral={"value": 0}))} def __get_crashed_sessions_ids(project_id, startTimestamp, endTimestamp): @@ -1698,9 +1657,8 @@ def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - resources.url_host AS domain, - AVG(resources.duration) AS avg + ch_query = f"""SELECT resources.url_host AS domain, + AVG(resources.duration) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY resources.url_host @@ -1732,7 +1690,7 @@ def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1) WHERE {" AND ".join(ch_sub_query)} GROUP BY resources.url_host ORDER BY errors_count DESC - LIMIT 10;""" + LIMIT 5;""" rows = ch.execute(query=ch_query, params={"project_id": project_id, "startTimestamp": startTimestamp, @@ -1747,15 +1705,13 @@ def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=- ch_sub_query += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - b.user_browser AS browser, - b.count, - groupArray([bv.user_browser_version, toString(bv.count)]) AS versions + ch_query = f"""SELECT b.user_browser AS browser, + b.count, + groupArray([bv.user_browser_version, toString(bv.count)]) AS versions FROM ( - SELECT - sessions.user_browser, - COUNT(sessions.session_id) AS count + SELECT sessions.user_browser, + COUNT(sessions.session_id) AS count FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY sessions.user_browser @@ -1764,10 +1720,9 @@ def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=- ) AS b INNER JOIN ( - SELECT - sessions.user_browser, - sessions.user_browser_version, - COUNT(sessions.session_id) AS count + SELECT sessions.user_browser, + sessions.user_browser_version, + COUNT(sessions.session_id) AS count FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY @@ -1934,8 +1889,8 @@ def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_d "endTimestamp": endTimestamp, **__get_constraint_values(args)} with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(resources.session_id) AS total, - SUM(if(resources.type='fetch',1,0)) AS xhr + COUNT(resources.session_id) AS total, + SUM(if(resources.type='fetch',1,0)) AS xhr FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -1946,7 +1901,7 @@ def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_d density=density, neutral={"total": 0, "xhr": 0}) ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - AVG(pages.response_end) AS avg_response_end + AVG(pages.response_end) AS avg_response_end FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart_response_end)} GROUP BY timestamp @@ -1969,8 +1924,8 @@ def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(de with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(errors.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, - COUNT(DISTINCT errors.session_id) AS sessions_count, - COUNT(DISTINCT errors.error_id) AS errors_count + COUNT(DISTINCT errors.session_id) AS sessions_count, + COUNT(DISTINCT errors.error_id) AS errors_count FROM errors {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp @@ -2008,15 +1963,13 @@ def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(de ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT - toUnixTimestamp(toStartOfInterval(s.base_datetime, toIntervalSecond(%(step_size)s))) * 1000 AS timestamp, - AVG(NULLIF(s.count,0)) AS avg, - groupArray([toString(t.type), toString(t.xavg)]) AS types + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(s.base_datetime, toIntervalSecond(%(step_size)s))) * 1000 AS timestamp, + AVG(NULLIF(s.count,0)) AS avg, + groupArray([toString(t.type), toString(t.xavg)]) AS types FROM - ( SELECT - resources.session_id, - MIN(resources.datetime) AS base_datetime, - COUNT(resources.url) AS count + ( SELECT resources.session_id, + MIN(resources.datetime) AS base_datetime, + COUNT(resources.url) AS count FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY resources.session_id @@ -2137,3 +2090,539 @@ def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1) density=density, neutral={"first_party": 0, "third_party": 0})) + + +def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with ch_client.ClickHouseClient() as ch: + row = __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + results["chart"] = get_performance_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + ch_query = f"""\ + SELECT AVG(NULLIF(pages.load_event_end ,0)) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + row = ch.execute(query=ch_query, params=params)[0] + result = row + for k in result: + if result[k] is None: + result[k] = 0 + return result + + +def get_performance_avg_page_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, resources=None, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) + location_constraints = [] + meta_condition = __get_meta_constraint(args) + + location_constraints_vals = {} + + if resources and len(resources) > 0: + for r in resources: + if r["type"] == "LOCATION": + location_constraints.append(f"pages.url_path = %(val_{len(location_constraints)})s") + location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value'] + + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + + ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, + data=args) + ch_sub_query_chart += meta_condition + + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(pages.load_event_end ,0)) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + GROUP BY timestamp + ORDER BY timestamp;""" + + rows = ch.execute(query=ch_query, + params={**params, **location_constraints_vals, **__get_constraint_values(args)}) + pages = [{"timestamp": i["timestamp"], "value": i["value"]} for i in + __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, + density=density, neutral={"value": 0})] + + for s in pages: + for k in s: + if s[k] is None: + s[k] = 0 + return pages + + +def get_application_activity_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with ch_client.ClickHouseClient() as ch: + row = __get_application_activity_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + results["chart"] = get_performance_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_application_activity_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="resources", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + ch_sub_query.append("resources.type= %(type)s") + ch_query = f"""\ + SELECT AVG(NULLIF(resources.duration,0)) AS value + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + row = ch.execute(query=ch_query, + params={"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})[0] + result = row + for k in result: + if result[k] is None: + result[k] = 0 + return result + + +def get_performance_avg_image_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, resources=None, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) + img_constraints = [] + ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query_chart += meta_condition + + img_constraints_vals = {} + + if resources and len(resources) > 0: + for r in resources: + if r["type"] == "IMG": + img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s") + img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value'] + + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS value + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + AND resources.type = 'img' + {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} + GROUP BY timestamp + ORDER BY timestamp;""" + rows = ch.execute(query=ch_query, params={**params, **img_constraints_vals, **__get_constraint_values(args)}) + images = [{"timestamp": i["timestamp"], "value": i["value"]} for i in + __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, + density=density, neutral={"value": 0})] + + for s in images: + for k in s: + if s[k] is None: + s[k] = 0 + return images + + +def get_application_activity_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with ch_client.ClickHouseClient() as ch: + row = __get_application_activity_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + results["chart"] = get_performance_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_application_activity_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="resources", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + ch_sub_query.append("resources.type= %(type)s") + ch_query = f"""\ + SELECT AVG(NULLIF(resources.duration,0)) AS value + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + row = ch.execute(query=ch_query, + params={"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})[0] + result = row + for k in result: + if result[k] is None: + result[k] = 0 + return result + + +def get_performance_avg_request_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, resources=None, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) + request_constraints = [] + ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query_chart += meta_condition + + request_constraints_vals = {} + + if resources and len(resources) > 0: + for r in resources: + if r["type"] != "IMG" and r["type"] == "LOCATION": + request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s") + request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(resources.duration,0)) AS value + FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + AND resources.type = 'fetch' + {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} + GROUP BY timestamp + ORDER BY timestamp;""" + rows = ch.execute(query=ch_query, + params={**params, **request_constraints_vals, **__get_constraint_values(args)}) + requests = [{"timestamp": i["timestamp"], "value": i["value"]} for i in + __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, density=density, + neutral={"value": 0})] + + for s in requests: + for k in s: + if s[k] is None: + s[k] = 0 + return requests + + +def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with ch_client.ClickHouseClient() as ch: + results = {} + rows = __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + results["chart"] = __get_page_metrics_avg_dom_content_load_start_chart(ch, project_id, startTimestamp, + endTimestamp, **args) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + results["unit"] = schemas.TemplatePredefinedUnits.millisecond + return results + + +def __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + ch_query = f"""\ + SELECT COALESCE(AVG(NULLIF(pages.dom_content_loaded_event_end ,0)),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + params = {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + rows = ch.execute(query=ch_query, params=params) + return rows + + +def __get_page_metrics_avg_dom_content_load_start_chart(ch, project_id, startTimestamp, endTimestamp, density=19, + **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density) + ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query_chart += meta_condition + + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, + AVG(NULLIF(pages.dom_content_loaded_event_end,0)) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + GROUP BY timestamp + ORDER BY timestamp;""" + rows = ch.execute(query=ch_query, params={**params, **__get_constraint_values(args)}) + rows = [{"timestamp": i["timestamp"], "value": i["value"]} for i in + __complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, + density=density, neutral={"value": 0})] + + for s in rows: + for k in s: + if s[k] is None: + s[k] = 0 + return rows + + +def get_page_metrics_avg_first_contentful_pixel(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with ch_client.ClickHouseClient() as ch: + rows = __get_page_metrics_avg_first_contentful_pixel(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_page_metrics_avg_first_contentful_pixel(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_page_metrics_avg_first_contentful_pixel(ch, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + # changed dom_content_loaded_event_start to dom_content_loaded_event_end + ch_query = f"""\ + SELECT COALESCE(AVG(NULLIF(pages.first_contentful_paint,0)),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + params = {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + rows = ch.execute(query=ch_query, params=params) + return rows + + +def get_user_activity_avg_visited_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + results = {} + + with ch_client.ClickHouseClient() as ch: + rows = __get_user_activity_avg_visited_pages(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + for key in results: + if isnan(results[key]): + results[key] = 0 + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_user_activity_avg_visited_pages(ch, project_id, startTimestamp, endTimestamp, **args) + + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_user_activity_avg_visited_pages(cur, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="sessions", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + ch_query = f"""\ + SELECT COALESCE(CEIL(AVG(NULLIF(sessions.pages_count,0))),0) AS value + FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + rows = cur.execute(query=ch_query, params=params) + + return rows + + +def get_user_activity_avg_session_duration(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + results = {} + + with ch_client.ClickHouseClient() as ch: + rows = __get_user_activity_avg_session_duration(ch, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + for key in results: + if isnan(results[key]): + results[key] = 0 + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_user_activity_avg_session_duration(ch, project_id, startTimestamp, endTimestamp, **args) + + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"]) + return results + + +def __get_user_activity_avg_session_duration(cur, project_id, startTimestamp, endTimestamp, **args): + ch_sub_query = __get_basic_constraints(table_name="sessions", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + ch_query = f"""\ + SELECT COALESCE(AVG(NULLIF(sessions.duration,0)),0) AS value + FROM sessions {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + rows = cur.execute(query=ch_query, params=params) + + return rows + + +def get_top_metrics_avg_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COALESCE(AVG(pages.response_time),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.response_time) AND pages.response_time>0;""" + rows = ch.execute(query=ch_query, + params={"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)}) + return helper.dict_to_camel_case(rows[0]) + + +def get_top_metrics_count_requests(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COUNT(pages.session_id) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)};""" + rows = ch.execute(query=ch_query, + params={"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)}) + return helper.dict_to_camel_case(rows[0]) + + +def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COALESCE(AVG(pages.first_paint),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.first_paint) AND pages.first_paint>0;""" + rows = ch.execute(query=ch_query, + params={"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)}) + return helper.dict_to_camel_case(rows[0]) + + +def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, density=19, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density) + ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query_chart += meta_condition + + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + ch_sub_query_chart.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COALESCE(AVG(pages.dom_content_loaded_event_time),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.dom_content_loaded_event_time) AND pages.dom_content_loaded_event_time>0;""" + params = {"step_size": step_size, "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)} + rows = ch.execute(query=ch_query, params=params) + results = helper.dict_to_camel_case(rows[0]) + ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, + COALESCE(AVG(NULLIF(pages.dom_content_loaded_event_time ,0)),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query_chart)} + GROUP BY timestamp + ORDER BY timestamp;;""" + rows = ch.execute(query=ch_query, params=params) + results["chart"] = helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, + end_time=endTimestamp, + density=density, + neutral={"value": 0})) + return results + + +def get_top_metrics_avg_till_first_bit(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COALESCE(AVG(pages.ttfb),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.ttfb) AND pages.ttfb>0;""" + rows = ch.execute(query=ch_query, + params={"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)}) + return helper.dict_to_camel_case(rows[0]) + + +def get_top_metrics_avg_time_to_interactive(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + ch_sub_query = __get_basic_constraints(table_name="pages", data=args) + meta_condition = __get_meta_constraint(args) + ch_sub_query += meta_condition + + if value is not None: + ch_sub_query.append("pages.url_path = %(value)s") + with ch_client.ClickHouseClient() as ch: + ch_query = f"""SELECT COALESCE(AVG(pages.time_to_interactive),0) AS value + FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.time_to_interactive) AND pages.time_to_interactive >0;""" + rows = ch.execute(query=ch_query, + params={"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)}) + return helper.dict_to_camel_case(rows[0]) diff --git a/ee/api/chalicelib/core/insights.py b/ee/api/chalicelib/core/insights.py index 387029fd4..ff9d4dad4 100644 --- a/ee/api/chalicelib/core/insights.py +++ b/ee/api/chalicelib/core/insights.py @@ -1,9 +1,9 @@ -from chalicelib.core import sessions_metas -from chalicelib.utils import helper, dev -from chalicelib.utils import ch_client -from chalicelib.utils.TimeUTC import TimeUTC -from chalicelib.core.dashboard import __get_constraint_values, __complete_missing_steps +import schemas from chalicelib.core.dashboard import __get_basic_constraints, __get_meta_constraint +from chalicelib.core.dashboard import __get_constraint_values, __complete_missing_steps +from chalicelib.utils import ch_client +from chalicelib.utils import helper, dev +from chalicelib.utils.TimeUTC import TimeUTC def __transform_journey(rows): @@ -42,7 +42,7 @@ def journey(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp= elif f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append(f"sessions_metadata.project_id = %(project_id)s") meta_condition.append(f"sessions_metadata.datetime >= toDateTime(%(startTimestamp)s / 1000)") @@ -303,7 +303,7 @@ def feature_retention(project_id, startTimestamp=TimeUTC.now(delta_days=-70), en elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.user_id IS NOT NULL") meta_condition.append("not empty(sessions_metadata.user_id)") @@ -404,7 +404,7 @@ def feature_acquisition(project_id, startTimestamp=TimeUTC.now(delta_days=-70), elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.user_id IS NOT NULL") meta_condition.append("not empty(sessions_metadata.user_id)") @@ -512,7 +512,7 @@ def feature_popularity_frequency(project_id, startTimestamp=TimeUTC.now(delta_da if f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.user_id IS NOT NULL") meta_condition.append("not empty(sessions_metadata.user_id)") @@ -586,7 +586,7 @@ def feature_adoption(project_id, startTimestamp=TimeUTC.now(delta_days=-70), end elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.user_id IS NOT NULL") meta_condition.append("not empty(sessions_metadata.user_id)") @@ -672,7 +672,7 @@ def feature_adoption_top_users(project_id, startTimestamp=TimeUTC.now(delta_days elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("user_id IS NOT NULL") meta_condition.append("not empty(sessions_metadata.user_id)") @@ -742,7 +742,7 @@ def feature_adoption_daily_usage(project_id, startTimestamp=TimeUTC.now(delta_da elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.project_id = %(project_id)s") meta_condition.append("sessions_metadata.datetime >= toDateTime(%(startTimestamp)s/1000)") @@ -807,7 +807,7 @@ def feature_intensity(project_id, startTimestamp=TimeUTC.now(delta_days=-70), en if f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.project_id = %(project_id)s") meta_condition.append("sessions_metadata.datetime >= toDateTime(%(startTimestamp)s/1000)") @@ -847,7 +847,7 @@ def users_active(project_id, startTimestamp=TimeUTC.now(delta_days=-70), endTime for f in filters: if f["type"] == "PERIOD" and f["value"] in ["DAY", "WEEK"]: period = f["value"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") extra_values["user_id"] = f["value"] period_function = PERIOD_TO_FUNCTION[period] @@ -940,7 +940,7 @@ def users_slipping(project_id, startTimestamp=TimeUTC.now(delta_days=-70), endTi elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: meta_condition.append(f"sessions_metadata.user_id = %(user_id)s") meta_condition.append("sessions_metadata.project_id = %(project_id)s") meta_condition.append("sessions_metadata.datetime >= toDateTime(%(startTimestamp)s/1000)") @@ -1044,4 +1044,4 @@ def search(text, feature_type, project_id, platform=None): rows = ch.execute(ch_query, params) else: return [] - return [helper.dict_to_camel_case(row) for row in rows] \ No newline at end of file + return [helper.dict_to_camel_case(row) for row in rows] diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index 75a3a31d0..0f2b62cc9 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -82,22 +82,22 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st rows = cur.fetchall() if recording_state: project_ids = [f'({r["project_id"]})' for r in rows] - query = f"""SELECT projects.project_id, COALESCE(MAX(start_ts), 0) AS last - FROM (VALUES {",".join(project_ids)}) AS projects(project_id) - LEFT JOIN sessions USING (project_id) - GROUP BY project_id;""" - cur.execute( - query=query - ) + query = cur.mogrify(f"""SELECT projects.project_id, COALESCE(MAX(start_ts), 0) AS last + FROM (VALUES {",".join(project_ids)}) AS projects(project_id) + LEFT JOIN sessions USING (project_id) + WHERE sessions.start_ts >= %(startDate)s AND sessions.start_ts <= %(endDate)s + GROUP BY project_id;""", + {"startDate": TimeUTC.now(delta_days=-3), "endDate": TimeUTC.now(delta_days=1)}) + + cur.execute(query=query) status = cur.fetchall() for r in rows: + r["status"] = "red" for s in status: if s["project_id"] == r["project_id"]: - if s["last"] < TimeUTC.now(-2): - r["status"] = "red" - elif s["last"] < TimeUTC.now(-1): + if TimeUTC.now(-2) <= s["last"] < TimeUTC.now(-1): r["status"] = "yellow" - else: + elif s["last"] >= TimeUTC.now(-1): r["status"] = "green" break @@ -257,7 +257,8 @@ def get_project_key(project_id): where project_id =%(project_id)s AND deleted_at ISNULL;""", {"project_id": project_id}) ) - return cur.fetchone()["project_key"] + project = cur.fetchone() + return project["project_key"] if project is not None else None def get_capture_status(project_id): @@ -324,7 +325,7 @@ def is_authorized_batch(project_ids, tenant_id): query = cur.mogrify("""\ SELECT project_id FROM public.projects - where tenant_id =%(tenant_id)s + WHERE tenant_id =%(tenant_id)s AND project_id IN %(project_ids)s AND deleted_at IS NULL;""", {"tenant_id": tenant_id, "project_ids": tuple(project_ids)}) @@ -334,3 +335,13 @@ def is_authorized_batch(project_ids, tenant_id): ) rows = cur.fetchall() return [r["project_id"] for r in rows] + + +def get_projects_ids(tenant_id): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify("""SELECT s.project_id + FROM public.projects AS s + WHERE tenant_id =%(tenant_id)s AND s.deleted_at IS NULL + ORDER BY s.project_id;""", {"tenant_id": tenant_id})) + rows = cur.fetchall() + return [r["project_id"] for r in rows] diff --git a/ee/api/chalicelib/core/resources.py b/ee/api/chalicelib/core/resources.py index 332d3709a..4e4f1c4e8 100644 --- a/ee/api/chalicelib/core/resources.py +++ b/ee/api/chalicelib/core/resources.py @@ -3,14 +3,14 @@ from chalicelib.utils import ch_client from chalicelib.utils.TimeUTC import TimeUTC -def get_by_session_id(session_id): +def get_by_session_id(session_id, project_id): with ch_client.ClickHouseClient() as ch: ch_query = """\ SELECT datetime,url,type,duration,ttfb,header_size,encoded_body_size,decoded_body_size,success,coalesce(status,if(success, 200, status)) AS status FROM resources - WHERE session_id = toUInt64(%(session_id)s);""" - params = {"session_id": session_id} + WHERE session_id = toUInt64(%(session_id)s) AND project_id=%(project_id)s;""" + params = {"session_id": session_id, "project_id": project_id} rows = ch.execute(query=ch_query, params=params) results = [] for r in rows: diff --git a/ee/api/chalicelib/core/telemetry.py b/ee/api/chalicelib/core/telemetry.py index d9843e37d..e05df7fdc 100644 --- a/ee/api/chalicelib/core/telemetry.py +++ b/ee/api/chalicelib/core/telemetry.py @@ -53,7 +53,7 @@ def compute(): RETURNING *,(SELECT email FROM users_ee WHERE role = 'owner' AND users_ee.tenant_id = tenants.tenant_id LIMIT 1);""" ) data = cur.fetchall() - requests.post('https://parrot.asayer.io/os/telemetry', + requests.post('https://api.openreplay.com/os/telemetry', json={"stats": [process_data(d, edition='ee') for d in data]}) @@ -65,4 +65,4 @@ def new_client(tenant_id): FROM public.tenants WHERE tenant_id=%(tenant_id)s;""", {"tenant_id": tenant_id})) data = cur.fetchone() - requests.post('https://parrot.asayer.io/os/signup', json=process_data(data, edition='ee')) \ No newline at end of file + requests.post('https://api.openreplay.com/os/signup', json=process_data(data, edition='ee')) \ No newline at end of file diff --git a/ee/api/chalicelib/core/users.py b/ee/api/chalicelib/core/users.py index 9ca77c1ea..b70f6a269 100644 --- a/ee/api/chalicelib/core/users.py +++ b/ee/api/chalicelib/core/users.py @@ -632,7 +632,6 @@ def change_jwt_iat(user_id): return cur.fetchone().get("jwt_iat") -@dev.timed def authenticate(email, password, for_change_password=False, for_plugin=False): with pg_client.PostgresClient() as cur: query = cur.mogrify( diff --git a/ee/api/chalicelib/utils/ch_client.py b/ee/api/chalicelib/utils/ch_client.py index babdd669a..aa45699f7 100644 --- a/ee/api/chalicelib/utils/ch_client.py +++ b/ee/api/chalicelib/utils/ch_client.py @@ -25,5 +25,8 @@ class ClickHouseClient: def client(self): return self.__client + def format(self, query, params): + return self.__client.substitute_params(query, params) + def __exit__(self, *args): pass diff --git a/ee/api/entrypoint.sh b/ee/api/entrypoint.sh deleted file mode 100755 index a092737be..000000000 --- a/ee/api/entrypoint.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -uvicorn app:app --host 0.0.0.0 --reload diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 84a372567..5909d31c1 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -4,11 +4,11 @@ boto3==1.16.1 pyjwt==1.7.1 psycopg2-binary==2.8.6 elasticsearch==7.9.1 -jira==2.0.0 +jira==3.1.1 clickhouse-driver==0.2.2 python3-saml==1.12.0 -fastapi==0.74.1 +fastapi==0.75.0 python-multipart==0.0.5 uvicorn[standard]==0.17.5 python-decouple==3.6 diff --git a/ee/api/routers/base.py b/ee/api/routers/base.py deleted file mode 100644 index 5c665b2d1..000000000 --- a/ee/api/routers/base.py +++ /dev/null @@ -1,14 +0,0 @@ -from fastapi import APIRouter, Depends - -from auth.auth_apikey import APIKeyAuth -from auth.auth_jwt import JWTAuth -from auth.auth_project import ProjectAuthorizer -from or_dependencies import ORRoute - - -def get_routers() -> (APIRouter, APIRouter, APIRouter): - public_app = APIRouter(route_class=ORRoute) - app = APIRouter(dependencies=[Depends(JWTAuth()), Depends(ProjectAuthorizer("projectId"))], route_class=ORRoute) - app_apikey = APIRouter(dependencies=[Depends(APIKeyAuth()), Depends(ProjectAuthorizer("projectKey"))], - route_class=ORRoute) - return public_app, app, app_apikey diff --git a/ee/api/routers/app/v1_api_ee.py b/ee/api/routers/subs/v1_api_ee.py similarity index 100% rename from ee/api/routers/app/v1_api_ee.py rename to ee/api/routers/subs/v1_api_ee.py diff --git a/ee/backend/pkg/kafka/consumer.go b/ee/backend/pkg/kafka/consumer.go index ca0544923..cb3714316 100644 --- a/ee/backend/pkg/kafka/consumer.go +++ b/ee/backend/pkg/kafka/consumer.go @@ -25,7 +25,12 @@ type Consumer struct { lastKafkaEventTs int64 } -func NewConsumer(group string, topics []string, messageHandler types.MessageHandler) *Consumer { +func NewConsumer( + group string, + topics []string, + messageHandler types.MessageHandler, + autoCommit bool, +) *Consumer { protocol := "plaintext" if env.Bool("KAFKA_USE_SSL") { protocol = "ssl" @@ -53,18 +58,19 @@ func NewConsumer(group string, topics []string, messageHandler types.MessageHand log.Fatalln(err) } + var commitTicker *time.Ticker + if autoCommit { + commitTicker = time.NewTicker(2 * time.Minute) + } + return &Consumer{ c: c, messageHandler: messageHandler, - commitTicker: time.NewTicker(2 * time.Minute), + commitTicker: commitTicker, pollTimeout: 200, } } -func (consumer *Consumer) DisableAutoCommit() { - consumer.commitTicker.Stop() -} - func (consumer *Consumer) Commit() error { consumer.c.Commit() // TODO: return error if it is not "No offset stored" return nil @@ -128,10 +134,12 @@ func (consumer *Consumer) ConsumeNext() error { return nil } - select { - case <-consumer.commitTicker.C: - consumer.Commit() - default: + if consumer.commitTicker != nil { + select { + case <-consumer.commitTicker.C: + consumer.Commit() + default: + } } switch e := ev.(type) { @@ -139,7 +147,7 @@ func (consumer *Consumer) ConsumeNext() error { if e.TopicPartition.Error != nil { return errors.Wrap(e.TopicPartition.Error, "Consumer Partition Error") } - ts := e.Timestamp.UnixNano() / 1e6 + ts := e.Timestamp.UnixMilli() consumer.messageHandler(decodeKey(e.Key), e.Value, &types.Meta{ Topic: *(e.TopicPartition.Topic), ID: uint64(e.TopicPartition.Offset), diff --git a/ee/backend/pkg/license/check.go b/ee/backend/pkg/license/check.go index 6b33a625e..771558946 100644 --- a/ee/backend/pkg/license/check.go +++ b/ee/backend/pkg/license/check.go @@ -33,7 +33,7 @@ func CheckLicense() { log.Fatal("Can not form a license check request.") } - resp, err := http.Post("https://parrot.asayer.io/os/license", "application/json", bytes.NewReader(requestBody)) + resp, err := http.Post("https://api.openreplay.com/os/license", "application/json", bytes.NewReader(requestBody)) if err != nil { log.Fatalf("Error while checking license. %v", err) } diff --git a/ee/backend/pkg/queue/import.go b/ee/backend/pkg/queue/import.go index abff07e9a..e95eb11e5 100644 --- a/ee/backend/pkg/queue/import.go +++ b/ee/backend/pkg/queue/import.go @@ -2,17 +2,16 @@ package queue import ( "openreplay/backend/pkg/kafka" - "openreplay/backend/pkg/queue/types" "openreplay/backend/pkg/license" + "openreplay/backend/pkg/queue/types" ) -func NewConsumer(group string, topics []string, handler types.MessageHandler) types.Consumer { +func NewConsumer(group string, topics []string, handler types.MessageHandler, autoCommit bool) types.Consumer { license.CheckLicense() - return kafka.NewConsumer(group, topics, handler) + return kafka.NewConsumer(group, topics, handler, autoCommit) } func NewProducer() types.Producer { license.CheckLicense() return kafka.NewProducer() } - diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql new file mode 100644 index 000000000..ba00ea335 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -0,0 +1,321 @@ +BEGIN; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.5.5-ee' +$$ LANGUAGE sql IMMUTABLE; + + +CREATE TABLE IF NOT EXISTS dashboards +( + dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT TRUE, + is_pinned boolean NOT NULL DEFAULT FALSE, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + deleted_at timestamp NULL DEFAULT NULL +); + + +ALTER TABLE IF EXISTS metrics + DROP CONSTRAINT IF EXISTS null_project_id_for_template_only, + DROP CONSTRAINT IF EXISTS unique_key; + +ALTER TABLE IF EXISTS metrics + ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', + ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS default_config jsonb NOT NULL DEFAULT '{ + "col": 2, + "row": 2, + "position": 0 + }'::jsonb, + ALTER COLUMN project_id DROP NOT NULL, + ADD CONSTRAINT null_project_id_for_template_only + CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), + ADD CONSTRAINT unique_key UNIQUE (predefined_key); + + + +CREATE TABLE IF NOT EXISTS dashboard_widgets +( + widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, + metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + config jsonb NOT NULL DEFAULT '{}'::jsonb +); + +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS host text NULL, + ADD COLUMN IF NOT EXISTS base_path text NULL, + ADD COLUMN IF NOT EXISTS query text NULL; + +ALTER TABLE events.pages + ADD COLUMN IF NOT EXISTS query text NULL; +COMMIT; + +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'barChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarLineChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'overview'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'map'; +ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'predefined'; + + +INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, + view_type) +VALUES ('Captured sessions', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 +}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('Request Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('Page Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('Image Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('DOM Content Load Start', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('First Meaningful paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('No. of Visited Pages', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('Session Duration', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('DOM Build Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('Pages Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('First Paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('DOM Content Loaded', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('Time Till First byte', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), + ('Time To Interactive', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('Captured requests', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'count_requests', 'predefined', 'overview'), + ('Time To Render', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('Memory Consumption', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('CPU Load', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('Frame rate', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_fps', 'predefined', 'overview'), + + ('Sessions Affected by JS Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('Top Domains with 4xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('Top Domains with 5xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('Errors per Domain', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('Fetch Calls with Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'calls_errors', 'predefined', 'table'), + ('Errors by Type', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('Errors by Origin', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + + ('Speed Index by Location', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'speed_location', 'predefined', 'map'), + ('Slowest Domains', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('Sessions per Browser', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('Time To Render', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('Sessions Impacted by Slow Pages', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('Memory Consumption', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('CPU Load', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('Frame Rate', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'fps', 'predefined', 'areaChart'), + ('Crashes', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('Resources Loaded vs Visually Complete', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('DOM Build Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('Pages Response Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('Pages Response Time Distribution', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), + + ('Missing Resources', 'resources', '{ + "col": 4, + "row": 2, + "position": 0 + }', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Resources Fetch Time', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_loading_time', 'predefined', 'table'), + ('Resource Loaded vs Response End', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), + ('Breakdown of Loaded Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') +ON CONFLICT (predefined_key) DO UPDATE + SET name=excluded.name, + category=excluded.category, + default_config=excluded.default_config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public, + metric_type=excluded.metric_type, + view_type=excluded.view_type; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 9adab50e0..bc8d65f93 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -7,7 +7,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.5.4-ee' +SELECT 'v1.5.5-ee' $$ LANGUAGE sql IMMUTABLE; @@ -106,6 +106,8 @@ $$ ('assigned_sessions'), ('autocomplete'), ('basic_authentication'), + ('dashboards'), + ('dashboard_widgets'), ('errors'), ('funnels'), ('integrations'), @@ -786,23 +788,37 @@ $$ CREATE INDEX IF NOT EXISTS traces_user_id_idx ON traces (user_id); CREATE INDEX IF NOT EXISTS traces_tenant_id_idx ON traces (tenant_id); - CREATE TYPE metric_type AS ENUM ('timeseries','table'); - CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart'); + CREATE TYPE metric_type AS ENUM ('timeseries','table', 'predefined'); + CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart','barChart','stackedBarChart','stackedBarLineChart','overview','map'); CREATE TABLE IF NOT EXISTS metrics ( - metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - user_id integer REFERENCES users (user_id) ON DELETE SET NULL, - name text NOT NULL, - is_public boolean NOT NULL DEFAULT FALSE, - active boolean NOT NULL DEFAULT TRUE, - created_at timestamp DEFAULT timezone('utc'::text, now()) not null, - deleted_at timestamp, - metric_type metric_type NOT NULL DEFAULT 'timeseries', - view_type metric_view_type NOT NULL DEFAULT 'lineChart', - metric_of text NOT NULL DEFAULT 'sessionCount', - metric_value text[] NOT NULL DEFAULT '{}'::text[], - metric_format text + metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT FALSE, + active boolean NOT NULL DEFAULT TRUE, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp, + edited_at timestamp, + metric_type metric_type NOT NULL DEFAULT 'timeseries', + view_type metric_view_type NOT NULL DEFAULT 'lineChart', + metric_of text NOT NULL DEFAULT 'sessionCount', + metric_value text[] NOT NULL DEFAULT '{}'::text[], + metric_format text, + category text NULL DEFAULT 'custom', + is_pinned boolean NOT NULL DEFAULT FALSE, + is_predefined boolean NOT NULL DEFAULT FALSE, + is_template boolean NOT NULL DEFAULT FALSE, + predefined_key text NULL DEFAULT NULL, + default_config jsonb NOT NULL DEFAULT '{ + "col": 2, + "row": 2, + "position": 0 + }'::jsonb, + CONSTRAINT null_project_id_for_template_only + CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), + CONSTRAINT unique_key UNIQUE (predefined_key) ); CREATE INDEX IF NOT EXISTS metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); CREATE TABLE IF NOT EXISTS metric_series @@ -817,6 +833,29 @@ $$ ); CREATE INDEX IF NOT EXISTS metric_series_metric_id_idx ON public.metric_series (metric_id); + + CREATE TABLE dashboards + ( + dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT TRUE, + is_pinned boolean NOT NULL DEFAULT FALSE, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + deleted_at timestamp NULL DEFAULT NULL + ); + + CREATE TABLE dashboard_widgets + ( + widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, + metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + config jsonb NOT NULL DEFAULT '{}'::jsonb + ); + CREATE TABLE IF NOT EXISTS searches ( search_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, @@ -900,6 +939,7 @@ $$ host text NOT NULL, path text NOT NULL, base_path text NOT NULL, + query text NULL, referrer text DEFAULT NULL, base_referrer text DEFAULT NULL, dom_building_time integer DEFAULT NULL, @@ -948,14 +988,20 @@ $$ CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR - first_paint_time > 0 OR - dom_content_loaded_time > 0 OR + first_paint_time > + 0 OR + dom_content_loaded_time > + 0 OR ttfb > 0 OR - time_to_interactive > 0; + time_to_interactive > + 0; CREATE INDEX IF NOT EXISTS pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); CREATE INDEX IF NOT EXISTS pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; + CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; + CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; + CREATE TABLE IF NOT EXISTS events.clicks ( @@ -1191,6 +1237,9 @@ $$ response_body text NULL, status_code smallint NULL, method http_method NULL, + host text NULL, + base_path text NULL, + query text NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); CREATE INDEX IF NOT EXISTS requests_url_idx ON events_common.requests (url); @@ -1212,6 +1261,12 @@ $$ CREATE INDEX IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; CREATE INDEX IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; CREATE INDEX IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; END IF; @@ -1219,5 +1274,244 @@ $$ $$ LANGUAGE plpgsql; +INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, + view_type) +VALUES ('Captured sessions', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 +}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('Request Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('Page Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('Image Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('DOM Content Load Start', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('First Meaningful paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('No. of Visited Pages', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('Session Duration', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('DOM Build Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('Pages Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('First Paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('DOM Content Loaded', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('Time Till First byte', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), + ('Time To Interactive', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('Captured requests', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'count_requests', 'predefined', 'overview'), + ('Time To Render', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('Memory Consumption', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('CPU Load', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('Frame rate', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_fps', 'predefined', 'overview'), + + ('Sessions Affected by JS Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('Top Domains with 4xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('Top Domains with 5xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('Errors per Domain', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('Fetch Calls with Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'calls_errors', 'predefined', 'table'), + ('Errors by Type', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('Errors by Origin', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + + ('Speed Index by Location', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'speed_location', 'predefined', 'map'), + ('Slowest Domains', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('Sessions per Browser', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('Time To Render', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('Sessions Impacted by Slow Pages', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('Memory Consumption', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('CPU Load', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('Frame Rate', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'fps', 'predefined', 'areaChart'), + ('Crashes', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('Resources Loaded vs Visually Complete', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('DOM Build Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('Pages Response Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('Pages Response Time Distribution', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), + + ('Missing Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{ + "col": 4, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Resources Fetch Time', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_loading_time', 'predefined', 'table'), + ('Resource Loaded vs Response End', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), + ('Breakdown of Loaded Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') +ON CONFLICT (predefined_key) DO UPDATE + SET name=excluded.name, + category=excluded.category, + default_config=excluded.default_config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public, + metric_type=excluded.metric_type, + view_type=excluded.view_type; COMMIT; \ No newline at end of file diff --git a/ee/utilities/.gitignore b/ee/utilities/.gitignore index fc05191e0..0aaf625c9 100644 --- a/ee/utilities/.gitignore +++ b/ee/utilities/.gitignore @@ -10,4 +10,6 @@ build.sh servers/peerjs-server.js servers/sourcemaps-handler.js servers/sourcemaps-server.js -#servers/websocket.js \ No newline at end of file +#servers/websocket.js +/utils +/Dockerfile diff --git a/ee/utilities/package-lock.json b/ee/utilities/package-lock.json index e7974f3f2..66fdb346f 100644 --- a/ee/utilities/package-lock.json +++ b/ee/utilities/package-lock.json @@ -11,12 +11,9 @@ "dependencies": { "@maxmind/geoip2-node": "^3.4.0", "@socket.io/redis-adapter": "^7.1.0", - "aws-sdk": "^2.992.0", "express": "^4.17.1", - "peer": "^0.6.1", "redis": "^4.0.3", "socket.io": "^4.4.1", - "source-map": "^0.7.3", "ua-parser-js": "^1.0.2", "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.6.0" } @@ -108,28 +105,11 @@ "node": ">=10.0.0" } }, - "node_modules/@types/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, "node_modules/@types/component-emitter": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.11.tgz", "integrity": "sha512-SRXjM+tfsSlA9VuG8hGO2nft2p8zjXCK1VcC6N4NXbBbYbSia9kzCChYQajIjzIqOOOuh5Ock6MmV2oux4jDZQ==" }, - "node_modules/@types/connect": { - "version": "3.4.35", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", - "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/cookie": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz", @@ -140,64 +120,11 @@ "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" }, - "node_modules/@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.18", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.17.28", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", - "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "node_modules/@types/mime": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", - "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" - }, "node_modules/@types/node": { "version": "17.0.21", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" }, - "node_modules/@types/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" - }, - "node_modules/@types/range-parser": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "node_modules/@types/serve-static": { - "version": "1.13.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", - "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", - "dependencies": { - "@types/mime": "^1", - "@types/node": "*" - } - }, - "node_modules/@types/ws": { - "version": "7.4.7", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", - "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -210,28 +137,6 @@ "node": ">= 0.6" } }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -245,44 +150,6 @@ "node": ">=0.8" } }, - "node_modules/aws-sdk": { - "version": "2.1087.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", - "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", - "dependencies": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "uuid": "3.3.2", - "xml2js": "0.4.19" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/base64id": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz", @@ -324,16 +191,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, - "node_modules/buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "dependencies": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -370,16 +227,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, "node_modules/cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", @@ -388,22 +235,6 @@ "node": ">=0.10.0" } }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, "node_modules/component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", @@ -474,14 +305,6 @@ } } }, - "node_modules/decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -500,11 +323,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -577,14 +395,6 @@ "node": ">= 0.6" } }, - "node_modules/events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/express": { "version": "4.17.3", "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", @@ -676,18 +486,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, - "node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -712,14 +510,6 @@ "node": ">= 4" } }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, "node_modules/http-errors": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", @@ -746,11 +536,6 @@ "node": ">=0.10.0" } }, - "node_modules/ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -773,27 +558,6 @@ "node": ">= 0.10" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "node_modules/jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", - "engines": { - "node": ">= 0.6.0" - } - }, "node_modules/json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", @@ -813,17 +577,6 @@ "verror": "1.10.0" } }, - "node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", @@ -950,39 +703,6 @@ "node": ">= 0.8" } }, - "node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "engines": { - "node": ">=6" - } - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -991,50 +711,11 @@ "node": ">= 0.8" } }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "engines": { - "node": ">=8" - } - }, "node_modules/path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "node_modules/peer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", - "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", - "dependencies": { - "@types/cors": "^2.8.6", - "@types/express": "^4.17.3", - "@types/ws": "^7.2.3", - "body-parser": "^1.19.0", - "cors": "^2.8.5", - "express": "^4.17.1", - "uuid": "^3.4.0", - "ws": "^7.2.3", - "yargs": "^15.3.1" - }, - "bin": { - "peerjs": "bin/peerjs" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/peer/node_modules/uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -1047,11 +728,6 @@ "node": ">= 0.10" } }, - "node_modules/punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" - }, "node_modules/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", @@ -1063,15 +739,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", - "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -1137,19 +804,6 @@ "node": ">=4" } }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1174,11 +828,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "node_modules/sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" - }, "node_modules/send": { "version": "0.17.2", "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", @@ -1234,11 +883,6 @@ "node": ">= 0.8.0" } }, - "node_modules/set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -1278,14 +922,6 @@ "node": ">=10.0.0" } }, - "node_modules/source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", - "engines": { - "node": ">= 8" - } - }, "node_modules/statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -1294,30 +930,6 @@ "node": ">= 0.6" } }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/tiny-lru": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-7.0.6.tgz", @@ -1388,15 +1000,6 @@ "node": ">= 0.8" } }, - "node_modules/url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", - "dependencies": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -1405,15 +1008,6 @@ "node": ">= 0.4.0" } }, - "node_modules/uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, "node_modules/uWebSockets.js": { "version": "20.6.0", "resolved": "git+ssh://git@github.com/uNetworking/uWebSockets.js.git#a58e810e47a23696410f6073c8c905dc38f75da5" @@ -1439,111 +1033,10 @@ "extsprintf": "^1.2.0" } }, - "node_modules/which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" - }, - "node_modules/wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/ws": { - "version": "7.5.7", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", - "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", - "engines": { - "node": ">=8.3.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/xml2js": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~9.0.1" - } - }, - "node_modules/xmlbuilder": { - "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/y18n": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" - }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "node_modules/yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", - "dependencies": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "dependencies": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/yargs-parser/node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "engines": { - "node": ">=6" - } } }, "dependencies": { @@ -1615,28 +1108,11 @@ "uid2": "0.0.3" } }, - "@types/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", - "requires": { - "@types/connect": "*", - "@types/node": "*" - } - }, "@types/component-emitter": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.11.tgz", "integrity": "sha512-SRXjM+tfsSlA9VuG8hGO2nft2p8zjXCK1VcC6N4NXbBbYbSia9kzCChYQajIjzIqOOOuh5Ock6MmV2oux4jDZQ==" }, - "@types/connect": { - "version": "3.4.35", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", - "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", - "requires": { - "@types/node": "*" - } - }, "@types/cookie": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz", @@ -1647,64 +1123,11 @@ "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" }, - "@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", - "requires": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.18", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "@types/express-serve-static-core": { - "version": "4.17.28", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", - "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", - "requires": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "@types/mime": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", - "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" - }, "@types/node": { "version": "17.0.21", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" }, - "@types/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" - }, - "@types/range-parser": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "@types/serve-static": { - "version": "1.13.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", - "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", - "requires": { - "@types/mime": "^1", - "@types/node": "*" - } - }, - "@types/ws": { - "version": "7.4.7", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", - "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", - "requires": { - "@types/node": "*" - } - }, "accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -1714,19 +1137,6 @@ "negotiator": "0.6.3" } }, - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -1737,27 +1147,6 @@ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, - "aws-sdk": { - "version": "2.1087.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", - "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", - "requires": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "uuid": "3.3.2", - "xml2js": "0.4.19" - } - }, - "base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" - }, "base64id": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz", @@ -1795,16 +1184,6 @@ } } }, - "buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, "bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -1826,34 +1205,11 @@ "type-fest": "^1.2.1" } }, - "cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, "cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, "component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", @@ -1904,11 +1260,6 @@ "ms": "2.1.2" } }, - "decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" - }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -1924,11 +1275,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -1977,11 +1323,6 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, - "events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" - }, "express": { "version": "4.17.3", "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", @@ -2068,15 +1409,6 @@ } } }, - "find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -2092,11 +1424,6 @@ "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", "integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==" }, - "get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" - }, "http-errors": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", @@ -2117,11 +1444,6 @@ "safer-buffer": ">= 2.1.2 < 3" } }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -2141,21 +1463,6 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==" - }, "json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", @@ -2172,14 +1479,6 @@ "verror": "1.10.0" } }, - "locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "requires": { - "p-locate": "^4.1.0" - } - }, "lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", @@ -2265,65 +1564,16 @@ "ee-first": "1.1.1" } }, - "p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "requires": { - "p-limit": "^2.2.0" - } - }, - "p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" - }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, - "path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" - }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "peer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", - "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", - "requires": { - "@types/cors": "^2.8.6", - "@types/express": "^4.17.3", - "@types/ws": "^7.2.3", - "body-parser": "^1.19.0", - "cors": "^2.8.5", - "express": "^4.17.1", - "uuid": "^3.4.0", - "ws": "^7.2.3", - "yargs": "^15.3.1" - }, - "dependencies": { - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - } - } - }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -2333,21 +1583,11 @@ "ipaddr.js": "1.9.1" } }, - "punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" - }, "qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==" }, - "querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" - }, "quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -2395,16 +1635,6 @@ "redis-errors": "^1.0.0" } }, - "require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" - }, - "require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2415,11 +1645,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" - }, "send": { "version": "0.17.2", "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", @@ -2473,11 +1698,6 @@ "send": "0.17.2" } }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, "setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -2511,34 +1731,11 @@ "debug": "~4.3.1" } }, - "source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==" - }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - }, "tiny-lru": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-7.0.6.tgz", @@ -2578,25 +1775,11 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, - "url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", - "requires": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" - }, "uWebSockets.js": { "version": "git+ssh://git@github.com/uNetworking/uWebSockets.js.git#a58e810e47a23696410f6073c8c905dc38f75da5", "from": "uWebSockets.js@github:uNetworking/uWebSockets.js#v20.6.0" @@ -2616,84 +1799,10 @@ "extsprintf": "^1.2.0" } }, - "which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" - }, - "wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, - "ws": { - "version": "7.5.7", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", - "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", - "requires": {} - }, - "xml2js": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", - "requires": { - "sax": ">=0.6.0", - "xmlbuilder": "~9.0.1" - } - }, - "xmlbuilder": { - "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" - }, - "y18n": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" - }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", - "requires": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - } - }, - "yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "dependencies": { - "camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" - } - } } } } diff --git a/ee/utilities/package.json b/ee/utilities/package.json index 87e1d1596..99c2666da 100644 --- a/ee/utilities/package.json +++ b/ee/utilities/package.json @@ -1,5 +1,5 @@ { - "name": "utilities_server", + "name": "utilities-server", "version": "1.0.0", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", @@ -12,7 +12,7 @@ "url": "git+https://github.com/openreplay/openreplay.git" }, "author": "KRAIEM Taha Yassine ", - "license": "MIT", + "license": "Elastic License 2.0 (ELv2)", "bugs": { "url": "https://github.com/openreplay/openreplay/issues" }, @@ -20,12 +20,9 @@ "dependencies": { "@maxmind/geoip2-node": "^3.4.0", "@socket.io/redis-adapter": "^7.1.0", - "aws-sdk": "^2.992.0", "express": "^4.17.1", - "peer": "^0.6.1", "redis": "^4.0.3", "socket.io": "^4.4.1", - "source-map": "^0.7.3", "ua-parser-js": "^1.0.2", "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.6.0" } diff --git a/ee/utilities/server.js b/ee/utilities/server.js index d049faa19..2c70067f1 100644 --- a/ee/utilities/server.js +++ b/ee/utilities/server.js @@ -1,8 +1,7 @@ -var sourcemapsReaderServer = require('./servers/sourcemaps-server'); -var {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); -var express = require('express'); -const {ExpressPeerServer} = require('peer'); -var socket; +const dumps = require('./utils/HeapSnapshot'); +const {request_logger} = require('./utils/helper'); +const express = require('express'); +let socket; if (process.env.redis === "true") { console.log("Using Redis"); socket = require("./servers/websocket-cluster"); @@ -11,62 +10,29 @@ if (process.env.redis === "true") { } const HOST = '0.0.0.0'; -const PORT = 9000; - -var app = express(); +const PORT = 9001; let debug = process.env.debug === "1" || false; -const request_logger = (identity) => { - return (req, res, next) => { - debug && console.log(identity, new Date().toTimeString(), 'REQUEST', req.method, req.originalUrl); - res.on('finish', function () { - if (this.statusCode !== 200 || debug) { - console.log(new Date().toTimeString(), 'RESPONSE', req.method, req.originalUrl, this.statusCode); - } - }) - - next(); - } -}; -app.use(request_logger("[app]")); - - -app.use('/sourcemaps', sourcemapsReaderServer); -app.use('/assist', peerRouter); - -const server = app.listen(PORT, HOST, () => { - console.log(`App listening on http://${HOST}:${PORT}`); - console.log('Press Ctrl+C to quit.'); -}); - -const peerServer = ExpressPeerServer(server, { - debug: true, - path: '/', - proxied: true, - allow_discovery: false -}); -peerServer.on('connection', peerConnection); -peerServer.on('disconnect', peerDisconnect); -peerServer.on('error', peerError); -app.use('/', peerServer); -app.enable('trust proxy'); +const PREFIX = process.env.prefix || `/assist` if (process.env.uws !== "true") { - var wsapp = express(); + let wsapp = express(); wsapp.use(request_logger("[wsapp]")); - wsapp.use('/assist', socket.wsRouter); - - const wsserver = wsapp.listen(PORT + 1, HOST, () => { - console.log(`WS App listening on http://${HOST}:${PORT + 1}`); + wsapp.use(request_logger("[app]")); + wsapp.use(`/heapdump/${process.env.S3_KEY}`, dumps.router); + wsapp.use(`${PREFIX}/${process.env.S3_KEY}`, socket.wsRouter); + wsapp.enable('trust proxy'); + const wsserver = wsapp.listen(PORT, HOST, () => { + console.log(`WS App listening on http://${HOST}:${PORT}`); console.log('Press Ctrl+C to quit.'); }); - wsapp.enable('trust proxy'); + socket.start(wsserver); - module.exports = {wsserver, server}; + module.exports = {wsserver}; } else { console.log("Using uWebSocket"); const {App} = require("uWebSockets.js"); - const PREFIX = process.env.prefix || '/assist' + const uapp = new App(); @@ -75,6 +41,7 @@ if (process.env.uws !== "true") { } uapp.get(PREFIX, healthFn); uapp.get(`${PREFIX}/`, healthFn); + uapp.get(`${PREFIX}/${process.env.S3_KEY}`, healthFn); /* Either onAborted or simply finished request */ @@ -110,11 +77,11 @@ if (process.env.uws !== "true") { socket.start(uapp); - uapp.listen(HOST, PORT + 1, (token) => { + uapp.listen(HOST, PORT, (token) => { if (!token) { console.warn("port already in use"); } - console.log(`WS App listening on http://${HOST}:${PORT + 1}`); + console.log(`WS App listening on http://${HOST}:${PORT}`); console.log('Press Ctrl+C to quit.'); }); @@ -124,5 +91,5 @@ if (process.env.uws !== "true") { debug && console.log(err.stack); // process.exit(1); }); - module.exports = {uapp, server}; + module.exports = {uapp}; } \ No newline at end of file diff --git a/ee/utilities/servers/websocket-cluster.js b/ee/utilities/servers/websocket-cluster.js index c044043a5..f21755f9b 100644 --- a/ee/utilities/servers/websocket-cluster.js +++ b/ee/utilities/servers/websocket-cluster.js @@ -1,8 +1,8 @@ const _io = require('socket.io'); const express = require('express'); const uaParser = require('ua-parser-js'); -const geoip2Reader = require('@maxmind/geoip2-node').Reader; -const {extractPeerId} = require('./peerjs-server'); +const {extractPeerId} = require('../utils/helper'); +const {geoip} = require('../utils/geoIP'); const {createAdapter} = require("@socket.io/redis-adapter"); const {createClient} = require("redis"); const wsRouter = express.Router(); @@ -14,6 +14,7 @@ const AGENT_DISCONNECT = "AGENT_DISCONNECTED"; const AGENTS_CONNECTED = "AGENTS_CONNECTED"; const NO_SESSIONS = "SESSION_DISCONNECTED"; const SESSION_ALREADY_CONNECTED = "SESSION_ALREADY_CONNECTED"; +const SESSION_RECONNECTED = "SESSION_RECONNECTED"; const REDIS_URL = process.env.REDIS_URL || "redis://localhost:6379"; const pubClient = createClient({url: REDIS_URL}); const subClient = pubClient.duplicate(); @@ -86,8 +87,7 @@ const extractProjectKeyFromRequest = function (req) { const getAvailableRooms = async function () { - let rooms = await io.of('/').adapter.allRooms(); - return rooms; + return io.of('/').adapter.allRooms(); } const respond = function (res, data) { @@ -159,7 +159,7 @@ const socketsLive = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey !== undefined) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -174,7 +174,7 @@ const socketsLive = async function (req, res) { } } } - liveSessions[projectKey] = uniqueSessions(liveSessions[_projectKey]); + liveSessions[projectKey] = uniqueSessions(liveSessions[projectKey]); } } respond(res, liveSessions); @@ -188,7 +188,7 @@ const socketsLiveByProject = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey === _projectKey) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -203,7 +203,7 @@ const socketsLiveByProject = async function (req, res) { } } } - liveSessions[projectKey] = uniqueSessions(liveSessions[_projectKey]); + liveSessions[projectKey] = uniqueSessions(liveSessions[projectKey] || []); } } respond(res, liveSessions[_projectKey] || []); @@ -254,6 +254,7 @@ async function get_all_agents_ids(io, socket) { return agents; } + function extractSessionInfo(socket) { if (socket.handshake.query.sessionInfo !== undefined) { debug && console.log("received headers"); @@ -267,21 +268,11 @@ function extractSessionInfo(socket) { socket.handshake.query.sessionInfo.userDevice = ua.device.model || null; socket.handshake.query.sessionInfo.userDeviceType = ua.device.type || 'desktop'; socket.handshake.query.sessionInfo.userCountry = null; - - const options = { - // you can use options like `cache` or `watchForUpdates` - }; - // console.log("Looking for MMDB file in " + process.env.MAXMINDDB_FILE); - geoip2Reader.open(process.env.MAXMINDDB_FILE, options) - .then(reader => { - debug && console.log("looking for location of "); - debug && console.log(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - let country = reader.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; - }) - .catch(error => { - console.error(error); - }); + if (geoip !== null) { + debug && console.log(`looking for location of ${socket.handshake.headers['x-forwarded-for'] || socket.handshake.address}`); + let country = geoip.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); + socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; + } } } @@ -293,10 +284,6 @@ module.exports = { debug && console.log(`WS started:${socket.id}, Query:${JSON.stringify(socket.handshake.query)}`); socket.peerId = socket.handshake.query.peerId; socket.identity = socket.handshake.query.identity; - const {projectKey, sessionId} = extractPeerId(socket.peerId); - socket.sessionId = sessionId; - socket.projectKey = projectKey; - socket.lastMessageReceivedAt = Date.now(); let {c_sessions, c_agents} = await sessions_agents_count(io, socket); if (socket.identity === IDENTITIES.session) { if (c_sessions > 0) { @@ -309,6 +296,7 @@ module.exports = { debug && console.log(`notifying new session about agent-existence`); let agents_ids = await get_all_agents_ids(io, socket); io.to(socket.id).emit(AGENTS_CONNECTED, agents_ids); + socket.to(socket.peerId).emit(SESSION_RECONNECTED, socket.id); } } else if (c_sessions <= 0) { @@ -359,7 +347,6 @@ module.exports = { }); socket.onAny(async (eventName, ...args) => { - socket.lastMessageReceivedAt = Date.now(); if (socket.identity === IDENTITIES.session) { debug && console.log(`received event:${eventName}, from:${socket.identity}, sending message to room:${socket.peerId}`); socket.to(socket.peerId).emit(eventName, args[0]); @@ -383,7 +370,7 @@ module.exports = { let rooms = await io.of('/').adapter.allRooms(); let validRooms = []; console.log(` ====== Rooms: ${rooms.size} ====== `); - const arr = Array.from(rooms) + // const arr = Array.from(rooms) // const filtered = arr.filter(room => !room[1].has(room[0])) for (let i of rooms) { let {projectKey, sessionId} = extractPeerId(i); diff --git a/ee/utilities/servers/websocket.js b/ee/utilities/servers/websocket.js index 0bd397d96..a5126a78f 100644 --- a/ee/utilities/servers/websocket.js +++ b/ee/utilities/servers/websocket.js @@ -1,8 +1,8 @@ const _io = require('socket.io'); const express = require('express'); const uaParser = require('ua-parser-js'); -const geoip2Reader = require('@maxmind/geoip2-node').Reader; -const {extractPeerId} = require('./peerjs-server'); +const {extractPeerId} = require('../utils/helper'); +const {geoip} = require('../utils/geoIP'); const wsRouter = express.Router(); const UPDATE_EVENT = "UPDATE_SESSION"; const IDENTITIES = {agent: 'agent', session: 'session'}; @@ -12,6 +12,7 @@ const AGENT_DISCONNECT = "AGENT_DISCONNECTED"; const AGENTS_CONNECTED = "AGENTS_CONNECTED"; const NO_SESSIONS = "SESSION_DISCONNECTED"; const SESSION_ALREADY_CONNECTED = "SESSION_ALREADY_CONNECTED"; +const SESSION_RECONNECTED = "SESSION_RECONNECTED"; let io; const debug = process.env.debug === "1" || false; @@ -107,7 +108,7 @@ const socketsList = async function (req, res) { } respond(res, liveSessions); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-list`, socketsList); +wsRouter.get(`/sockets-list`, socketsList); const socketsListByProject = async function (req, res) { debug && console.log("[WS]looking for available sessions"); @@ -133,7 +134,7 @@ const socketsListByProject = async function (req, res) { } respond(res, liveSessions[_projectKey] || []); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-list/:projectKey`, socketsListByProject); +wsRouter.get(`/sockets-list/:projectKey`, socketsListByProject); const socketsLive = async function (req, res) { debug && console.log("[WS]looking for all available LIVE sessions"); @@ -141,7 +142,7 @@ const socketsLive = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey !== undefined) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -160,7 +161,7 @@ const socketsLive = async function (req, res) { } respond(res, liveSessions); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-live`, socketsLive); +wsRouter.get(`/sockets-live`, socketsLive); const socketsLiveByProject = async function (req, res) { debug && console.log("[WS]looking for available LIVE sessions"); @@ -169,7 +170,7 @@ const socketsLiveByProject = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey === _projectKey) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -188,7 +189,7 @@ const socketsLiveByProject = async function (req, res) { } respond(res, liveSessions[_projectKey] || []); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-live/:projectKey`, socketsLiveByProject); +wsRouter.get(`/sockets-live/:projectKey`, socketsLiveByProject); const findSessionSocketId = async (io, peerId) => { const connected_sockets = await io.in(peerId).fetchSockets(); @@ -232,6 +233,7 @@ async function get_all_agents_ids(io, socket) { return agents; } + function extractSessionInfo(socket) { if (socket.handshake.query.sessionInfo !== undefined) { debug && console.log("received headers"); @@ -245,21 +247,11 @@ function extractSessionInfo(socket) { socket.handshake.query.sessionInfo.userDevice = ua.device.model || null; socket.handshake.query.sessionInfo.userDeviceType = ua.device.type || 'desktop'; socket.handshake.query.sessionInfo.userCountry = null; - - const options = { - // you can use options like `cache` or `watchForUpdates` - }; - // console.log("Looking for MMDB file in " + process.env.MAXMINDDB_FILE); - geoip2Reader.open(process.env.MAXMINDDB_FILE, options) - .then(reader => { - debug && console.log("looking for location of "); - debug && console.log(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - let country = reader.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; - }) - .catch(error => { - console.error(error); - }); + if (geoip !== null) { + debug && console.log(`looking for location of ${socket.handshake.headers['x-forwarded-for'] || socket.handshake.address}`); + let country = geoip.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); + socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; + } } } @@ -271,10 +263,6 @@ module.exports = { debug && console.log(`WS started:${socket.id}, Query:${JSON.stringify(socket.handshake.query)}`); socket.peerId = socket.handshake.query.peerId; socket.identity = socket.handshake.query.identity; - const {projectKey, sessionId} = extractPeerId(socket.peerId); - socket.sessionId = sessionId; - socket.projectKey = projectKey; - socket.lastMessageReceivedAt = Date.now(); let {c_sessions, c_agents} = await sessions_agents_count(io, socket); if (socket.identity === IDENTITIES.session) { if (c_sessions > 0) { @@ -287,6 +275,7 @@ module.exports = { debug && console.log(`notifying new session about agent-existence`); let agents_ids = await get_all_agents_ids(io, socket); io.to(socket.id).emit(AGENTS_CONNECTED, agents_ids); + socket.to(socket.peerId).emit(SESSION_RECONNECTED, socket.id); } } else if (c_sessions <= 0) { @@ -335,7 +324,6 @@ module.exports = { }); socket.onAny(async (eventName, ...args) => { - socket.lastMessageReceivedAt = Date.now(); if (socket.identity === IDENTITIES.session) { debug && console.log(`received event:${eventName}, from:${socket.identity}, sending message to room:${socket.peerId}`); socket.to(socket.peerId).emit(eventName, args[0]); diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager.ts b/frontend/app/player/MessageDistributor/managers/AssistManager.ts index b684919f9..e888e18a4 100644 --- a/frontend/app/player/MessageDistributor/managers/AssistManager.ts +++ b/frontend/app/player/MessageDistributor/managers/AssistManager.ts @@ -144,7 +144,6 @@ export default class AssistManager { }) socket.on('messages', messages => { //console.log(messages.filter(m => m._id === 41 || m._id === 44)) - showDisconnectTimeout && clearTimeout(showDisconnectTimeout); jmr.append(messages) // as RawMessage[] if (waitingForMessages) { @@ -168,6 +167,9 @@ export default class AssistManager { socket.on("control_rejected", id => { id === socket.id && this.toggleRemoteControl(false) }) + socket.on('SESSION_RECONNECTED', () => { + showDisconnectTimeout && clearTimeout(showDisconnectTimeout) + }) socket.on('SESSION_DISCONNECTED', e => { waitingForMessages = true showDisconnectTimeout = setTimeout(() => { diff --git a/peers/.gitignore b/peers/.gitignore new file mode 100644 index 000000000..a4b05b411 --- /dev/null +++ b/peers/.gitignore @@ -0,0 +1,6 @@ +.idea +node_modules +npm-debug.log +.cache +test.html +/utils/ diff --git a/peers/Dockerfile b/peers/Dockerfile new file mode 100644 index 000000000..d053bf3af --- /dev/null +++ b/peers/Dockerfile @@ -0,0 +1,14 @@ +FROM node:17-stretch +WORKDIR /work +COPY . . +RUN npm install + +# Add Tini +# Startup daemon +ENV TINI_VERSION v0.19.0 +ARG envarg +ENV ENTERPRISE_BUILD ${envarg} +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] +CMD npm start \ No newline at end of file diff --git a/peers/build.sh b/peers/build.sh new file mode 100644 index 000000000..c15921ea8 --- /dev/null +++ b/peers/build.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Script to build api module +# flags to accept: +# Default will be OSS build. + +# Usage: IMAGE_TAG=latest DOCKER_REPO=myDockerHubID bash build.sh + +git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)} +check_prereq() { + which docker || { + echo "Docker not installed, please install docker." + exit=1 + } + [[ exit -eq 1 ]] && exit 1 +} + +function build_api(){ + cp -R ../utilities/utils . + # Copy enterprise code + [[ $1 == "ee" ]] && { + cp -rf ../ee/peers/* ./ + } + docker build -f ./Dockerfile -t ${DOCKER_REPO:-'local'}/peers:${git_sha1} . + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/peers:${git_sha1} + docker tag ${DOCKER_REPO:-'local'}/peers:${git_sha1} ${DOCKER_REPO:-'local'}/peers:latest + docker push ${DOCKER_REPO:-'local'}/peers:latest + } +} + +check_prereq +build_api $1 diff --git a/peers/package-lock.json b/peers/package-lock.json new file mode 100644 index 000000000..5f253d3bc --- /dev/null +++ b/peers/package-lock.json @@ -0,0 +1,1584 @@ +{ + "name": "utilities_server", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "utilities_server", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "express": "^4.17.1", + "peer": "^0.6.1" + } + }, + "node_modules/@types/body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.35", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", + "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/cors": { + "version": "2.8.12", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", + "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" + }, + "node_modules/@types/express": { + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", + "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.18", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.17.28", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", + "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*" + } + }, + "node_modules/@types/mime": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", + "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" + }, + "node_modules/@types/node": { + "version": "17.0.21", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", + "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" + }, + "node_modules/@types/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" + }, + "node_modules/@types/range-parser": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" + }, + "node_modules/@types/serve-static": { + "version": "1.13.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", + "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/ws": { + "version": "7.4.7", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", + "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "node_modules/body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.9.7", + "raw-body": "2.4.3", + "type-is": "~1.6.18" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cliui": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", + "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^6.2.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", + "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.19.2", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.4.2", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.9.7", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", + "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.34", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", + "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", + "dependencies": { + "mime-db": "1.51.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "node_modules/peer": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", + "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", + "dependencies": { + "@types/cors": "^2.8.6", + "@types/express": "^4.17.3", + "@types/ws": "^7.2.3", + "body-parser": "^1.19.0", + "cors": "^2.8.5", + "express": "^4.17.1", + "uuid": "^3.4.0", + "ws": "^7.2.3", + "yargs": "^15.3.1" + }, + "bin": { + "peerjs": "bin/peerjs" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/peer/node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==", + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", + "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "dependencies": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "1.8.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/serve-static": { + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" + }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ws": { + "version": "7.5.7", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", + "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" + }, + "node_modules/yargs": { + "version": "15.4.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", + "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "dependencies": { + "cliui": "^6.0.0", + "decamelize": "^1.2.0", + "find-up": "^4.1.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^4.2.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^18.1.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs-parser/node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "engines": { + "node": ">=6" + } + } + }, + "dependencies": { + "@types/body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", + "requires": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "@types/connect": { + "version": "3.4.35", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", + "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", + "requires": { + "@types/node": "*" + } + }, + "@types/cors": { + "version": "2.8.12", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", + "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" + }, + "@types/express": { + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", + "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "requires": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.18", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "@types/express-serve-static-core": { + "version": "4.17.28", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", + "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", + "requires": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*" + } + }, + "@types/mime": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", + "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" + }, + "@types/node": { + "version": "17.0.21", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", + "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" + }, + "@types/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" + }, + "@types/range-parser": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" + }, + "@types/serve-static": { + "version": "1.13.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", + "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", + "requires": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "@types/ws": { + "version": "7.4.7", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", + "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", + "requires": { + "@types/node": "*" + } + }, + "accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "requires": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + } + }, + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", + "requires": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.9.7", + "raw-body": "2.4.3", + "type-is": "~1.6.18" + } + }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "cliui": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", + "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^6.2.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "requires": { + "safe-buffer": "5.2.1" + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "requires": { + "object-assign": "^4", + "vary": "^1" + } + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" + }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + }, + "express": { + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", + "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", + "requires": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.19.2", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.4.2", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.9.7", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + } + }, + "finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, + "http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "requires": { + "p-locate": "^4.1.0" + } + }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + }, + "mime-db": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", + "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==" + }, + "mime-types": { + "version": "2.1.34", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", + "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", + "requires": { + "mime-db": "1.51.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "requires": { + "p-limit": "^2.2.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "peer": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", + "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", + "requires": { + "@types/cors": "^2.8.6", + "@types/express": "^4.17.3", + "@types/ws": "^7.2.3", + "body-parser": "^1.19.0", + "cors": "^2.8.5", + "express": "^4.17.1", + "uuid": "^3.4.0", + "ws": "^7.2.3", + "yargs": "^15.3.1" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } + } + }, + "proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "requires": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + } + }, + "qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==" + }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" + }, + "raw-body": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", + "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", + "requires": { + "bytes": "3.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "1.8.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + }, + "serve-static": { + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.2" + } + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" + }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" + }, + "wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, + "ws": { + "version": "7.5.7", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", + "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", + "requires": {} + }, + "y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" + }, + "yargs": { + "version": "15.4.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", + "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "requires": { + "cliui": "^6.0.0", + "decamelize": "^1.2.0", + "find-up": "^4.1.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^4.2.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^18.1.2" + } + }, + "yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "dependencies": { + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" + } + } + } + } +} diff --git a/peers/package.json b/peers/package.json new file mode 100644 index 000000000..8a87b3b6a --- /dev/null +++ b/peers/package.json @@ -0,0 +1,24 @@ +{ + "name": "peers-server", + "version": "1.0.0", + "description": "assist server to get live sessions & sourcemaps reader to get stack trace", + "main": "peerjs-server.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node server.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/openreplay/openreplay.git" + }, + "author": "KRAIEM Taha Yassine ", + "license": "Elastic License 2.0 (ELv2)", + "bugs": { + "url": "https://github.com/openreplay/openreplay/issues" + }, + "homepage": "https://github.com/openreplay/openreplay#readme", + "dependencies": { + "express": "^4.17.1", + "peer": "^0.6.1" + } +} diff --git a/peers/server.js b/peers/server.js new file mode 100644 index 000000000..39f46d4f1 --- /dev/null +++ b/peers/server.js @@ -0,0 +1,33 @@ +const dumps = require('./utils/HeapSnapshot'); +const {request_logger} = require('./utils/helper'); +const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); +const express = require('express'); +const {ExpressPeerServer} = require('peer'); + +const HOST = '0.0.0.0'; +const PORT = 9000; + +const app = express(); + +app.use(request_logger("[app]")); + +app.use(`/${process.env.S3_KEY}/assist`, peerRouter); +app.use(`/${process.env.S3_KEY}/heapdump`, dumps.router); + +const server = app.listen(PORT, HOST, () => { + console.log(`App listening on http://${HOST}:${PORT}`); + console.log('Press Ctrl+C to quit.'); +}); + +const peerServer = ExpressPeerServer(server, { + debug: true, + path: '/', + proxied: true, + allow_discovery: false +}); +peerServer.on('connection', peerConnection); +peerServer.on('disconnect', peerDisconnect); +peerServer.on('error', peerError); +app.use('/', peerServer); +app.enable('trust proxy'); +module.exports = {server}; \ No newline at end of file diff --git a/utilities/servers/peerjs-server.js b/peers/servers/peerjs-server.js similarity index 70% rename from utilities/servers/peerjs-server.js rename to peers/servers/peerjs-server.js index 434e5f4bc..a99ec1665 100644 --- a/utilities/servers/peerjs-server.js +++ b/peers/servers/peerjs-server.js @@ -1,20 +1,9 @@ -var express = require('express'); -var peerRouter = express.Router(); +const express = require('express'); +const peerRouter = express.Router(); +const {extractPeerId} = require('../utils/helper'); -let PROJECT_KEY_LENGTH = parseInt(process.env.PROJECT_KEY_LENGTH) || 20; let debug = process.env.debug === "1" || false; -const extractPeerId = (peerId) => { - let splited = peerId.split("-"); - if (splited.length !== 2) { - debug && console.error(`cannot split peerId: ${peerId}`); - return {}; - } - if (PROJECT_KEY_LENGTH > 0 && splited[0].length !== PROJECT_KEY_LENGTH) { - debug && console.error(`wrong project key length for peerId: ${peerId}`); - return {}; - } - return {projectKey: splited[0], sessionId: splited[1]}; -}; + const connectedPeers = {}; const peerConnection = (client) => { @@ -53,13 +42,13 @@ const peerError = (error) => { } -peerRouter.get(`/${process.env.S3_KEY}/peers`, function (req, res) { +peerRouter.get(`/peers`, function (req, res) { debug && console.log("looking for all available sessions"); res.statusCode = 200; res.setHeader('Content-Type', 'application/json'); res.end(JSON.stringify({"data": connectedPeers})); }); -peerRouter.get(`/${process.env.S3_KEY}/peers/:projectKey`, function (req, res) { +peerRouter.get(`/peers/:projectKey`, function (req, res) { debug && console.log(`looking for available sessions for ${req.params.projectKey}`); res.statusCode = 200; res.setHeader('Content-Type', 'application/json'); diff --git a/scripts/helm/build_deploy.sh b/scripts/helm/build_deploy.sh index 86925ebe8..7a75fad8b 100644 --- a/scripts/helm/build_deploy.sh +++ b/scripts/helm/build_deploy.sh @@ -17,4 +17,6 @@ echo $DOCKER_REPO PUSH_IMAGE=1 bash build.sh $@ cd ../utilities PUSH_IMAGE=1 bash build.sh $@ + cd ../peers + PUSH_IMAGE=1 bash build.sh $@ } diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql new file mode 100644 index 000000000..9e6421b07 --- /dev/null +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -0,0 +1,136 @@ +BEGIN; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.5.5' +$$ LANGUAGE sql IMMUTABLE; + + +CREATE TABLE IF NOT EXISTS dashboards +( + dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT TRUE, + is_pinned boolean NOT NULL DEFAULT FALSE, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + deleted_at timestamp NULL DEFAULT NULL +); + + +ALTER TABLE IF EXISTS metrics + DROP CONSTRAINT IF EXISTS null_project_id_for_template_only, + DROP CONSTRAINT IF EXISTS unique_key; + +ALTER TABLE IF EXISTS metrics + ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom', + ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS default_config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb, + ALTER COLUMN project_id DROP NOT NULL, + ADD CONSTRAINT null_project_id_for_template_only + CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), + ADD CONSTRAINT unique_key UNIQUE (predefined_key); + + + +CREATE TABLE IF NOT EXISTS dashboard_widgets +( + widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, + metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + config jsonb NOT NULL DEFAULT '{}'::jsonb +); + +ALTER TABLE events_common.requests + ADD COLUMN IF NOT EXISTS host text NULL, + ADD COLUMN IF NOT EXISTS base_path text NULL, + ADD COLUMN IF NOT EXISTS query text NULL; + +ALTER TABLE events.pages + ADD COLUMN IF NOT EXISTS query text NULL; +COMMIT; + +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'barChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarLineChart'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'overview'; +ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'map'; +ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'predefined'; + +INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type) +VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('Image Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('DOM Content Load Start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('First Meaningful paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('No. of Visited Pages', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('Session Duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('DOM Build Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('Pages Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('First Paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('DOM Content Loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('Time Till First byte', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), + ('Time To Interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('Captured requests', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'), + ('Time To Render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('Memory Consumption', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('CPU Load', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('Frame rate', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_fps', 'predefined', 'overview'), + + ('Sessions Affected by JS Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('Top Domains with 4xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('Top Domains with 5xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('Errors per Domain', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('Fetch Calls with Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'), + ('Errors by Type', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('Errors by Origin', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + + ('Speed Index by Location', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'), + ('Slowest Domains', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('Sessions per Browser', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('Time To Render', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('Sessions Impacted by Slow Pages', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('Memory Consumption', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('CPU Load', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('Frame Rate', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'), + ('Crashes', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('Resources Loaded vs Visually Complete', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('DOM Build Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), + + ('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{"col":4,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'), + ('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), + ('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') +ON CONFLICT (predefined_key) DO UPDATE + SET name=excluded.name, + category=excluded.category, + default_config=excluded.default_config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public, + metric_type=excluded.metric_type, + view_type=excluded.view_type; + + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; +CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 834a79968..7f4d864fe 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS events; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.5.4' +SELECT 'v1.5.5' $$ LANGUAGE sql IMMUTABLE; -- --- accounts.sql --- @@ -645,6 +645,9 @@ $$ response_body text NULL, status_code smallint NULL, method http_method NULL, + host text NULL, + base_path text NULL, + query text NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); CREATE INDEX requests_url_idx ON events_common.requests (url); @@ -664,6 +667,12 @@ $$ CREATE INDEX requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL; CREATE INDEX requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; CREATE INDEX requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + CREATE INDEX requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; + CREATE INDEX requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; + CREATE INDEX requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL; + CREATE INDEX requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL; + CREATE INDEX requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL; + CREATE INDEX requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; -- --- events.sql --- CREATE SCHEMA IF NOT EXISTS events; @@ -676,6 +685,7 @@ $$ host text NOT NULL, path text NOT NULL, base_path text NOT NULL, + query text NULL, referrer text DEFAULT NULL, base_referrer text DEFAULT NULL, dom_building_time integer DEFAULT NULL, @@ -731,6 +741,8 @@ $$ CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; CREATE INDEX pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; + CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL; + CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; CREATE TABLE events.clicks @@ -935,24 +947,39 @@ $$ CREATE INDEX jobs_start_at_idx ON jobs (start_at); CREATE INDEX jobs_project_id_idx ON jobs (project_id); - CREATE TYPE metric_type AS ENUM ('timeseries','table'); - CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart'); + CREATE TYPE metric_type AS ENUM ('timeseries','table', 'predefined'); + CREATE TYPE metric_view_type AS ENUM ('lineChart','progress','table','pieChart','areaChart','barChart','stackedBarChart','stackedBarLineChart','overview','map'); CREATE TABLE metrics ( - metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - user_id integer REFERENCES users (user_id) ON DELETE SET NULL, - name text NOT NULL, - is_public boolean NOT NULL DEFAULT FALSE, - active boolean NOT NULL DEFAULT TRUE, - created_at timestamp default timezone('utc'::text, now()) not null, - deleted_at timestamp, - metric_type metric_type NOT NULL DEFAULT 'timeseries', - view_type metric_view_type NOT NULL DEFAULT 'lineChart', - metric_of text NOT NULL DEFAULT 'sessionCount', - metric_value text[] NOT NULL DEFAULT '{}'::text[], - metric_format text + metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT FALSE, + active boolean NOT NULL DEFAULT TRUE, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp, + edited_at timestamp, + metric_type metric_type NOT NULL DEFAULT 'timeseries', + view_type metric_view_type NOT NULL DEFAULT 'lineChart', + metric_of text NOT NULL DEFAULT 'sessionCount', + metric_value text[] NOT NULL DEFAULT '{}'::text[], + metric_format text, + category text NULL DEFAULT 'custom', + is_pinned boolean NOT NULL DEFAULT FALSE, + is_predefined boolean NOT NULL DEFAULT FALSE, + is_template boolean NOT NULL DEFAULT FALSE, + predefined_key text NULL DEFAULT NULL, + default_config jsonb NOT NULL DEFAULT '{ + "col": 2, + "row": 2, + "position": 0 + }'::jsonb, + CONSTRAINT null_project_id_for_template_only + CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ), + CONSTRAINT unique_key UNIQUE (predefined_key) ); + CREATE INDEX metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); CREATE TABLE metric_series ( @@ -966,6 +993,28 @@ $$ ); CREATE INDEX metric_series_metric_id_idx ON public.metric_series (metric_id); + CREATE TABLE dashboards + ( + dashboard_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT TRUE, + is_pinned boolean NOT NULL DEFAULT FALSE, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + deleted_at timestamp NULL DEFAULT NULL + ); + + CREATE TABLE dashboard_widgets + ( + widget_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + dashboard_id integer NOT NULL REFERENCES dashboards (dashboard_id) ON DELETE CASCADE, + metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + config jsonb NOT NULL DEFAULT '{}'::jsonb + ); + CREATE TABLE searches ( @@ -1015,4 +1064,244 @@ $$ $$ LANGUAGE plpgsql; +INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, + view_type) +VALUES ('Captured sessions', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 +}', true, true, true, 'count_sessions', 'predefined', 'overview'), + ('Request Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_request_load_time', 'predefined', 'overview'), + ('Page Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_page_load_time', 'predefined', 'overview'), + ('Image Load Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_image_load_time', 'predefined', 'overview'), + ('DOM Content Load Start', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'), + ('First Meaningful paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'), + ('No. of Visited Pages', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_visited_pages', 'predefined', 'overview'), + ('Session Duration', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_session_duration', 'predefined', 'overview'), + ('DOM Build Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'), + ('Pages Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'), + ('Response Time', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_response_time', 'predefined', 'overview'), + ('First Paint', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_first_paint', 'predefined', 'overview'), + ('DOM Content Loaded', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'), + ('Time Till First byte', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'), + ('Time To Interactive', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'), + ('Captured requests', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'count_requests', 'predefined', 'overview'), + ('Time To Render', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_time_to_render', 'predefined', 'overview'), + ('Memory Consumption', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'), + ('CPU Load', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_cpu', 'predefined', 'overview'), + ('Frame rate', 'overview', '{ + "col": 1, + "row": 1, + "position": 0 + }', true, true, true, 'avg_fps', 'predefined', 'overview'), + + ('Sessions Affected by JS Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'), + ('Top Domains with 4xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'), + ('Top Domains with 5xx Fetch Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'), + ('Errors per Domain', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_domains', 'predefined', 'table'), + ('Fetch Calls with Errors', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'calls_errors', 'predefined', 'table'), + ('Errors by Type', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'errors_per_type', 'predefined', 'barChart'), + ('Errors by Origin', 'errors', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'), + + ('Speed Index by Location', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'speed_location', 'predefined', 'map'), + ('Slowest Domains', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_domains', 'predefined', 'table'), + ('Sessions per Browser', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'sessions_per_browser', 'predefined', 'table'), + ('Time To Render', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'time_to_render', 'predefined', 'areaChart'), + ('Sessions Impacted by Slow Pages', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'), + ('Memory Consumption', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'memory_consumption', 'predefined', 'areaChart'), + ('CPU Load', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'cpu', 'predefined', 'areaChart'), + ('Frame Rate', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'fps', 'predefined', 'areaChart'), + ('Crashes', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'crashes', 'predefined', 'areaChart'), + ('Resources Loaded vs Visually Complete', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'), + ('DOM Build Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'), + ('Pages Response Time', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time', 'predefined', 'areaChart'), + ('Pages Response Time Distribution', 'performance', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'), + + ('Missing Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'missing_resources', 'predefined', 'table'), + ('Slowest Resources', 'resources', '{ + "col": 4, + "row": 2, + "position": 0 + }', true, true, true, 'slowest_resources', 'predefined', 'table'), + ('Resources Fetch Time', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_loading_time', 'predefined', 'table'), + ('Resource Loaded vs Response End', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'), + ('Breakdown of Loaded Resources', 'resources', '{ + "col": 2, + "row": 2, + "position": 0 + }', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart') +ON CONFLICT (predefined_key) DO UPDATE + SET name=excluded.name, + category=excluded.category, + default_config=excluded.default_config, + is_predefined=excluded.is_predefined, + is_template=excluded.is_template, + is_public=excluded.is_public, + metric_type=excluded.metric_type, + view_type=excluded.view_type; + COMMIT; \ No newline at end of file diff --git a/scripts/helm/install.sh b/scripts/helm/install.sh index 529d08a8c..2bf39988c 100755 --- a/scripts/helm/install.sh +++ b/scripts/helm/install.sh @@ -26,7 +26,7 @@ which docker &> /dev/null || { } -# https://parrot.asayer.io/os/license +# https://api.openreplay.com/os/license # payload: {"mid": "UUID of the machine", "license": ""} # response {"data":{"valid": TRUE|FALSE, "expiration": expiration date in ms}} diff --git a/scripts/helm/roles/openreplay/tasks/pre-check.yaml b/scripts/helm/roles/openreplay/tasks/pre-check.yaml index 7f85d0ea1..90e6021c8 100644 --- a/scripts/helm/roles/openreplay/tasks/pre-check.yaml +++ b/scripts/helm/roles/openreplay/tasks/pre-check.yaml @@ -108,7 +108,7 @@ - all - name: Checking Enterprise Licence uri: - url: https://parrot.asayer.io/os/license + url: https://api.openreplay.com/os/license body: mid: "UUID of the machine" license: "{{ enterprise_edition_license }}" diff --git a/scripts/helmcharts/certmanager.sh b/scripts/helmcharts/certmanager.sh new file mode 100644 index 000000000..daab4d661 --- /dev/null +++ b/scripts/helmcharts/certmanager.sh @@ -0,0 +1,39 @@ +# --- helper functions for logs --- +info() +{ + echo '[INFO] ' "$@" +} +warn() +{ + echo '[WARN] ' "$@" >&2 +} +fatal() +{ + echo '[ERROR] ' "$@" >&2 + exit 1 +} + +read -p "enter openreplay domain name: " domain +nslookup $domain > /dev/null || { + fatal "Domain name does not have ip associated with it. Please check your DNS record." +} + +# Reading email address for ssl certificate +[[ -z $EMAIL_ADDRESS ]] && { + read -p "Enter your email address for letsencrypt certificate: " EMAIL_ADDRESS + echo +} +if [[ "$EMAIL_ADDRESS" =~ ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}$ ]] +then + info "Email address $EMAIL_ADDRESS is valid." +else + fatal "Email address $EMAIL_ADDRESS is invalid." +fi + +sed -i "s/email: .*/email: \"${EMAIL_ADDRESS}\"/g" clusterIssuer.yaml +info "Installing cert-manager for auto letsencrypt certificate" +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.0/cert-manager.crds.yaml +helm repo add jetstack https://charts.jetstack.io +helm repo update +helm upgrade --install cert-manager --namespace cert-manager --version v1.8.0 jetstack/cert-manager --create-namespace +kubectl apply -f clusterIssuer.yaml diff --git a/scripts/helmcharts/clusterIssuer.yaml b/scripts/helmcharts/clusterIssuer.yaml new file mode 100644 index 000000000..fa18dfb3e --- /dev/null +++ b/scripts/helmcharts/clusterIssuer.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + # The ACME server URL + server: https://acme-v02.api.letsencrypt.org/directory + # Email address used for ACME registration + email: + # Name of a secret used to store the ACME account private key + privateKeySecretRef: + name: letsencrypt-prod + # Enable the HTTP-01 challenge provider + solvers: + - http01: + ingress: + class: openreplay diff --git a/scripts/helmcharts/init.sh b/scripts/helmcharts/init.sh index 281e8586b..078723007 100644 --- a/scripts/helmcharts/init.sh +++ b/scripts/helmcharts/init.sh @@ -19,7 +19,7 @@ version="v1.5.4" usr=`whoami` # Installing k3s -curl -sL https://get.k3s.io | sudo K3S_KUBECONFIG_MODE="644" INSTALL_K3S_VERSION='v1.19.5+k3s2' INSTALL_K3S_EXEC="--no-deploy=traefik" sh - +curl -sL https://get.k3s.io | sudo K3S_KUBECONFIG_MODE="644" INSTALL_K3S_VERSION='v1.22.8+k3s1' INSTALL_K3S_EXEC="--no-deploy=traefik" sh - mkdir ~/.kube sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config chmod 0644 ~/.kube/config @@ -81,7 +81,6 @@ sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"$(randomPass)\"/g" v sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"$(randomPass)\"/g" vars.yaml sed -i "s/domainName: \"\"/domainName: \"${DOMAIN_NAME}\"/g" vars.yaml - ## Installing OpenReplay info "Installing databases" helm upgrade --install databases ./databases -n db --create-namespace --wait -f ./vars.yaml --atomic diff --git a/scripts/helmcharts/openreplay/Chart.yaml b/scripts/helmcharts/openreplay/Chart.yaml index be16d6344..827f8566e 100644 --- a/scripts/helmcharts/openreplay/Chart.yaml +++ b/scripts/helmcharts/openreplay/Chart.yaml @@ -23,3 +23,8 @@ version: 0.1.0 # It is recommended to use it with quotes. # Ref: https://github.com/helm/helm/issues/7858#issuecomment-608114589 AppVersion: "v1.5.4" + +dependencies: + - name: ingress-nginx + version: "4.x.x" + repository: "https://kubernetes.github.io/ingress-nginx" diff --git a/scripts/helmcharts/openreplay/charts/assets/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/ingress.yaml index 62c421a9a..3c6ae561b 100644 --- a/scripts/helmcharts/openreplay/charts/assets/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/templates/ingress.yaml @@ -1,61 +1,34 @@ -{{- if .Values.ingress.enabled -}} +{{- if .Values.ingress.enabled }} {{- $fullName := include "assets.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} kind: Ingress metadata: name: {{ $fullName }} labels: {{- include "assets.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 + {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} + ingressClassName: "{{ tpl .Values.ingress.className . }}" tls: - {{- range .Values.ingress.tls }} - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} + - host: {{ .Values.global.domainName }} http: paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} + - pathType: Prefix backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: name: {{ $fullName }} port: number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} + path: /assets/(.*) {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/assets/values.yaml b/scripts/helmcharts/openreplay/charts/assets/values.yaml index 2069e0fe8..709f502d1 100644 --- a/scripts/helmcharts/openreplay/charts/assets/values.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/values.yaml @@ -41,20 +41,13 @@ service: port: 9000 ingress: - enabled: false - className: "" + enabled: true + className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}" annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local + tls: + secretName: openreplay-ssl resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/.helmignore b/scripts/helmcharts/openreplay/charts/assist/.helmignore similarity index 100% rename from scripts/helmcharts/openreplay/charts/nginx-ingress/.helmignore rename to scripts/helmcharts/openreplay/charts/assist/.helmignore diff --git a/scripts/helmcharts/openreplay/charts/utilities/Chart.yaml b/scripts/helmcharts/openreplay/charts/assist/Chart.yaml similarity index 81% rename from scripts/helmcharts/openreplay/charts/utilities/Chart.yaml rename to scripts/helmcharts/openreplay/charts/assist/Chart.yaml index 203871558..0bcde4fd3 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/Chart.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: utilities +name: assist description: A Helm chart for Kubernetes # A chart can be either an 'application' or a 'library' chart. @@ -7,8 +7,8 @@ description: A Helm chart for Kubernetes # Application charts are a collection of templates that can be packaged into versioned archives # to be deployed. # -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering +# Library charts provide useful assist or functions for the chart developer. They're included as +# a dependency of application charts to inject those assist and functions into the rendering # pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/assist/templates/NOTES.txt similarity index 77% rename from scripts/helmcharts/openreplay/charts/utilities/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/assist/templates/NOTES.txt index 323bc20c1..57c8efcf4 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/NOTES.txt +++ b/scripts/helmcharts/openreplay/charts/assist/templates/NOTES.txt @@ -6,16 +6,16 @@ {{- end }} {{- end }} {{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "utilities.fullname" . }}) + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "assist.fullname" . }}) export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo http://$NODE_IP:$NODE_PORT {{- else if contains "LoadBalancer" .Values.service.type }} NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "utilities.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "utilities.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "assist.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "assist.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") echo http://$SERVICE_IP:{{ .Values.service.port }} {{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "utilities.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "assist.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") echo "Visit http://127.0.0.1:8080 to use your application" kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/assist/templates/_helpers.tpl similarity index 75% rename from scripts/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/assist/templates/_helpers.tpl index 8999db4be..95cab7e2b 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl +++ b/scripts/helmcharts/openreplay/charts/assist/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "utilities.name" -}} +{{- define "assist.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "utilities.fullname" -}} +{{- define "assist.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "utilities.chart" -}} +{{- define "assist.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "utilities.labels" -}} -helm.sh/chart: {{ include "utilities.chart" . }} -{{ include "utilities.selectorLabels" . }} +{{- define "assist.labels" -}} +helm.sh/chart: {{ include "assist.chart" . }} +{{ include "assist.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "utilities.selectorLabels" -}} -app.kubernetes.io/name: {{ include "utilities.name" . }} +{{- define "assist.selectorLabels" -}} +app.kubernetes.io/name: {{ include "assist.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Create the name of the service account to use */}} -{{- define "utilities.serviceAccountName" -}} +{{- define "assist.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} -{{- default (include "utilities.fullname" .) .Values.serviceAccount.name }} +{{- default (include "assist.fullname" .) .Values.serviceAccount.name }} {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml similarity index 88% rename from scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml index 7a3d5b0b6..be98db206 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml @@ -1,16 +1,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "utilities.fullname" . }} + name: {{ include "assist.fullname" . }} labels: - {{- include "utilities.labels" . | nindent 4 }} + {{- include "assist.labels" . | nindent 4 }} spec: {{- if not .Values.autoscaling.enabled }} replicas: {{ .Values.replicaCount }} {{- end }} selector: matchLabels: - {{- include "utilities.selectorLabels" . | nindent 6 }} + {{- include "assist.selectorLabels" . | nindent 6 }} template: metadata: {{- with .Values.podAnnotations }} @@ -18,13 +18,13 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "utilities.selectorLabels" . | nindent 8 }} + {{- include "assist.selectorLabels" . | nindent 8 }} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "utilities.serviceAccountName" . }} + serviceAccountName: {{ include "assist.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/hpa.yaml similarity index 84% rename from scripts/helmcharts/openreplay/charts/utilities/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/assist/templates/hpa.yaml index 8944056ea..b0b394a20 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/hpa.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/hpa.yaml @@ -2,14 +2,14 @@ apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: - name: {{ include "utilities.fullname" . }} + name: {{ include "assist.fullname" . }} labels: - {{- include "utilities.labels" . | nindent 4 }} + {{- include "assist.labels" . | nindent 4 }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ include "utilities.fullname" . }} + name: {{ include "assist.fullname" . }} minReplicas: {{ .Values.autoscaling.minReplicas }} maxReplicas: {{ .Values.autoscaling.maxReplicas }} metrics: diff --git a/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml new file mode 100644 index 000000000..f791024bb --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml @@ -0,0 +1,35 @@ +{{- if .Values.ingress.enabled }} +{{- $fullName := include "assist.fullname" . -}} +{{- $socketioSvcPort := .Values.service.ports.socketio -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "assist.labels" . | nindent 4 }} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/upstream-hash-by: $http_x_forwarded_for + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: "{{ tpl .Values.ingress.className . }}" + tls: + - hosts: + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} + rules: + - host: {{ .Values.global.domainName }} + http: + paths: + - pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ $socketioSvcPort }} + path: /ws-assist/(.*) +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/assist/templates/service.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/service.yaml new file mode 100644 index 000000000..13b78654c --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/assist/templates/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "assist.fullname" . }} + labels: + {{- include "assist.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + {{- range $key, $val := .Values.service.ports }} + - port: {{ $val }} + targetPort: {{ $key }} + protocol: TCP + name: {{ $key }} + {{- end}} + selector: + {{- include "assist.selectorLabels" . | nindent 4 }} diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/serviceaccount.yaml similarity index 66% rename from scripts/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/assist/templates/serviceaccount.yaml index dd5c35012..18e86a3e8 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/serviceaccount.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "utilities.serviceAccountName" . }} + name: {{ include "assist.serviceAccountName" . }} labels: - {{- include "utilities.labels" . | nindent 4 }} + {{- include "assist.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/scripts/helmcharts/openreplay/charts/utilities/values.yaml b/scripts/helmcharts/openreplay/charts/assist/values.yaml similarity index 82% rename from scripts/helmcharts/openreplay/charts/utilities/values.yaml rename to scripts/helmcharts/openreplay/charts/assist/values.yaml index 8265b5aa5..a78be6a33 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/values.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/values.yaml @@ -5,14 +5,14 @@ replicaCount: 1 image: - repository: rg.fr-par.scw.cloud/foss/utilities + repository: rg.fr-par.scw.cloud/foss/assist pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "" imagePullSecrets: [] -nameOverride: "utililities" -fullnameOverride: "utililities" +nameOverride: "assist" +fullnameOverride: "assist" serviceAccount: # Specifies whether a service account should be created @@ -43,24 +43,16 @@ securityContext: {} service: type: ClusterIP ports: - peerjs: 9000 socketio: 9001 ingress: - enabled: false - className: "" + enabled: true + className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}" annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local + tls: + secretName: openreplay-ssl resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/ingress.yaml index 8f680e949..d870e7673 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/ingress.yaml @@ -1,61 +1,34 @@ -{{- if .Values.ingress.enabled -}} +{{- if .Values.ingress.enabled }} {{- $fullName := include "chalice.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} kind: Ingress metadata: name: {{ $fullName }} labels: {{- include "chalice.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 + {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} + ingressClassName: "{{ tpl .Values.ingress.className . }}" tls: - {{- range .Values.ingress.tls }} - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} + - host: {{ .Values.global.domainName }} http: paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} + - pathType: Prefix backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: name: {{ $fullName }} port: number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} + path: /api/(.*) {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index 3290b9872..a95ed4732 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -41,20 +41,13 @@ service: port: 8000 ingress: - enabled: false - className: "" + enabled: true + className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}" annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local + tls: + secretName: openreplay-ssl resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml index 51a20e2a6..8888c7ae9 100644 --- a/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml @@ -1,61 +1,93 @@ -{{- if .Values.ingress.enabled -}} +{{- if .Values.ingress.enabled }} {{- $fullName := include "http.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} kind: Ingress metadata: name: {{ $fullName }} labels: {{- include "http.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/upstream-hash-by: $http_x_forwarded_for + {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} + ingressClassName: "{{ tpl .Values.ingress.className . }}" tls: - {{- range .Values.ingress.tls }} - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} + - host: {{ .Values.global.domainName }} http: paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} + - pathType: Prefix backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: name: {{ $fullName }} port: number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} + path: /ingest/(.*) {{- end }} + +## TODO: +## Frontend service from minio will be migrated to nginx atomic container. +## This ingress is just a workaround. +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: minio + namespace: db +spec: + ingressClassName: "{{ tpl .Values.ingress.className . }}" + rules: + - host: {{ .Values.global.domainName }} + http: + paths: + - pathType: Prefix + backend: + service: + name: minio + port: + number: 9000 + path: /(minio|mobs|sessions-assets|frontend|static|sourcemaps|ios-images)/ + tls: + - hosts: + - {{ .Values.global.domainName }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: minio-frontend + namespace: db + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /frontend/$1 + nginx.ingress.kubernetes.io/configuration-snippet: | + index /index.html; + rewrite ^((?!.(js|css|png|svg|jpg|woff|woff2)).)*$ /frontend/index.html break; + proxy_intercept_errors on; # see http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_intercept_errors + error_page 404 =200 /index.html; +spec: + ingressClassName: "{{ tpl .Values.ingress.className . }}" + rules: + - host: {{ .Values.global.domainName }} + http: + paths: + - pathType: Prefix + backend: + service: + name: minio + port: + number: 9000 + path: /(.*) + tls: + - hosts: + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} diff --git a/scripts/helmcharts/openreplay/charts/http/values.yaml b/scripts/helmcharts/openreplay/charts/http/values.yaml index 0eb280a12..1f52ed5cf 100644 --- a/scripts/helmcharts/openreplay/charts/http/values.yaml +++ b/scripts/helmcharts/openreplay/charts/http/values.yaml @@ -41,20 +41,17 @@ service: port: 80 ingress: - enabled: false - className: "" - annotations: {} + enabled: true + className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}" + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "120" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local + tls: + secretName: openreplay-ssl resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/.helmignore b/scripts/helmcharts/openreplay/charts/ingress-nginx/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/CHANGELOG.md b/scripts/helmcharts/openreplay/charts/ingress-nginx/CHANGELOG.md new file mode 100644 index 000000000..f3f44c336 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/CHANGELOG.md @@ -0,0 +1,375 @@ +# Changelog + +This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). + +### 4.0.18 +"[8291](https://github.com/kubernetes/ingress-nginx/pull/8291) remove git tag env from cloud build" +"[8286](https://github.com/kubernetes/ingress-nginx/pull/8286) Fix OpenTelemetry sidecar image build" +"[8277](https://github.com/kubernetes/ingress-nginx/pull/8277) Add OpenSSF Best practices badge" +"[8273](https://github.com/kubernetes/ingress-nginx/pull/8273) Issue#8241" +"[8267](https://github.com/kubernetes/ingress-nginx/pull/8267) Add fsGroup value to admission-webhooks/job-patch charts" +"[8262](https://github.com/kubernetes/ingress-nginx/pull/8262) Updated confusing error" +"[8256](https://github.com/kubernetes/ingress-nginx/pull/8256) fix: deny locations with invalid auth-url annotation" +"[8253](https://github.com/kubernetes/ingress-nginx/pull/8253) Add a certificate info metric" +"[8236](https://github.com/kubernetes/ingress-nginx/pull/8236) webhook: remove useless code." +"[8227](https://github.com/kubernetes/ingress-nginx/pull/8227) Update libraries in webhook image" +"[8225](https://github.com/kubernetes/ingress-nginx/pull/8225) fix inconsistent-label-cardinality for prometheus metrics: nginx_ingress_controller_requests" +"[8221](https://github.com/kubernetes/ingress-nginx/pull/8221) Do not validate ingresses with unknown ingress class in admission webhook endpoint" +"[8210](https://github.com/kubernetes/ingress-nginx/pull/8210) Bump github.com/prometheus/client_golang from 1.11.0 to 1.12.1" +"[8209](https://github.com/kubernetes/ingress-nginx/pull/8209) Bump google.golang.org/grpc from 1.43.0 to 1.44.0" +"[8204](https://github.com/kubernetes/ingress-nginx/pull/8204) Add Artifact Hub lint" +"[8203](https://github.com/kubernetes/ingress-nginx/pull/8203) Fix Indentation of example and link to cert-manager tutorial" +"[8201](https://github.com/kubernetes/ingress-nginx/pull/8201) feat(metrics): add path and method labels to requests countera" +"[8199](https://github.com/kubernetes/ingress-nginx/pull/8199) use functional options to reduce number of methods creating an EchoDeployment" +"[8196](https://github.com/kubernetes/ingress-nginx/pull/8196) docs: fix inconsistent controller annotation" +"[8191](https://github.com/kubernetes/ingress-nginx/pull/8191) Using Go install for misspell" +"[8186](https://github.com/kubernetes/ingress-nginx/pull/8186) prometheus+grafana using servicemonitor" +"[8185](https://github.com/kubernetes/ingress-nginx/pull/8185) Append elements on match, instead of removing for cors-annotations" +"[8179](https://github.com/kubernetes/ingress-nginx/pull/8179) Bump github.com/opencontainers/runc from 1.0.3 to 1.1.0" +"[8173](https://github.com/kubernetes/ingress-nginx/pull/8173) Adding annotations to the controller service account" +"[8163](https://github.com/kubernetes/ingress-nginx/pull/8163) Update the $req_id placeholder description" +"[8162](https://github.com/kubernetes/ingress-nginx/pull/8162) Versioned static manifests" +"[8159](https://github.com/kubernetes/ingress-nginx/pull/8159) Adding some geoip variables and default values" +"[8155](https://github.com/kubernetes/ingress-nginx/pull/8155) #7271 feat: avoid-pdb-creation-when-default-backend-disabled-and-replicas-gt-1" +"[8151](https://github.com/kubernetes/ingress-nginx/pull/8151) Automatically generate helm docs" +"[8143](https://github.com/kubernetes/ingress-nginx/pull/8143) Allow to configure delay before controller exits" +"[8136](https://github.com/kubernetes/ingress-nginx/pull/8136) add ingressClass option to helm chart - back compatibility with ingress.class annotations" +"[8126](https://github.com/kubernetes/ingress-nginx/pull/8126) Example for JWT" + + +### 4.0.15 + +- [8120] https://github.com/kubernetes/ingress-nginx/pull/8120 Update go in runner and release v1.1.1 +- [8119] https://github.com/kubernetes/ingress-nginx/pull/8119 Update to go v1.17.6 +- [8118] https://github.com/kubernetes/ingress-nginx/pull/8118 Remove deprecated libraries, update other libs +- [8117] https://github.com/kubernetes/ingress-nginx/pull/8117 Fix codegen errors +- [8115] https://github.com/kubernetes/ingress-nginx/pull/8115 chart/ghaction: set the correct permission to have access to push a release +- [8098] https://github.com/kubernetes/ingress-nginx/pull/8098 generating SHA for CA only certs in backend_ssl.go + comparision of P… +- [8088] https://github.com/kubernetes/ingress-nginx/pull/8088 Fix Edit this page link to use main branch +- [8072] https://github.com/kubernetes/ingress-nginx/pull/8072 Expose GeoIP2 Continent code as variable +- [8061] https://github.com/kubernetes/ingress-nginx/pull/8061 docs(charts): using helm-docs for chart +- [8058] https://github.com/kubernetes/ingress-nginx/pull/8058 Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 +- [8054] https://github.com/kubernetes/ingress-nginx/pull/8054 Bump google.golang.org/grpc from 1.41.0 to 1.43.0 +- [8051] https://github.com/kubernetes/ingress-nginx/pull/8051 align bug report with feature request regarding kind documentation +- [8046] https://github.com/kubernetes/ingress-nginx/pull/8046 Report expired certificates (#8045) +- [8044] https://github.com/kubernetes/ingress-nginx/pull/8044 remove G109 check till gosec resolves issues +- [8042] https://github.com/kubernetes/ingress-nginx/pull/8042 docs_multiple_instances_one_cluster_ticket_7543 +- [8041] https://github.com/kubernetes/ingress-nginx/pull/8041 docs: fix typo'd executible name +- [8035] https://github.com/kubernetes/ingress-nginx/pull/8035 Comment busy owners +- [8029] https://github.com/kubernetes/ingress-nginx/pull/8029 Add stream-snippet as a ConfigMap and Annotation option +- [8023] https://github.com/kubernetes/ingress-nginx/pull/8023 fix nginx compilation flags +- [8021] https://github.com/kubernetes/ingress-nginx/pull/8021 Disable default modsecurity_rules_file if modsecurity-snippet is specified +- [8019] https://github.com/kubernetes/ingress-nginx/pull/8019 Revise main documentation page +- [8018] https://github.com/kubernetes/ingress-nginx/pull/8018 Preserve order of plugin invocation +- [8015] https://github.com/kubernetes/ingress-nginx/pull/8015 Add newline indenting to admission webhook annotations +- [8014] https://github.com/kubernetes/ingress-nginx/pull/8014 Add link to example error page manifest in docs +- [8009] https://github.com/kubernetes/ingress-nginx/pull/8009 Fix spelling in documentation and top-level files +- [8008] https://github.com/kubernetes/ingress-nginx/pull/8008 Add relabelings in controller-servicemonitor.yaml +- [8003] https://github.com/kubernetes/ingress-nginx/pull/8003 Minor improvements (formatting, consistency) in install guide +- [8001] https://github.com/kubernetes/ingress-nginx/pull/8001 fix: go-grpc Dockerfile +- [7999] https://github.com/kubernetes/ingress-nginx/pull/7999 images: use k8s-staging-test-infra/gcb-docker-gcloud +- [7996] https://github.com/kubernetes/ingress-nginx/pull/7996 doc: improvement +- [7983] https://github.com/kubernetes/ingress-nginx/pull/7983 Fix a couple of misspellings in the annotations documentation. +- [7979] https://github.com/kubernetes/ingress-nginx/pull/7979 allow set annotations for admission Jobs +- [7977] https://github.com/kubernetes/ingress-nginx/pull/7977 Add ssl_reject_handshake to defaul server +- [7975] https://github.com/kubernetes/ingress-nginx/pull/7975 add legacy version update v0.50.0 to main changelog +- [7972] https://github.com/kubernetes/ingress-nginx/pull/7972 updated service upstream definition + +### 4.0.14 + +- [8061] https://github.com/kubernetes/ingress-nginx/pull/8061 Using helm-docs to populate values table in README.md + +### 4.0.13 + +- [8008] https://github.com/kubernetes/ingress-nginx/pull/8008 Add relabelings in controller-servicemonitor.yaml + +### 4.0.12 + +- [7978] https://github.com/kubernetes/ingress-nginx/pull/7979 Support custom annotations in admissions Jobs + +### 4.0.11 + +- [7873] https://github.com/kubernetes/ingress-nginx/pull/7873 Makes the [appProtocol](https://kubernetes.io/docs/concepts/services-networking/_print/#application-protocol) field optional. + +### 4.0.10 + +- [7964] https://github.com/kubernetes/ingress-nginx/pull/7964 Update controller version to v1.1.0 + +### 4.0.9 + +- [6992] https://github.com/kubernetes/ingress-nginx/pull/6992 Add ability to specify labels for all resources + +### 4.0.7 + +- [7923] https://github.com/kubernetes/ingress-nginx/pull/7923 Release v1.0.5 of ingress-nginx +- [7806] https://github.com/kubernetes/ingress-nginx/pull/7806 Choice option for internal/external loadbalancer type service + +### 4.0.6 + +- [7804] https://github.com/kubernetes/ingress-nginx/pull/7804 Release v1.0.4 of ingress-nginx +- [7651] https://github.com/kubernetes/ingress-nginx/pull/7651 Support ipFamilyPolicy and ipFamilies fields in Helm Chart +- [7798] https://github.com/kubernetes/ingress-nginx/pull/7798 Exoscale: use HTTP Healthcheck mode +- [7793] https://github.com/kubernetes/ingress-nginx/pull/7793 Update kube-webhook-certgen to v1.1.1 + +### 4.0.5 + +- [7740] https://github.com/kubernetes/ingress-nginx/pull/7740 Release v1.0.3 of ingress-nginx + +### 4.0.3 + +- [7707] https://github.com/kubernetes/ingress-nginx/pull/7707 Release v1.0.2 of ingress-nginx + +### 4.0.2 + +- [7681] https://github.com/kubernetes/ingress-nginx/pull/7681 Release v1.0.1 of ingress-nginx + +### 4.0.1 + +- [7535] https://github.com/kubernetes/ingress-nginx/pull/7535 Release v1.0.0 ingress-nginx + +### 3.34.0 + +- [7256] https://github.com/kubernetes/ingress-nginx/pull/7256 Add namespace field in the namespace scoped resource templates + +### 3.33.0 + +- [7164] https://github.com/kubernetes/ingress-nginx/pull/7164 Update nginx to v1.20.1 + +### 3.32.0 + +- [7117] https://github.com/kubernetes/ingress-nginx/pull/7117 Add annotations for HPA + +### 3.31.0 + +- [7137] https://github.com/kubernetes/ingress-nginx/pull/7137 Add support for custom probes + +### 3.30.0 + +- [#7092](https://github.com/kubernetes/ingress-nginx/pull/7092) Removes the possibility of using localhost in ExternalNames as endpoints + +### 3.29.0 + +- [X] [#6945](https://github.com/kubernetes/ingress-nginx/pull/7020) Add option to specify job label for ServiceMonitor + +### 3.28.0 + +- [ ] [#6900](https://github.com/kubernetes/ingress-nginx/pull/6900) Support existing PSPs + +### 3.27.0 + +- Update ingress-nginx v0.45.0 + +### 3.26.0 + +- [X] [#6979](https://github.com/kubernetes/ingress-nginx/pull/6979) Changed servicePort value for metrics + +### 3.25.0 + +- [X] [#6957](https://github.com/kubernetes/ingress-nginx/pull/6957) Add ability to specify automountServiceAccountToken + +### 3.24.0 + +- [X] [#6908](https://github.com/kubernetes/ingress-nginx/pull/6908) Add volumes to default-backend deployment + +### 3.23.0 + +- Update ingress-nginx v0.44.0 + +### 3.22.0 + +- [X] [#6802](https://github.com/kubernetes/ingress-nginx/pull/6802) Add value for configuring a custom Diffie-Hellman parameters file +- [X] [#6815](https://github.com/kubernetes/ingress-nginx/pull/6815) Allow use of numeric namespaces in helm chart + +### 3.21.0 + +- [X] [#6783](https://github.com/kubernetes/ingress-nginx/pull/6783) Add custom annotations to ScaledObject +- [X] [#6761](https://github.com/kubernetes/ingress-nginx/pull/6761) Adding quotes in the serviceAccount name in Helm values +- [X] [#6767](https://github.com/kubernetes/ingress-nginx/pull/6767) Remove ClusterRole when scope option is enabled +- [X] [#6785](https://github.com/kubernetes/ingress-nginx/pull/6785) Update kube-webhook-certgen image to v1.5.1 + +### 3.20.1 + +- Do not create KEDA in case of DaemonSets. +- Fix KEDA v2 definition + +### 3.20.0 + +- [X] [#6730](https://github.com/kubernetes/ingress-nginx/pull/6730) Do not create HPA for defaultBackend if not enabled. + +### 3.19.0 + +- Update ingress-nginx v0.43.0 + +### 3.18.0 + +- [X] [#6688](https://github.com/kubernetes/ingress-nginx/pull/6688) Allow volume-type emptyDir in controller podsecuritypolicy +- [X] [#6691](https://github.com/kubernetes/ingress-nginx/pull/6691) Improve parsing of helm parameters + +### 3.17.0 + +- Update ingress-nginx v0.42.0 + +### 3.16.1 + +- Fix chart-releaser action + +### 3.16.0 + +- [X] [#6646](https://github.com/kubernetes/ingress-nginx/pull/6646) Added LoadBalancerIP value for internal service + +### 3.15.1 + +- Fix chart-releaser action + +### 3.15.0 + +- [X] [#6586](https://github.com/kubernetes/ingress-nginx/pull/6586) Fix 'maxmindLicenseKey' location in values.yaml + +### 3.14.0 + +- [X] [#6469](https://github.com/kubernetes/ingress-nginx/pull/6469) Allow custom service names for controller and backend + +### 3.13.0 + +- [X] [#6544](https://github.com/kubernetes/ingress-nginx/pull/6544) Fix default backend HPA name variable + +### 3.12.0 + +- [X] [#6514](https://github.com/kubernetes/ingress-nginx/pull/6514) Remove helm2 support and update docs + +### 3.11.1 + +- [X] [#6505](https://github.com/kubernetes/ingress-nginx/pull/6505) Reorder HPA resource list to work with GitOps tooling + +### 3.11.0 + +- Support Keda Autoscaling + +### 3.10.1 + +- Fix regression introduced in 0.41.0 with external authentication + +### 3.10.0 + +- Fix routing regression introduced in 0.41.0 with PathType Exact + +### 3.9.0 + +- [X] [#6423](https://github.com/kubernetes/ingress-nginx/pull/6423) Add Default backend HPA autoscaling + +### 3.8.0 + +- [X] [#6395](https://github.com/kubernetes/ingress-nginx/pull/6395) Update jettech/kube-webhook-certgen image +- [X] [#6377](https://github.com/kubernetes/ingress-nginx/pull/6377) Added loadBalancerSourceRanges for internal lbs +- [X] [#6356](https://github.com/kubernetes/ingress-nginx/pull/6356) Add securitycontext settings on defaultbackend +- [X] [#6401](https://github.com/kubernetes/ingress-nginx/pull/6401) Fix controller service annotations +- [X] [#6403](https://github.com/kubernetes/ingress-nginx/pull/6403) Initial helm chart changelog + +### 3.7.1 + +- [X] [#6326](https://github.com/kubernetes/ingress-nginx/pull/6326) Fix liveness and readiness probe path in daemonset chart + +### 3.7.0 + +- [X] [#6316](https://github.com/kubernetes/ingress-nginx/pull/6316) Numerals in podAnnotations in quotes [#6315](https://github.com/kubernetes/ingress-nginx/issues/6315) + +### 3.6.0 + +- [X] [#6305](https://github.com/kubernetes/ingress-nginx/pull/6305) Add default linux nodeSelector + +### 3.5.1 + +- [X] [#6299](https://github.com/kubernetes/ingress-nginx/pull/6299) Fix helm chart release + +### 3.5.0 + +- [X] [#6260](https://github.com/kubernetes/ingress-nginx/pull/6260) Allow Helm Chart to customize admission webhook's annotations, timeoutSeconds, namespaceSelector, objectSelector and cert files locations + +### 3.4.0 + +- [X] [#6268](https://github.com/kubernetes/ingress-nginx/pull/6268) Update to 0.40.2 in helm chart #6288 + +### 3.3.1 + +- [X] [#6259](https://github.com/kubernetes/ingress-nginx/pull/6259) Release helm chart +- [X] [#6258](https://github.com/kubernetes/ingress-nginx/pull/6258) Fix chart markdown link +- [X] [#6253](https://github.com/kubernetes/ingress-nginx/pull/6253) Release v0.40.0 + +### 3.3.1 + +- [X] [#6233](https://github.com/kubernetes/ingress-nginx/pull/6233) Add admission controller e2e test + +### 3.3.0 + +- [X] [#6203](https://github.com/kubernetes/ingress-nginx/pull/6203) Refactor parsing of key values +- [X] [#6162](https://github.com/kubernetes/ingress-nginx/pull/6162) Add helm chart options to expose metrics service as NodePort +- [X] [#6180](https://github.com/kubernetes/ingress-nginx/pull/6180) Fix helm chart admissionReviewVersions regression +- [X] [#6169](https://github.com/kubernetes/ingress-nginx/pull/6169) Fix Typo in example prometheus rules + +### 3.0.0 + +- [X] [#6167](https://github.com/kubernetes/ingress-nginx/pull/6167) Update chart requirements + +### 2.16.0 + +- [X] [#6154](https://github.com/kubernetes/ingress-nginx/pull/6154) add `topologySpreadConstraint` to controller + +### 2.15.0 + +- [X] [#6087](https://github.com/kubernetes/ingress-nginx/pull/6087) Adding parameter for externalTrafficPolicy in internal controller service spec + +### 2.14.0 + +- [X] [#6104](https://github.com/kubernetes/ingress-nginx/pull/6104) Misc fixes for nginx-ingress chart for better keel and prometheus-operator integration + +### 2.13.0 + +- [X] [#6093](https://github.com/kubernetes/ingress-nginx/pull/6093) Release v0.35.0 + +### 2.13.0 + +- [X] [#6093](https://github.com/kubernetes/ingress-nginx/pull/6093) Release v0.35.0 +- [X] [#6080](https://github.com/kubernetes/ingress-nginx/pull/6080) Switch images to k8s.gcr.io after Vanity Domain Flip + +### 2.12.1 + +- [X] [#6075](https://github.com/kubernetes/ingress-nginx/pull/6075) Sync helm chart affinity examples + +### 2.12.0 + +- [X] [#6039](https://github.com/kubernetes/ingress-nginx/pull/6039) Add configurable serviceMonitor metricRelabelling and targetLabels +- [X] [#6044](https://github.com/kubernetes/ingress-nginx/pull/6044) Fix YAML linting + +### 2.11.3 + +- [X] [#6038](https://github.com/kubernetes/ingress-nginx/pull/6038) Bump chart version PATCH + +### 2.11.2 + +- [X] [#5951](https://github.com/kubernetes/ingress-nginx/pull/5951) Bump chart patch version + +### 2.11.1 + +- [X] [#5900](https://github.com/kubernetes/ingress-nginx/pull/5900) Release helm chart for v0.34.1 + +### 2.11.0 + +- [X] [#5879](https://github.com/kubernetes/ingress-nginx/pull/5879) Update helm chart for v0.34.0 +- [X] [#5671](https://github.com/kubernetes/ingress-nginx/pull/5671) Make liveness probe more fault tolerant than readiness probe + +### 2.10.0 + +- [X] [#5843](https://github.com/kubernetes/ingress-nginx/pull/5843) Update jettech/kube-webhook-certgen image + +### 2.9.1 + +- [X] [#5823](https://github.com/kubernetes/ingress-nginx/pull/5823) Add quoting to sysctls because numeric values need to be presented as strings (#5823) + +### 2.9.0 + +- [X] [#5795](https://github.com/kubernetes/ingress-nginx/pull/5795) Use fully qualified images to avoid cri-o issues + + +### TODO + +Keep building the changelog using *git log charts* checking the tag diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/Chart.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/Chart.yaml new file mode 100644 index 000000000..6445231d5 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/Chart.yaml @@ -0,0 +1,25 @@ +annotations: + artifacthub.io/changes: | + - "#8253 Add a certificate info metric" + - "#8225 fix inconsistent-label-cardinality for prometheus metrics: nginx_ingress_controller_requests" + - "#8162 Versioned static manifests" + - "#8159 Adding some geoip variables and default values" + - "#8221 Do not validate ingresses with unknown ingress class in admission webhook endpoint" + artifacthub.io/prerelease: "false" +apiVersion: v2 +appVersion: 1.1.2 +description: Ingress controller for Kubernetes using NGINX as a reverse proxy and + load balancer +home: https://github.com/kubernetes/ingress-nginx +icon: https://upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Nginx_logo.svg/500px-Nginx_logo.svg.png +keywords: +- ingress +- nginx +kubeVersion: '>=1.19.0-0' +maintainers: +- name: ChiefAlexander +name: ingress-nginx +sources: +- https://github.com/kubernetes/ingress-nginx +type: application +version: 4.0.18 diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/OWNERS b/scripts/helmcharts/openreplay/charts/ingress-nginx/OWNERS new file mode 100644 index 000000000..6b7e049ca --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/OWNERS @@ -0,0 +1,10 @@ +# See the OWNERS docs: https://github.com/kubernetes/community/blob/master/contributors/guide/owners.md + +approvers: +- ingress-nginx-helm-maintainers + +reviewers: +- ingress-nginx-helm-reviewers + +labels: +- area/helm diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md b/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md new file mode 100644 index 000000000..ad641f726 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md @@ -0,0 +1,486 @@ +# ingress-nginx + +[ingress-nginx](https://github.com/kubernetes/ingress-nginx) Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer + +![Version: 4.0.18](https://img.shields.io/badge/Version-4.0.18-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.2](https://img.shields.io/badge/AppVersion-1.1.2-informational?style=flat-square) + +To use, add `ingressClassName: nginx` spec field or the `kubernetes.io/ingress.class: nginx` annotation to your Ingress resources. + +This chart bootstraps an ingress-nginx deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Chart version 3.x.x: Kubernetes v1.16+ +- Chart version 4.x.x and above: Kubernetes v1.19+ + +## Get Repo Info + +```console +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +``` + +## Install Chart + +**Important:** only helm3 is supported + +```console +helm install [RELEASE_NAME] ingress-nginx/ingress-nginx +``` + +The command deploys ingress-nginx on the Kubernetes cluster in the default configuration. + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading With Zero Downtime in Production + +By default the ingress-nginx controller has service interruptions whenever it's pods are restarted or redeployed. In order to fix that, see the excellent blog post by Lindsay Landry from Codecademy: [Kubernetes: Nginx and Zero Downtime in Production](https://medium.com/codecademy-engineering/kubernetes-nginx-and-zero-downtime-in-production-2c910c6a5ed8). + +### Migrating from stable/nginx-ingress + +There are two main ways to migrate a release from `stable/nginx-ingress` to `ingress-nginx/ingress-nginx` chart: + +1. For Nginx Ingress controllers used for non-critical services, the easiest method is to [uninstall](#uninstall-chart) the old release and [install](#install-chart) the new one +1. For critical services in production that require zero-downtime, you will want to: + 1. [Install](#install-chart) a second Ingress controller + 1. Redirect your DNS traffic from the old controller to the new controller + 1. Log traffic from both controllers during this changeover + 1. [Uninstall](#uninstall-chart) the old controller once traffic has fully drained from it + 1. For details on all of these steps see [Upgrading With Zero Downtime in Production](#upgrading-with-zero-downtime-in-production) + +Note that there are some different and upgraded configurations between the two charts, described by Rimas Mocevicius from JFrog in the "Upgrading to ingress-nginx Helm chart" section of [Migrating from Helm chart nginx-ingress to ingress-nginx](https://rimusz.net/migrating-to-ingress-nginx). As the `ingress-nginx/ingress-nginx` chart continues to update, you will want to check current differences by running [helm configuration](#configuration) commands on both charts. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values ingress-nginx/ingress-nginx +``` + +### PodDisruptionBudget + +Note that the PodDisruptionBudget resource will only be defined if the replicaCount is greater than one, +else it would make it impossible to evacuate a node. See [gh issue #7127](https://github.com/helm/charts/issues/7127) for more info. + +### Prometheus Metrics + +The Nginx ingress controller can export Prometheus metrics, by setting `controller.metrics.enabled` to `true`. + +You can add Prometheus annotations to the metrics service using `controller.metrics.service.annotations`. +Alternatively, if you use the Prometheus Operator, you can enable ServiceMonitor creation using `controller.metrics.serviceMonitor.enabled`. And set `controller.metrics.serviceMonitor.additionalLabels.release="prometheus"`. "release=prometheus" should match the label configured in the prometheus servicemonitor ( see `kubectl get servicemonitor prometheus-kube-prom-prometheus -oyaml -n prometheus`) + +### ingress-nginx nginx\_status page/stats server + +Previous versions of this chart had a `controller.stats.*` configuration block, which is now obsolete due to the following changes in nginx ingress controller: + +- In [0.16.1](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0161), the vts (virtual host traffic status) dashboard was removed +- In [0.23.0](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230), the status page at port 18080 is now a unix socket webserver only available at localhost. + You can use `curl --unix-socket /tmp/nginx-status-server.sock http://localhost/nginx_status` inside the controller container to access it locally, or use the snippet from [nginx-ingress changelog](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230) to re-enable the http server + +### ExternalDNS Service Configuration + +Add an [ExternalDNS](https://github.com/kubernetes-incubator/external-dns) annotation to the LoadBalancer service: + +```yaml +controller: + service: + annotations: + external-dns.alpha.kubernetes.io/hostname: kubernetes-example.com. +``` + +### AWS L7 ELB with SSL Termination + +Annotate the controller as shown in the [nginx-ingress l7 patch](https://github.com/kubernetes/ingress-nginx/blob/main/deploy/aws/l7/service-l7.yaml): + +```yaml +controller: + service: + targetPorts: + http: http + https: http + annotations: + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:XX-XXXX-X:XXXXXXXXX:certificate/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http" + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https" + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '3600' +``` + +### AWS route53-mapper + +To configure the LoadBalancer service with the [route53-mapper addon](https://github.com/kubernetes/kops/tree/master/addons/route53-mapper), add the `domainName` annotation and `dns` label: + +```yaml +controller: + service: + labels: + dns: "route53" + annotations: + domainName: "kubernetes-example.com" +``` + +### Additional Internal Load Balancer + +This setup is useful when you need both external and internal load balancers but don't want to have multiple ingress controllers and multiple ingress objects per application. + +By default, the ingress object will point to the external load balancer address, but if correctly configured, you can make use of the internal one if the URL you are looking up resolves to the internal load balancer's URL. + +You'll need to set both the following values: + +`controller.service.internal.enabled` +`controller.service.internal.annotations` + +If one of them is missing the internal load balancer will not be deployed. Example you may have `controller.service.internal.enabled=true` but no annotations set, in this case no action will be taken. + +`controller.service.internal.annotations` varies with the cloud service you're using. + +Example for AWS: + +```yaml +controller: + service: + internal: + enabled: true + annotations: + # Create internal ELB + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +Example for GCE: + +```yaml +controller: + service: + internal: + enabled: true + annotations: + # Create internal LB. More informations: https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing + # For GKE versions 1.17 and later + networking.gke.io/load-balancer-type: "Internal" + # For earlier versions + # cloud.google.com/load-balancer-type: "Internal" + + # Any other annotation can be declared here. +``` + +Example for Azure: + +```yaml +controller: + service: + annotations: + # Create internal LB + service.beta.kubernetes.io/azure-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +Example for Oracle Cloud Infrastructure: + +```yaml +controller: + service: + annotations: + # Create internal LB + service.beta.kubernetes.io/oci-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +An use case for this scenario is having a split-view DNS setup where the public zone CNAME records point to the external balancer URL while the private zone CNAME records point to the internal balancer URL. This way, you only need one ingress kubernetes object. + +Optionally you can set `controller.service.loadBalancerIP` if you need a static IP for the resulting `LoadBalancer`. + +### Ingress Admission Webhooks + +With nginx-ingress-controller version 0.25+, the nginx ingress controller pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent bad ingress from being added to the cluster. +**This feature is enabled by default since 0.31.0.** + +With nginx-ingress-controller in 0.25.* work only with kubernetes 1.14+, 0.26 fix [this issue](https://github.com/kubernetes/ingress-nginx/pull/4521) + +### Helm Error When Upgrading: spec.clusterIP: Invalid value: "" + +If you are upgrading this chart from a version between 0.31.0 and 1.2.2 then you may get an error like this: + +```console +Error: UPGRADE FAILED: Service "?????-controller" is invalid: spec.clusterIP: Invalid value: "": field is immutable +``` + +Detail of how and why are in [this issue](https://github.com/helm/charts/pull/13646) but to resolve this you can set `xxxx.service.omitClusterIP` to `true` where `xxxx` is the service referenced in the error. + +As of version `1.26.0` of this chart, by simply not providing any clusterIP value, `invalid: spec.clusterIP: Invalid value: "": field is immutable` will no longer occur since `clusterIP: ""` will not be rendered. + +## Requirements + +Kubernetes: `>=1.19.0-0` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| commonLabels | object | `{}` | | +| controller.addHeaders | object | `{}` | Will add custom headers before sending response traffic to the client according to: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#add-headers | +| controller.admissionWebhooks.annotations | object | `{}` | | +| controller.admissionWebhooks.certificate | string | `"/usr/local/certificates/cert"` | | +| controller.admissionWebhooks.createSecretJob.resources | object | `{}` | | +| controller.admissionWebhooks.enabled | bool | `true` | | +| controller.admissionWebhooks.existingPsp | string | `""` | Use an existing PSP instead of creating one | +| controller.admissionWebhooks.failurePolicy | string | `"Fail"` | | +| controller.admissionWebhooks.key | string | `"/usr/local/certificates/key"` | | +| controller.admissionWebhooks.labels | object | `{}` | Labels to be added to admission webhooks | +| controller.admissionWebhooks.namespaceSelector | object | `{}` | | +| controller.admissionWebhooks.objectSelector | object | `{}` | | +| controller.admissionWebhooks.patch.enabled | bool | `true` | | +| controller.admissionWebhooks.patch.fsGroup | int | `2000` | | +| controller.admissionWebhooks.patch.image.digest | string | `"sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660"` | | +| controller.admissionWebhooks.patch.image.image | string | `"ingress-nginx/kube-webhook-certgen"` | | +| controller.admissionWebhooks.patch.image.pullPolicy | string | `"IfNotPresent"` | | +| controller.admissionWebhooks.patch.image.registry | string | `"k8s.gcr.io"` | | +| controller.admissionWebhooks.patch.image.tag | string | `"v1.1.1"` | | +| controller.admissionWebhooks.patch.labels | object | `{}` | Labels to be added to patch job resources | +| controller.admissionWebhooks.patch.nodeSelector."kubernetes.io/os" | string | `"linux"` | | +| controller.admissionWebhooks.patch.podAnnotations | object | `{}` | | +| controller.admissionWebhooks.patch.priorityClassName | string | `""` | Provide a priority class name to the webhook patching job | +| controller.admissionWebhooks.patch.runAsUser | int | `2000` | | +| controller.admissionWebhooks.patch.tolerations | list | `[]` | | +| controller.admissionWebhooks.patchWebhookJob.resources | object | `{}` | | +| controller.admissionWebhooks.port | int | `8443` | | +| controller.admissionWebhooks.service.annotations | object | `{}` | | +| controller.admissionWebhooks.service.externalIPs | list | `[]` | | +| controller.admissionWebhooks.service.loadBalancerSourceRanges | list | `[]` | | +| controller.admissionWebhooks.service.servicePort | int | `443` | | +| controller.admissionWebhooks.service.type | string | `"ClusterIP"` | | +| controller.affinity | object | `{}` | Affinity and anti-affinity rules for server scheduling to nodes | +| controller.allowSnippetAnnotations | bool | `true` | This configuration defines if Ingress Controller should allow users to set their own *-snippet annotations, otherwise this is forbidden / dropped when users add those annotations. Global snippets in ConfigMap are still respected | +| controller.annotations | object | `{}` | Annotations to be added to the controller Deployment or DaemonSet | +| controller.autoscaling.behavior | object | `{}` | | +| controller.autoscaling.enabled | bool | `false` | | +| controller.autoscaling.maxReplicas | int | `11` | | +| controller.autoscaling.minReplicas | int | `1` | | +| controller.autoscaling.targetCPUUtilizationPercentage | int | `50` | | +| controller.autoscaling.targetMemoryUtilizationPercentage | int | `50` | | +| controller.autoscalingTemplate | list | `[]` | | +| controller.config | object | `{}` | Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/ | +| controller.configAnnotations | object | `{}` | Annotations to be added to the controller config configuration configmap. | +| controller.configMapNamespace | string | `""` | Allows customization of the configmap / nginx-configmap namespace; defaults to $(POD_NAMESPACE) | +| controller.containerName | string | `"controller"` | Configures the controller container name | +| controller.containerPort | object | `{"http":80,"https":443}` | Configures the ports that the nginx-controller listens on | +| controller.customTemplate.configMapKey | string | `""` | | +| controller.customTemplate.configMapName | string | `""` | | +| controller.dnsConfig | object | `{}` | Optionally customize the pod dnsConfig. | +| controller.dnsPolicy | string | `"ClusterFirst"` | Optionally change this to ClusterFirstWithHostNet in case you have 'hostNetwork: true'. By default, while using host network, name resolution uses the host's DNS. If you wish nginx-controller to keep resolving names inside the k8s network, use ClusterFirstWithHostNet. | +| controller.electionID | string | `"ingress-controller-leader"` | Election ID to use for status update | +| controller.enableMimalloc | bool | `true` | Enable mimalloc as a drop-in replacement for malloc. | +| controller.existingPsp | string | `""` | Use an existing PSP instead of creating one | +| controller.extraArgs | object | `{}` | Additional command line arguments to pass to nginx-ingress-controller E.g. to specify the default SSL certificate you can use | +| controller.extraContainers | list | `[]` | Additional containers to be added to the controller pod. See https://github.com/lemonldap-ng-controller/lemonldap-ng-controller as example. | +| controller.extraEnvs | list | `[]` | Additional environment variables to set | +| controller.extraInitContainers | list | `[]` | Containers, which are run before the app containers are started. | +| controller.extraModules | list | `[]` | | +| controller.extraVolumeMounts | list | `[]` | Additional volumeMounts to the controller main container. | +| controller.extraVolumes | list | `[]` | Additional volumes to the controller pod. | +| controller.healthCheckHost | string | `""` | Address to bind the health check endpoint. It is better to set this option to the internal node address if the ingress nginx controller is running in the `hostNetwork: true` mode. | +| controller.healthCheckPath | string | `"/healthz"` | Path of the health check endpoint. All requests received on the port defined by the healthz-port parameter are forwarded internally to this path. | +| controller.hostNetwork | bool | `false` | Required for use with CNI based kubernetes installations (such as ones set up by kubeadm), since CNI and hostport don't mix yet. Can be deprecated once https://github.com/kubernetes/kubernetes/issues/23920 is merged | +| controller.hostPort.enabled | bool | `false` | Enable 'hostPort' or not | +| controller.hostPort.ports.http | int | `80` | 'hostPort' http port | +| controller.hostPort.ports.https | int | `443` | 'hostPort' https port | +| controller.hostname | object | `{}` | Optionally customize the pod hostname. | +| controller.image.allowPrivilegeEscalation | bool | `true` | | +| controller.image.digest | string | `"sha256:28b11ce69e57843de44e3db6413e98d09de0f6688e33d4bd384002a44f78405c"` | | +| controller.image.image | string | `"ingress-nginx/controller"` | | +| controller.image.pullPolicy | string | `"IfNotPresent"` | | +| controller.image.registry | string | `"k8s.gcr.io"` | | +| controller.image.runAsUser | int | `101` | | +| controller.image.tag | string | `"v1.1.2"` | | +| controller.ingressClass | string | `"nginx"` | For backwards compatibility with ingress.class annotation, use ingressClass. Algorithm is as follows, first ingressClassName is considered, if not present, controller looks for ingress.class annotation | +| controller.ingressClassByName | bool | `false` | Process IngressClass per name (additionally as per spec.controller). | +| controller.ingressClassResource.controllerValue | string | `"k8s.io/ingress-nginx"` | Controller-value of the controller that is processing this ingressClass | +| controller.ingressClassResource.default | bool | `false` | Is this the default ingressClass for the cluster | +| controller.ingressClassResource.enabled | bool | `true` | Is this ingressClass enabled or not | +| controller.ingressClassResource.name | string | `"nginx"` | Name of the ingressClass | +| controller.ingressClassResource.parameters | object | `{}` | Parameters is a link to a custom resource containing additional configuration for the controller. This is optional if the controller does not require extra parameters. | +| controller.keda.apiVersion | string | `"keda.sh/v1alpha1"` | | +| controller.keda.behavior | object | `{}` | | +| controller.keda.cooldownPeriod | int | `300` | | +| controller.keda.enabled | bool | `false` | | +| controller.keda.maxReplicas | int | `11` | | +| controller.keda.minReplicas | int | `1` | | +| controller.keda.pollingInterval | int | `30` | | +| controller.keda.restoreToOriginalReplicaCount | bool | `false` | | +| controller.keda.scaledObject.annotations | object | `{}` | | +| controller.keda.triggers | list | `[]` | | +| controller.kind | string | `"Deployment"` | Use a `DaemonSet` or `Deployment` | +| controller.labels | object | `{}` | Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels | +| controller.lifecycle | object | `{"preStop":{"exec":{"command":["/wait-shutdown"]}}}` | Improve connection draining when ingress controller pod is deleted using a lifecycle hook: With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds to 300, allowing the draining of connections up to five minutes. If the active connections end before that, the pod will terminate gracefully at that time. To effectively take advantage of this feature, the Configmap feature worker-shutdown-timeout new value is 240s instead of 10s. | +| controller.livenessProbe.failureThreshold | int | `5` | | +| controller.livenessProbe.httpGet.path | string | `"/healthz"` | | +| controller.livenessProbe.httpGet.port | int | `10254` | | +| controller.livenessProbe.httpGet.scheme | string | `"HTTP"` | | +| controller.livenessProbe.initialDelaySeconds | int | `10` | | +| controller.livenessProbe.periodSeconds | int | `10` | | +| controller.livenessProbe.successThreshold | int | `1` | | +| controller.livenessProbe.timeoutSeconds | int | `1` | | +| controller.maxmindLicenseKey | string | `""` | Maxmind license key to download GeoLite2 Databases. | +| controller.metrics.enabled | bool | `false` | | +| controller.metrics.port | int | `10254` | | +| controller.metrics.prometheusRule.additionalLabels | object | `{}` | | +| controller.metrics.prometheusRule.enabled | bool | `false` | | +| controller.metrics.prometheusRule.rules | list | `[]` | | +| controller.metrics.service.annotations | object | `{}` | | +| controller.metrics.service.externalIPs | list | `[]` | List of IP addresses at which the stats-exporter service is available | +| controller.metrics.service.loadBalancerSourceRanges | list | `[]` | | +| controller.metrics.service.servicePort | int | `10254` | | +| controller.metrics.service.type | string | `"ClusterIP"` | | +| controller.metrics.serviceMonitor.additionalLabels | object | `{}` | | +| controller.metrics.serviceMonitor.enabled | bool | `false` | | +| controller.metrics.serviceMonitor.metricRelabelings | list | `[]` | | +| controller.metrics.serviceMonitor.namespace | string | `""` | | +| controller.metrics.serviceMonitor.namespaceSelector | object | `{}` | | +| controller.metrics.serviceMonitor.relabelings | list | `[]` | | +| controller.metrics.serviceMonitor.scrapeInterval | string | `"30s"` | | +| controller.metrics.serviceMonitor.targetLabels | list | `[]` | | +| controller.minAvailable | int | `1` | | +| controller.minReadySeconds | int | `0` | `minReadySeconds` to avoid killing pods before we are ready | +| controller.name | string | `"controller"` | | +| controller.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for controller pod assignment | +| controller.podAnnotations | object | `{}` | Annotations to be added to controller pods | +| controller.podLabels | object | `{}` | Labels to add to the pod container metadata | +| controller.podSecurityContext | object | `{}` | Security Context policies for controller pods | +| controller.priorityClassName | string | `""` | | +| controller.proxySetHeaders | object | `{}` | Will add custom headers before sending traffic to backends according to https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/custom-headers | +| controller.publishService | object | `{"enabled":true,"pathOverride":""}` | Allows customization of the source of the IP address or FQDN to report in the ingress status field. By default, it reads the information provided by the service. If disable, the status field reports the IP address of the node or nodes where an ingress controller pod is running. | +| controller.publishService.enabled | bool | `true` | Enable 'publishService' or not | +| controller.publishService.pathOverride | string | `""` | Allows overriding of the publish service to bind to Must be / | +| controller.readinessProbe.failureThreshold | int | `3` | | +| controller.readinessProbe.httpGet.path | string | `"/healthz"` | | +| controller.readinessProbe.httpGet.port | int | `10254` | | +| controller.readinessProbe.httpGet.scheme | string | `"HTTP"` | | +| controller.readinessProbe.initialDelaySeconds | int | `10` | | +| controller.readinessProbe.periodSeconds | int | `10` | | +| controller.readinessProbe.successThreshold | int | `1` | | +| controller.readinessProbe.timeoutSeconds | int | `1` | | +| controller.replicaCount | int | `1` | | +| controller.reportNodeInternalIp | bool | `false` | Bare-metal considerations via the host network https://kubernetes.github.io/ingress-nginx/deploy/baremetal/#via-the-host-network Ingress status was blank because there is no Service exposing the NGINX Ingress controller in a configuration using the host network, the default --publish-service flag used in standard cloud setups does not apply | +| controller.resources.requests.cpu | string | `"100m"` | | +| controller.resources.requests.memory | string | `"90Mi"` | | +| controller.scope.enabled | bool | `false` | Enable 'scope' or not | +| controller.scope.namespace | string | `""` | Namespace to limit the controller to; defaults to $(POD_NAMESPACE) | +| controller.scope.namespaceSelector | string | `""` | When scope.enabled == false, instead of watching all namespaces, we watching namespaces whose labels only match with namespaceSelector. Format like foo=bar. Defaults to empty, means watching all namespaces. | +| controller.service.annotations | object | `{}` | | +| controller.service.appProtocol | bool | `true` | If enabled is adding an appProtocol option for Kubernetes service. An appProtocol field replacing annotations that were using for setting a backend protocol. Here is an example for AWS: service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http It allows choosing the protocol for each backend specified in the Kubernetes service. See the following GitHub issue for more details about the purpose: https://github.com/kubernetes/kubernetes/issues/40244 Will be ignored for Kubernetes versions older than 1.20 | +| controller.service.enableHttp | bool | `true` | | +| controller.service.enableHttps | bool | `true` | | +| controller.service.enabled | bool | `true` | | +| controller.service.external.enabled | bool | `true` | | +| controller.service.externalIPs | list | `[]` | List of IP addresses at which the controller services are available | +| controller.service.internal.annotations | object | `{}` | Annotations are mandatory for the load balancer to come up. Varies with the cloud service. | +| controller.service.internal.enabled | bool | `false` | Enables an additional internal load balancer (besides the external one). | +| controller.service.internal.loadBalancerSourceRanges | list | `[]` | Restrict access For LoadBalancer service. Defaults to 0.0.0.0/0. | +| controller.service.ipFamilies | list | `["IPv4"]` | List of IP families (e.g. IPv4, IPv6) assigned to the service. This field is usually assigned automatically based on cluster configuration and the ipFamilyPolicy field. | +| controller.service.ipFamilyPolicy | string | `"SingleStack"` | Represents the dual-stack-ness requested or required by this Service. Possible values are SingleStack, PreferDualStack or RequireDualStack. The ipFamilies and clusterIPs fields depend on the value of this field. | +| controller.service.labels | object | `{}` | | +| controller.service.loadBalancerSourceRanges | list | `[]` | | +| controller.service.nodePorts.http | string | `""` | | +| controller.service.nodePorts.https | string | `""` | | +| controller.service.nodePorts.tcp | object | `{}` | | +| controller.service.nodePorts.udp | object | `{}` | | +| controller.service.ports.http | int | `80` | | +| controller.service.ports.https | int | `443` | | +| controller.service.targetPorts.http | string | `"http"` | | +| controller.service.targetPorts.https | string | `"https"` | | +| controller.service.type | string | `"LoadBalancer"` | | +| controller.sysctls | object | `{}` | See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for notes on enabling and using sysctls | +| controller.tcp.annotations | object | `{}` | Annotations to be added to the tcp config configmap | +| controller.tcp.configMapNamespace | string | `""` | Allows customization of the tcp-services-configmap; defaults to $(POD_NAMESPACE) | +| controller.terminationGracePeriodSeconds | int | `300` | `terminationGracePeriodSeconds` to avoid killing pods before we are ready | +| controller.tolerations | list | `[]` | Node tolerations for server scheduling to nodes with taints | +| controller.topologySpreadConstraints | list | `[]` | Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. | +| controller.udp.annotations | object | `{}` | Annotations to be added to the udp config configmap | +| controller.udp.configMapNamespace | string | `""` | Allows customization of the udp-services-configmap; defaults to $(POD_NAMESPACE) | +| controller.updateStrategy | object | `{}` | The update strategy to apply to the Deployment or DaemonSet | +| controller.watchIngressWithoutClass | bool | `false` | Process Ingress objects without ingressClass annotation/ingressClassName field Overrides value for --watch-ingress-without-class flag of the controller binary Defaults to false | +| defaultBackend.affinity | object | `{}` | | +| defaultBackend.autoscaling.annotations | object | `{}` | | +| defaultBackend.autoscaling.enabled | bool | `false` | | +| defaultBackend.autoscaling.maxReplicas | int | `2` | | +| defaultBackend.autoscaling.minReplicas | int | `1` | | +| defaultBackend.autoscaling.targetCPUUtilizationPercentage | int | `50` | | +| defaultBackend.autoscaling.targetMemoryUtilizationPercentage | int | `50` | | +| defaultBackend.containerSecurityContext | object | `{}` | Security Context policies for controller main container. See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for notes on enabling and using sysctls | +| defaultBackend.enabled | bool | `false` | | +| defaultBackend.existingPsp | string | `""` | Use an existing PSP instead of creating one | +| defaultBackend.extraArgs | object | `{}` | | +| defaultBackend.extraEnvs | list | `[]` | Additional environment variables to set for defaultBackend pods | +| defaultBackend.extraVolumeMounts | list | `[]` | | +| defaultBackend.extraVolumes | list | `[]` | | +| defaultBackend.image.allowPrivilegeEscalation | bool | `false` | | +| defaultBackend.image.image | string | `"defaultbackend-amd64"` | | +| defaultBackend.image.pullPolicy | string | `"IfNotPresent"` | | +| defaultBackend.image.readOnlyRootFilesystem | bool | `true` | | +| defaultBackend.image.registry | string | `"k8s.gcr.io"` | | +| defaultBackend.image.runAsNonRoot | bool | `true` | | +| defaultBackend.image.runAsUser | int | `65534` | | +| defaultBackend.image.tag | string | `"1.5"` | | +| defaultBackend.labels | object | `{}` | Labels to be added to the default backend resources | +| defaultBackend.livenessProbe.failureThreshold | int | `3` | | +| defaultBackend.livenessProbe.initialDelaySeconds | int | `30` | | +| defaultBackend.livenessProbe.periodSeconds | int | `10` | | +| defaultBackend.livenessProbe.successThreshold | int | `1` | | +| defaultBackend.livenessProbe.timeoutSeconds | int | `5` | | +| defaultBackend.minAvailable | int | `1` | | +| defaultBackend.name | string | `"defaultbackend"` | | +| defaultBackend.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for default backend pod assignment | +| defaultBackend.podAnnotations | object | `{}` | Annotations to be added to default backend pods | +| defaultBackend.podLabels | object | `{}` | Labels to add to the pod container metadata | +| defaultBackend.podSecurityContext | object | `{}` | Security Context policies for controller pods See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for notes on enabling and using sysctls | +| defaultBackend.port | int | `8080` | | +| defaultBackend.priorityClassName | string | `""` | | +| defaultBackend.readinessProbe.failureThreshold | int | `6` | | +| defaultBackend.readinessProbe.initialDelaySeconds | int | `0` | | +| defaultBackend.readinessProbe.periodSeconds | int | `5` | | +| defaultBackend.readinessProbe.successThreshold | int | `1` | | +| defaultBackend.readinessProbe.timeoutSeconds | int | `5` | | +| defaultBackend.replicaCount | int | `1` | | +| defaultBackend.resources | object | `{}` | | +| defaultBackend.service.annotations | object | `{}` | | +| defaultBackend.service.externalIPs | list | `[]` | List of IP addresses at which the default backend service is available | +| defaultBackend.service.loadBalancerSourceRanges | list | `[]` | | +| defaultBackend.service.servicePort | int | `80` | | +| defaultBackend.service.type | string | `"ClusterIP"` | | +| defaultBackend.serviceAccount.automountServiceAccountToken | bool | `true` | | +| defaultBackend.serviceAccount.create | bool | `true` | | +| defaultBackend.serviceAccount.name | string | `""` | | +| defaultBackend.tolerations | list | `[]` | Node tolerations for server scheduling to nodes with taints | +| dhParam | string | `nil` | A base64-encoded Diffie-Hellman parameter. This can be generated with: `openssl dhparam 4096 2> /dev/null | base64` | +| imagePullSecrets | list | `[]` | Optional array of imagePullSecrets containing private registry credentials | +| podSecurityPolicy.enabled | bool | `false` | | +| rbac.create | bool | `true` | | +| rbac.scope | bool | `false` | | +| revisionHistoryLimit | int | `10` | Rollback limit | +| serviceAccount.annotations | object | `{}` | Annotations for the controller service account | +| serviceAccount.automountServiceAccountToken | bool | `true` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| tcp | object | `{}` | TCP service key:value pairs | +| udp | object | `{}` | UDP service key:value pairs | + diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md.gotmpl b/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md.gotmpl new file mode 100644 index 000000000..5cd9e59e1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/README.md.gotmpl @@ -0,0 +1,235 @@ +{{ template "chart.header" . }} +[ingress-nginx](https://github.com/kubernetes/ingress-nginx) Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +To use, add `ingressClassName: nginx` spec field or the `kubernetes.io/ingress.class: nginx` annotation to your Ingress resources. + +This chart bootstraps an ingress-nginx deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Chart version 3.x.x: Kubernetes v1.16+ +- Chart version 4.x.x and above: Kubernetes v1.19+ + +## Get Repo Info + +```console +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +``` + +## Install Chart + +**Important:** only helm3 is supported + +```console +helm install [RELEASE_NAME] ingress-nginx/ingress-nginx +``` + +The command deploys ingress-nginx on the Kubernetes cluster in the default configuration. + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading With Zero Downtime in Production + +By default the ingress-nginx controller has service interruptions whenever it's pods are restarted or redeployed. In order to fix that, see the excellent blog post by Lindsay Landry from Codecademy: [Kubernetes: Nginx and Zero Downtime in Production](https://medium.com/codecademy-engineering/kubernetes-nginx-and-zero-downtime-in-production-2c910c6a5ed8). + +### Migrating from stable/nginx-ingress + +There are two main ways to migrate a release from `stable/nginx-ingress` to `ingress-nginx/ingress-nginx` chart: + +1. For Nginx Ingress controllers used for non-critical services, the easiest method is to [uninstall](#uninstall-chart) the old release and [install](#install-chart) the new one +1. For critical services in production that require zero-downtime, you will want to: + 1. [Install](#install-chart) a second Ingress controller + 1. Redirect your DNS traffic from the old controller to the new controller + 1. Log traffic from both controllers during this changeover + 1. [Uninstall](#uninstall-chart) the old controller once traffic has fully drained from it + 1. For details on all of these steps see [Upgrading With Zero Downtime in Production](#upgrading-with-zero-downtime-in-production) + +Note that there are some different and upgraded configurations between the two charts, described by Rimas Mocevicius from JFrog in the "Upgrading to ingress-nginx Helm chart" section of [Migrating from Helm chart nginx-ingress to ingress-nginx](https://rimusz.net/migrating-to-ingress-nginx). As the `ingress-nginx/ingress-nginx` chart continues to update, you will want to check current differences by running [helm configuration](#configuration) commands on both charts. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values ingress-nginx/ingress-nginx +``` + +### PodDisruptionBudget + +Note that the PodDisruptionBudget resource will only be defined if the replicaCount is greater than one, +else it would make it impossible to evacuate a node. See [gh issue #7127](https://github.com/helm/charts/issues/7127) for more info. + +### Prometheus Metrics + +The Nginx ingress controller can export Prometheus metrics, by setting `controller.metrics.enabled` to `true`. + +You can add Prometheus annotations to the metrics service using `controller.metrics.service.annotations`. +Alternatively, if you use the Prometheus Operator, you can enable ServiceMonitor creation using `controller.metrics.serviceMonitor.enabled`. And set `controller.metrics.serviceMonitor.additionalLabels.release="prometheus"`. "release=prometheus" should match the label configured in the prometheus servicemonitor ( see `kubectl get servicemonitor prometheus-kube-prom-prometheus -oyaml -n prometheus`) + +### ingress-nginx nginx\_status page/stats server + +Previous versions of this chart had a `controller.stats.*` configuration block, which is now obsolete due to the following changes in nginx ingress controller: + +- In [0.16.1](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0161), the vts (virtual host traffic status) dashboard was removed +- In [0.23.0](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230), the status page at port 18080 is now a unix socket webserver only available at localhost. + You can use `curl --unix-socket /tmp/nginx-status-server.sock http://localhost/nginx_status` inside the controller container to access it locally, or use the snippet from [nginx-ingress changelog](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230) to re-enable the http server + +### ExternalDNS Service Configuration + +Add an [ExternalDNS](https://github.com/kubernetes-incubator/external-dns) annotation to the LoadBalancer service: + +```yaml +controller: + service: + annotations: + external-dns.alpha.kubernetes.io/hostname: kubernetes-example.com. +``` + +### AWS L7 ELB with SSL Termination + +Annotate the controller as shown in the [nginx-ingress l7 patch](https://github.com/kubernetes/ingress-nginx/blob/main/deploy/aws/l7/service-l7.yaml): + +```yaml +controller: + service: + targetPorts: + http: http + https: http + annotations: + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:XX-XXXX-X:XXXXXXXXX:certificate/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http" + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https" + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '3600' +``` + +### AWS route53-mapper + +To configure the LoadBalancer service with the [route53-mapper addon](https://github.com/kubernetes/kops/tree/master/addons/route53-mapper), add the `domainName` annotation and `dns` label: + +```yaml +controller: + service: + labels: + dns: "route53" + annotations: + domainName: "kubernetes-example.com" +``` + +### Additional Internal Load Balancer + +This setup is useful when you need both external and internal load balancers but don't want to have multiple ingress controllers and multiple ingress objects per application. + +By default, the ingress object will point to the external load balancer address, but if correctly configured, you can make use of the internal one if the URL you are looking up resolves to the internal load balancer's URL. + +You'll need to set both the following values: + +`controller.service.internal.enabled` +`controller.service.internal.annotations` + +If one of them is missing the internal load balancer will not be deployed. Example you may have `controller.service.internal.enabled=true` but no annotations set, in this case no action will be taken. + +`controller.service.internal.annotations` varies with the cloud service you're using. + +Example for AWS: + +```yaml +controller: + service: + internal: + enabled: true + annotations: + # Create internal ELB + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +Example for GCE: + +```yaml +controller: + service: + internal: + enabled: true + annotations: + # Create internal LB. More informations: https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing + # For GKE versions 1.17 and later + networking.gke.io/load-balancer-type: "Internal" + # For earlier versions + # cloud.google.com/load-balancer-type: "Internal" + + # Any other annotation can be declared here. +``` + +Example for Azure: + +```yaml +controller: + service: + annotations: + # Create internal LB + service.beta.kubernetes.io/azure-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +Example for Oracle Cloud Infrastructure: + +```yaml +controller: + service: + annotations: + # Create internal LB + service.beta.kubernetes.io/oci-load-balancer-internal: "true" + # Any other annotation can be declared here. +``` + +An use case for this scenario is having a split-view DNS setup where the public zone CNAME records point to the external balancer URL while the private zone CNAME records point to the internal balancer URL. This way, you only need one ingress kubernetes object. + +Optionally you can set `controller.service.loadBalancerIP` if you need a static IP for the resulting `LoadBalancer`. + +### Ingress Admission Webhooks + +With nginx-ingress-controller version 0.25+, the nginx ingress controller pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent bad ingress from being added to the cluster. +**This feature is enabled by default since 0.31.0.** + +With nginx-ingress-controller in 0.25.* work only with kubernetes 1.14+, 0.26 fix [this issue](https://github.com/kubernetes/ingress-nginx/pull/4521) + +### Helm Error When Upgrading: spec.clusterIP: Invalid value: "" + +If you are upgrading this chart from a version between 0.31.0 and 1.2.2 then you may get an error like this: + +```console +Error: UPGRADE FAILED: Service "?????-controller" is invalid: spec.clusterIP: Invalid value: "": field is immutable +``` + +Detail of how and why are in [this issue](https://github.com/helm/charts/pull/13646) but to resolve this you can set `xxxx.service.omitClusterIP` to `true` where `xxxx` is the service referenced in the error. + +As of version `1.26.0` of this chart, by simply not providing any clusterIP value, `invalid: spec.clusterIP: Invalid value: "": field is immutable` will no longer occur since `clusterIP: ""` will not be rendered. + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} + +{{ template "helm-docs.versionFooter" . }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/controller-custom-ingressclass-flags.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/controller-custom-ingressclass-flags.yaml new file mode 100644 index 000000000..b28a2326e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/controller-custom-ingressclass-flags.yaml @@ -0,0 +1,7 @@ +controller: + watchIngressWithoutClass: true + ingressClassResource: + name: custom-nginx + enabled: true + default: true + controllerValue: "k8s.io/custom-nginx" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customconfig-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customconfig-values.yaml new file mode 100644 index 000000000..4393a5bc0 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customconfig-values.yaml @@ -0,0 +1,14 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + kind: DaemonSet + allowSnippetAnnotations: false + admissionWebhooks: + enabled: false + service: + type: ClusterIP + + config: + use-proxy-protocol: "true" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customnodeport-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customnodeport-values.yaml new file mode 100644 index 000000000..1d94be219 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-customnodeport-values.yaml @@ -0,0 +1,22 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + + service: + type: NodePort + nodePorts: + tcp: + 9000: 30090 + udp: + 9001: 30091 + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-extra-modules.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-extra-modules.yaml new file mode 100644 index 000000000..f299dbf1c --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-extra-modules.yaml @@ -0,0 +1,10 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + service: + type: ClusterIP + extraModules: + - name: opentelemetry + image: busybox diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-headers-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-headers-values.yaml new file mode 100644 index 000000000..ab7d47bd4 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-headers-values.yaml @@ -0,0 +1,14 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + addHeaders: + X-Frame-Options: deny + proxySetHeaders: + X-Forwarded-Proto: https + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-internal-lb-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-internal-lb-values.yaml new file mode 100644 index 000000000..0a200a746 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-internal-lb-values.yaml @@ -0,0 +1,14 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + internal: + enabled: true + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: "true" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-nodeport-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-nodeport-values.yaml new file mode 100644 index 000000000..3b7aa2fcd --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-nodeport-values.yaml @@ -0,0 +1,10 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: NodePort diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-podannotations-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-podannotations-values.yaml new file mode 100644 index 000000000..0b55306a1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-podannotations-values.yaml @@ -0,0 +1,17 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + metrics: + enabled: true + service: + type: ClusterIP + podAnnotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scheme: http + prometheus.io/scrape: "true" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-configMapNamespace-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-configMapNamespace-values.yaml new file mode 100644 index 000000000..acd86a77a --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-configMapNamespace-values.yaml @@ -0,0 +1,20 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + tcp: + configMapNamespace: default + udp: + configMapNamespace: default + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-values.yaml new file mode 100644 index 000000000..25ee64d85 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-udp-values.yaml @@ -0,0 +1,16 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-values.yaml new file mode 100644 index 000000000..380c8b4b1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/daemonset-tcp-values.yaml @@ -0,0 +1,14 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-default-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-default-values.yaml new file mode 100644 index 000000000..82fa23e85 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-default-values.yaml @@ -0,0 +1,10 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-metrics-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-metrics-values.yaml new file mode 100644 index 000000000..cb3cb54be --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-metrics-values.yaml @@ -0,0 +1,12 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + metrics: + enabled: true + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-psp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-psp-values.yaml new file mode 100644 index 000000000..8026a6356 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-psp-values.yaml @@ -0,0 +1,13 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-and-psp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-and-psp-values.yaml new file mode 100644 index 000000000..fccdb134c --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-and-psp-values.yaml @@ -0,0 +1,13 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: true + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-values.yaml new file mode 100644 index 000000000..54d364df1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deamonset-webhook-values.yaml @@ -0,0 +1,10 @@ +controller: + kind: DaemonSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: true + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-behavior-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-behavior-values.yaml new file mode 100644 index 000000000..dca3f35f8 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-behavior-values.yaml @@ -0,0 +1,14 @@ +controller: + autoscaling: + enabled: true + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Pods + value: 1 + periodSeconds: 180 + admissionWebhooks: + enabled: false + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-values.yaml new file mode 100644 index 000000000..b8b3ac686 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-autoscaling-values.yaml @@ -0,0 +1,11 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + autoscaling: + enabled: true + admissionWebhooks: + enabled: false + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customconfig-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customconfig-values.yaml new file mode 100644 index 000000000..174941848 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customconfig-values.yaml @@ -0,0 +1,12 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + config: + use-proxy-protocol: "true" + allowSnippetAnnotations: false + admissionWebhooks: + enabled: false + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customnodeport-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customnodeport-values.yaml new file mode 100644 index 000000000..a564eaf93 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-customnodeport-values.yaml @@ -0,0 +1,20 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: NodePort + nodePorts: + tcp: + 9000: 30090 + udp: + 9001: 30091 + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-default-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-default-values.yaml new file mode 100644 index 000000000..9f46b4e7e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-default-values.yaml @@ -0,0 +1,8 @@ +# Left blank to test default values +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-extra-modules.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-extra-modules.yaml new file mode 100644 index 000000000..ec5923548 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-extra-modules.yaml @@ -0,0 +1,10 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + service: + type: ClusterIP + extraModules: + - name: opentelemetry + image: busybox diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-headers-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-headers-values.yaml new file mode 100644 index 000000000..17a11ac37 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-headers-values.yaml @@ -0,0 +1,13 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + addHeaders: + X-Frame-Options: deny + proxySetHeaders: + X-Forwarded-Proto: https + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-internal-lb-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-internal-lb-values.yaml new file mode 100644 index 000000000..fd8df8de5 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-internal-lb-values.yaml @@ -0,0 +1,13 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + internal: + enabled: true + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: "true" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-metrics-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-metrics-values.yaml new file mode 100644 index 000000000..9209ad5a6 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-metrics-values.yaml @@ -0,0 +1,11 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + metrics: + enabled: true + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-nodeport-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-nodeport-values.yaml new file mode 100644 index 000000000..cd9b32352 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-nodeport-values.yaml @@ -0,0 +1,9 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: NodePort diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-podannotations-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-podannotations-values.yaml new file mode 100644 index 000000000..b48d93c46 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-podannotations-values.yaml @@ -0,0 +1,16 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + metrics: + enabled: true + service: + type: ClusterIP + podAnnotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scheme: http + prometheus.io/scrape: "true" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-psp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-psp-values.yaml new file mode 100644 index 000000000..2f332a7b2 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-psp-values.yaml @@ -0,0 +1,10 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-configMapNamespace-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-configMapNamespace-values.yaml new file mode 100644 index 000000000..c51a4e91f --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-configMapNamespace-values.yaml @@ -0,0 +1,19 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + tcp: + configMapNamespace: default + udp: + configMapNamespace: default + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-values.yaml new file mode 100644 index 000000000..5b45b69dc --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-udp-values.yaml @@ -0,0 +1,15 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-values.yaml new file mode 100644 index 000000000..ac0b6e60e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-tcp-values.yaml @@ -0,0 +1,11 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + 9001: "default/test:8080" diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-and-psp-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-and-psp-values.yaml new file mode 100644 index 000000000..6195bb339 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-and-psp-values.yaml @@ -0,0 +1,12 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: true + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-resources-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-resources-values.yaml new file mode 100644 index 000000000..49ebbb02c --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-resources-values.yaml @@ -0,0 +1,23 @@ +controller: + service: + type: ClusterIP + admissionWebhooks: + enabled: true + createSecretJob: + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + patchWebhookJob: + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + patch: + enabled: true diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-values.yaml new file mode 100644 index 000000000..76669a530 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/ci/deployment-webhook-values.yaml @@ -0,0 +1,9 @@ +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + admissionWebhooks: + enabled: true + service: + type: ClusterIP diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/NOTES.txt new file mode 100644 index 000000000..8985c56c0 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/NOTES.txt @@ -0,0 +1,80 @@ +The ingress-nginx controller has been installed. + +{{- if contains "NodePort" .Values.controller.service.type }} +Get the application URL by running these commands: + +{{- if (not (empty .Values.controller.service.nodePorts.http)) }} + export HTTP_NODE_PORT={{ .Values.controller.service.nodePorts.http }} +{{- else }} + export HTTP_NODE_PORT=$(kubectl --namespace {{ .Release.Namespace }} get services -o jsonpath="{.spec.ports[0].nodePort}" {{ include "ingress-nginx.controller.fullname" . }}) +{{- end }} +{{- if (not (empty .Values.controller.service.nodePorts.https)) }} + export HTTPS_NODE_PORT={{ .Values.controller.service.nodePorts.https }} +{{- else }} + export HTTPS_NODE_PORT=$(kubectl --namespace {{ .Release.Namespace }} get services -o jsonpath="{.spec.ports[1].nodePort}" {{ include "ingress-nginx.controller.fullname" . }}) +{{- end }} + export NODE_IP=$(kubectl --namespace {{ .Release.Namespace }} get nodes -o jsonpath="{.items[0].status.addresses[1].address}") + + echo "Visit http://$NODE_IP:$HTTP_NODE_PORT to access your application via HTTP." + echo "Visit https://$NODE_IP:$HTTPS_NODE_PORT to access your application via HTTPS." +{{- else if contains "LoadBalancer" .Values.controller.service.type }} +It may take a few minutes for the LoadBalancer IP to be available. +You can watch the status by running 'kubectl --namespace {{ .Release.Namespace }} get services -o wide -w {{ include "ingress-nginx.controller.fullname" . }}' +{{- else if contains "ClusterIP" .Values.controller.service.type }} +Get the application URL by running these commands: + export POD_NAME=$(kubectl --namespace {{ .Release.Namespace }} get pods -o jsonpath="{.items[0].metadata.name}" -l "app={{ template "ingress-nginx.name" . }},component={{ .Values.controller.name }},release={{ .Release.Name }}") + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:80 + echo "Visit http://127.0.0.1:8080 to access your application." +{{- end }} + +An example Ingress that makes use of the controller: + +{{- $isV1 := semverCompare ">=1" .Chart.AppVersion}} + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: example + namespace: foo + {{- if eq $isV1 false }} + annotations: + kubernetes.io/ingress.class: {{ .Values.controller.ingressClass }} + {{- end }} + spec: + {{- if $isV1 }} + ingressClassName: {{ .Values.controller.ingressClassResource.name }} + {{- end }} + rules: + - host: www.example.com + http: + paths: + - pathType: Prefix + backend: + service: + name: exampleService + port: + number: 80 + path: / + # This section is only required if TLS is to be enabled for the Ingress + tls: + - hosts: + - www.example.com + secretName: example-tls + +If TLS is enabled for the Ingress, a Secret containing the certificate and key must also be provided: + + apiVersion: v1 + kind: Secret + metadata: + name: example-tls + namespace: foo + data: + tls.crt: + tls.key: + type: kubernetes.io/tls + +{{- if .Values.controller.headers }} +################################################################################# +###### WARNING: `controller.headers` has been deprecated! ##### +###### It has been renamed to `controller.proxySetHeaders`. ##### +################################################################################# +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_helpers.tpl new file mode 100644 index 000000000..a72af5d9d --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_helpers.tpl @@ -0,0 +1,156 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "ingress-nginx.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ingress-nginx.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "ingress-nginx.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + + +{{/* +Container SecurityContext. +*/}} +{{- define "controller.containerSecurityContext" -}} +{{- if .Values.controller.containerSecurityContext -}} +{{- toYaml .Values.controller.containerSecurityContext -}} +{{- else -}} +capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE +runAsUser: {{ .Values.controller.image.runAsUser }} +allowPrivilegeEscalation: {{ .Values.controller.image.allowPrivilegeEscalation }} +{{- end }} +{{- end -}} + +{{/* +Create a default fully qualified controller name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "ingress-nginx.controller.fullname" -}} +{{- printf "%s-%s" (include "ingress-nginx.fullname" .) .Values.controller.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Construct the path for the publish-service. + +By convention this will simply use the / to match the name of the +service generated. + +Users can provide an override for an explicit service they want bound via `.Values.controller.publishService.pathOverride` + +*/}} +{{- define "ingress-nginx.controller.publishServicePath" -}} +{{- $defServiceName := printf "%s/%s" "$(POD_NAMESPACE)" (include "ingress-nginx.controller.fullname" .) -}} +{{- $servicePath := default $defServiceName .Values.controller.publishService.pathOverride }} +{{- print $servicePath | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified default backend name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "ingress-nginx.defaultBackend.fullname" -}} +{{- printf "%s-%s" (include "ingress-nginx.fullname" .) .Values.defaultBackend.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "ingress-nginx.labels" -}} +helm.sh/chart: {{ include "ingress-nginx.chart" . }} +{{ include "ingress-nginx.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/part-of: {{ template "ingress-nginx.name" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "ingress-nginx.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ingress-nginx.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the controller service account to use +*/}} +{{- define "ingress-nginx.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "ingress-nginx.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the backend service account to use - only used when podsecuritypolicy is also enabled +*/}} +{{- define "ingress-nginx.defaultBackend.serviceAccountName" -}} +{{- if .Values.defaultBackend.serviceAccount.create -}} + {{ default (printf "%s-backend" (include "ingress-nginx.fullname" .)) .Values.defaultBackend.serviceAccount.name }} +{{- else -}} + {{ default "default-backend" .Values.defaultBackend.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiGroup for PodSecurityPolicy. +*/}} +{{- define "podSecurityPolicy.apiGroup" -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "policy" -}} +{{- else -}} +{{- print "extensions" -}} +{{- end -}} +{{- end -}} + +{{/* +Check the ingress controller version tag is at most three versions behind the last release +*/}} +{{- define "isControllerTagValid" -}} +{{- if not (semverCompare ">=0.27.0-0" .Values.controller.image.tag) -}} +{{- fail "Controller container image tag should be 0.27.0 or higher" -}} +{{- end -}} +{{- end -}} + +{{/* +IngressClass parameters. +*/}} +{{- define "ingressClass.parameters" -}} + {{- if .Values.controller.ingressClassResource.parameters -}} + parameters: +{{ toYaml .Values.controller.ingressClassResource.parameters | indent 4}} + {{ end }} +{{- end -}} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_params.tpl b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_params.tpl new file mode 100644 index 000000000..305ce0dd2 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/_params.tpl @@ -0,0 +1,62 @@ +{{- define "ingress-nginx.params" -}} +- /nginx-ingress-controller +{{- if .Values.defaultBackend.enabled }} +- --default-backend-service=$(POD_NAMESPACE)/{{ include "ingress-nginx.defaultBackend.fullname" . }} +{{- end }} +{{- if and .Values.controller.publishService.enabled .Values.controller.service.enabled }} +{{- if .Values.controller.service.external.enabled }} +- --publish-service={{ template "ingress-nginx.controller.publishServicePath" . }} +{{- else if .Values.controller.service.internal.enabled }} +- --publish-service={{ template "ingress-nginx.controller.publishServicePath" . }}-internal +{{- end }} +{{- end }} +- --election-id={{ .Values.controller.electionID }} +- --controller-class={{ .Values.controller.ingressClassResource.controllerValue }} +{{- if .Values.controller.ingressClass }} +- --ingress-class={{ .Values.controller.ingressClass }} +{{- end }} +- --configmap={{ default "$(POD_NAMESPACE)" .Values.controller.configMapNamespace }}/{{ include "ingress-nginx.controller.fullname" . }} +{{- if .Values.tcp }} +- --tcp-services-configmap={{ default "$(POD_NAMESPACE)" .Values.controller.tcp.configMapNamespace }}/{{ include "ingress-nginx.fullname" . }}-tcp +{{- end }} +{{- if .Values.udp }} +- --udp-services-configmap={{ default "$(POD_NAMESPACE)" .Values.controller.udp.configMapNamespace }}/{{ include "ingress-nginx.fullname" . }}-udp +{{- end }} +{{- if .Values.controller.scope.enabled }} +- --watch-namespace={{ default "$(POD_NAMESPACE)" .Values.controller.scope.namespace }} +{{- end }} +{{- if and (not .Values.controller.scope.enabled) .Values.controller.scope.namespaceSelector }} +- --watch-namespace-selector={{ default "" .Values.controller.scope.namespaceSelector }} +{{- end }} +{{- if and .Values.controller.reportNodeInternalIp .Values.controller.hostNetwork }} +- --report-node-internal-ip-address={{ .Values.controller.reportNodeInternalIp }} +{{- end }} +{{- if .Values.controller.admissionWebhooks.enabled }} +- --validating-webhook=:{{ .Values.controller.admissionWebhooks.port }} +- --validating-webhook-certificate={{ .Values.controller.admissionWebhooks.certificate }} +- --validating-webhook-key={{ .Values.controller.admissionWebhooks.key }} +{{- end }} +{{- if .Values.controller.maxmindLicenseKey }} +- --maxmind-license-key={{ .Values.controller.maxmindLicenseKey }} +{{- end }} +{{- if .Values.controller.healthCheckHost }} +- --healthz-host={{ .Values.controller.healthCheckHost }} +{{- end }} +{{- if not (eq .Values.controller.healthCheckPath "/healthz") }} +- --health-check-path={{ .Values.controller.healthCheckPath }} +{{- end }} +{{- if .Values.controller.ingressClassByName }} +- --ingress-class-by-name=true +{{- end }} +{{- if .Values.controller.watchIngressWithoutClass }} +- --watch-ingress-without-class=true +{{- end }} +{{- range $key, $value := .Values.controller.extraArgs }} +{{- /* Accept keys without values or with false as value */}} +{{- if eq ($value | quote | len) 2 }} +- --{{ $key }} +{{- else }} +- --{{ $key }}={{ $value }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml new file mode 100644 index 000000000..5659a1f10 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update +{{- if .Values.podSecurityPolicy.enabled }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + {{- with .Values.controller.admissionWebhooks.existingPsp }} + - {{ . }} + {{- else }} + - {{ include "ingress-nginx.fullname" . }}-admission + {{- end }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100644 index 000000000..abf17fb9f --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "ingress-nginx.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100644 index 000000000..f20e247f9 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,76 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission-create + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + {{- with .Values.controller.admissionWebhooks.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 +{{- end }} + template: + metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission-create + {{- if .Values.controller.admissionWebhooks.patch.podAnnotations }} + annotations: {{ toYaml .Values.controller.admissionWebhooks.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.controller.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.controller.admissionWebhooks.patch.priorityClassName }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + containers: + - name: create + {{- with .Values.controller.admissionWebhooks.patch.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + {{- end }} + imagePullPolicy: {{ .Values.controller.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{ include "ingress-nginx.controller.fullname" . }}-admission,{{ include "ingress-nginx.controller.fullname" . }}-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name={{ include "ingress-nginx.fullname" . }}-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + allowPrivilegeEscalation: false + {{- if .Values.controller.admissionWebhooks.createSecretJob.resources }} + resources: {{ toYaml .Values.controller.admissionWebhooks.createSecretJob.resources | nindent 12 }} + {{- end }} + restartPolicy: OnFailure + serviceAccountName: {{ include "ingress-nginx.fullname" . }}-admission + {{- if .Values.controller.admissionWebhooks.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.controller.admissionWebhooks.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.controller.admissionWebhooks.patch.tolerations }} + tolerations: {{ toYaml .Values.controller.admissionWebhooks.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.controller.admissionWebhooks.patch.runAsUser }} + fsGroup: {{ .Values.controller.admissionWebhooks.patch.fsGroup }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100644 index 000000000..8583685fa --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,78 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission-patch + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + {{- with .Values.controller.admissionWebhooks.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 +{{- end }} + template: + metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission-patch + {{- if .Values.controller.admissionWebhooks.patch.podAnnotations }} + annotations: {{ toYaml .Values.controller.admissionWebhooks.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.controller.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.controller.admissionWebhooks.patch.priorityClassName }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + containers: + - name: patch + {{- with .Values.controller.admissionWebhooks.patch.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + {{- end }} + imagePullPolicy: {{ .Values.controller.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ include "ingress-nginx.fullname" . }}-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name={{ include "ingress-nginx.fullname" . }}-admission + - --patch-failure-policy={{ .Values.controller.admissionWebhooks.failurePolicy }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + allowPrivilegeEscalation: false + {{- if .Values.controller.admissionWebhooks.patchWebhookJob.resources }} + resources: {{ toYaml .Values.controller.admissionWebhooks.patchWebhookJob.resources | nindent 12 }} + {{- end }} + restartPolicy: OnFailure + serviceAccountName: {{ include "ingress-nginx.fullname" . }}-admission + {{- if .Values.controller.admissionWebhooks.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.controller.admissionWebhooks.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.controller.admissionWebhooks.patch.tolerations }} + tolerations: {{ toYaml .Values.controller.admissionWebhooks.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.controller.admissionWebhooks.patch.runAsUser }} + fsGroup: {{ .Values.controller.admissionWebhooks.patch.fsGroup }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/psp.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 000000000..70edde334 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled .Values.podSecurityPolicy.enabled (empty .Values.controller.admissionWebhooks.existingPsp) -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + allowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + requiredDropCapabilities: + - ALL + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - emptyDir + - projected + - secret + - downwardAPI +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml new file mode 100644 index 000000000..795bac6b9 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml new file mode 100644 index 000000000..698c5c864 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "ingress-nginx.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100644 index 000000000..eae475118 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.patch.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml new file mode 100644 index 000000000..8caffcb03 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml @@ -0,0 +1,48 @@ +{{- if .Values.controller.admissionWebhooks.enabled -}} +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + {{- if .Values.controller.admissionWebhooks.annotations }} + annotations: {{ toYaml .Values.controller.admissionWebhooks.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook + {{- with .Values.controller.admissionWebhooks.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }}-admission +webhooks: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: {{ .Values.controller.admissionWebhooks.failurePolicy | default "Fail" }} + sideEffects: None + admissionReviewVersions: + - v1 + clientConfig: + service: + namespace: {{ .Release.Namespace | quote }} + name: {{ include "ingress-nginx.controller.fullname" . }}-admission + path: /networking/v1/ingresses + {{- if .Values.controller.admissionWebhooks.timeoutSeconds }} + timeoutSeconds: {{ .Values.controller.admissionWebhooks.timeoutSeconds }} + {{- end }} + {{- if .Values.controller.admissionWebhooks.namespaceSelector }} + namespaceSelector: {{ toYaml .Values.controller.admissionWebhooks.namespaceSelector | nindent 6 }} + {{- end }} + {{- if .Values.controller.admissionWebhooks.objectSelector }} + objectSelector: {{ toYaml .Values.controller.admissionWebhooks.objectSelector | nindent 6 }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrole.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrole.yaml new file mode 100644 index 000000000..c093f048a --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrole.yaml @@ -0,0 +1,87 @@ +{{- if .Values.rbac.create }} + +{{- if and .Values.rbac.scope (not .Values.controller.scope.enabled) -}} + {{ required "Invalid configuration: 'rbac.scope' should be equal to 'controller.scope.enabled' (true/false)." (index (dict) ".") }} +{{- end }} + +{{- if not .Values.rbac.scope -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }} +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets +{{- if not .Values.controller.scope.enabled }} + - namespaces +{{- end}} + verbs: + - list + - watch +{{- if and .Values.controller.scope.enabled .Values.controller.scope.namespace }} + - apiGroups: + - "" + resources: + - namespaces + resourceNames: + - "{{ .Values.controller.scope.namespace }}" + verbs: + - get +{{- end }} + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +{{- end }} + +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrolebinding.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..acbbd8b10 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create (not .Values.rbac.scope) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "ingress-nginx.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "ingress-nginx.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml new file mode 100644 index 000000000..dfd49a126 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml @@ -0,0 +1,14 @@ +{{- if .Values.controller.addHeaders -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }}-custom-add-headers + namespace: {{ .Release.Namespace }} +data: {{ toYaml .Values.controller.addHeaders | nindent 2 }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-proxyheaders.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-proxyheaders.yaml new file mode 100644 index 000000000..f8d15faf9 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-proxyheaders.yaml @@ -0,0 +1,19 @@ +{{- if or .Values.controller.proxySetHeaders .Values.controller.headers -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }}-custom-proxy-headers + namespace: {{ .Release.Namespace }} +data: +{{- if .Values.controller.proxySetHeaders }} +{{ toYaml .Values.controller.proxySetHeaders | indent 2 }} +{{ else if and .Values.controller.headers (not .Values.controller.proxySetHeaders) }} +{{ toYaml .Values.controller.headers | indent 2 }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-tcp.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-tcp.yaml new file mode 100644 index 000000000..0f6088ea9 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-tcp.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tcp -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.controller.tcp.annotations }} + annotations: {{ toYaml .Values.controller.tcp.annotations | nindent 4 }} +{{- end }} + name: {{ include "ingress-nginx.fullname" . }}-tcp + namespace: {{ .Release.Namespace }} +data: {{ tpl (toYaml .Values.tcp) . | nindent 2 }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-udp.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-udp.yaml new file mode 100644 index 000000000..3772ec514 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap-udp.yaml @@ -0,0 +1,17 @@ +{{- if .Values.udp -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.controller.udp.annotations }} + annotations: {{ toYaml .Values.controller.udp.annotations | nindent 4 }} +{{- end }} + name: {{ include "ingress-nginx.fullname" . }}-udp + namespace: {{ .Release.Namespace }} +data: {{ tpl (toYaml .Values.udp) . | nindent 2 }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap.yaml new file mode 100644 index 000000000..f28b26e1e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-configmap.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.controller.configAnnotations }} + annotations: {{ toYaml .Values.controller.configAnnotations | nindent 4 }} +{{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} +data: + allow-snippet-annotations: "{{ .Values.controller.allowSnippetAnnotations }}" +{{- if .Values.controller.addHeaders }} + add-headers: {{ .Release.Namespace }}/{{ include "ingress-nginx.fullname" . }}-custom-add-headers +{{- end }} +{{- if or .Values.controller.proxySetHeaders .Values.controller.headers }} + proxy-set-headers: {{ .Release.Namespace }}/{{ include "ingress-nginx.fullname" . }}-custom-proxy-headers +{{- end }} +{{- if .Values.dhParam }} + ssl-dh-param: {{ printf "%s/%s" .Release.Namespace (include "ingress-nginx.controller.fullname" .) }} +{{- end }} +{{- range $key, $value := .Values.controller.config }} + {{- $key | nindent 2 }}: {{ $value | quote }} +{{- end }} + diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-daemonset.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-daemonset.yaml new file mode 100644 index 000000000..72811fbe4 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-daemonset.yaml @@ -0,0 +1,227 @@ +{{- if or (eq .Values.controller.kind "DaemonSet") (eq .Values.controller.kind "Both") -}} +{{- include "isControllerTagValid" . -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} + {{- if .Values.controller.annotations }} + annotations: {{ toYaml .Values.controller.annotations | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.controller.updateStrategy }} + updateStrategy: {{ toYaml .Values.controller.updateStrategy | nindent 4 }} + {{- end }} + minReadySeconds: {{ .Values.controller.minReadySeconds }} + template: + metadata: + {{- if .Values.controller.podAnnotations }} + annotations: + {{- range $key, $value := .Values.controller.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + labels: + {{- include "ingress-nginx.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.controller.podLabels }} + {{- toYaml .Values.controller.podLabels | nindent 8 }} + {{- end }} + spec: + {{- if .Values.controller.dnsConfig }} + dnsConfig: {{ toYaml .Values.controller.dnsConfig | nindent 8 }} + {{- end }} + {{- if .Values.controller.hostname }} + hostname: {{ toYaml .Values.controller.hostname | nindent 8 }} + {{- end }} + dnsPolicy: {{ .Values.controller.dnsPolicy }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if .Values.controller.priorityClassName }} + priorityClassName: {{ .Values.controller.priorityClassName }} + {{- end }} + {{- if or .Values.controller.podSecurityContext .Values.controller.sysctls }} + securityContext: + {{- end }} + {{- if .Values.controller.podSecurityContext }} + {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + {{- end }} + {{- if .Values.controller.sysctls }} + sysctls: + {{- range $sysctl, $value := .Values.controller.sysctls }} + - name: {{ $sysctl | quote }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + containers: + - name: {{ .Values.controller.containerName }} + {{- with .Values.controller.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + {{- end }} + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} + {{- if .Values.controller.lifecycle }} + lifecycle: {{ toYaml .Values.controller.lifecycle | nindent 12 }} + {{- end }} + args: + {{- include "ingress-nginx.params" . | nindent 12 }} + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: {{ .Values.controller.image.runAsUser }} + allowPrivilegeEscalation: {{ .Values.controller.image.allowPrivilegeEscalation }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.controller.enableMimalloc }} + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + {{- end }} + {{- if .Values.controller.extraEnvs }} + {{- toYaml .Values.controller.extraEnvs | nindent 12 }} + {{- end }} + {{- if .Values.controller.startupProbe }} + startupProbe: {{ toYaml .Values.controller.startupProbe | nindent 12 }} + {{- end }} + livenessProbe: {{ toYaml .Values.controller.livenessProbe | nindent 12 }} + readinessProbe: {{ toYaml .Values.controller.readinessProbe | nindent 12 }} + ports: + {{- range $key, $value := .Values.controller.containerPort }} + - name: {{ $key }} + containerPort: {{ $value }} + protocol: TCP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ index $.Values.controller.hostPort.ports $key | default $value }} + {{- end }} + {{- end }} + {{- if .Values.controller.metrics.enabled }} + - name: metrics + containerPort: {{ .Values.controller.metrics.port }} + protocol: TCP + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook + containerPort: {{ .Values.controller.admissionWebhooks.port }} + protocol: TCP + {{- end }} + {{- range $key, $value := .Values.tcp }} + - name: {{ $key }}-tcp + containerPort: {{ $key }} + protocol: TCP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ $key }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.udp }} + - name: {{ $key }}-udp + containerPort: {{ $key }} + protocol: UDP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ $key }} + {{- end }} + {{- end }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }} + volumeMounts: + {{- if .Values.controller.extraModules }} + - name: modules + mountPath: /modules_mount + {{- end }} + {{- if .Values.controller.customTemplate.configMapName }} + - mountPath: /etc/nginx/template + name: nginx-template-volume + readOnly: true + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + {{- end }} + {{- if .Values.controller.extraVolumeMounts }} + {{- toYaml .Values.controller.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.controller.resources }} + resources: {{ toYaml .Values.controller.resources | nindent 12 }} + {{- end }} + {{- if .Values.controller.extraContainers }} + {{ toYaml .Values.controller.extraContainers | nindent 8 }} + {{- end }} + + + {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }} + initContainers: + {{- if .Values.controller.extraInitContainers }} + {{ toYaml .Values.controller.extraInitContainers | nindent 8 }} + {{- end }} + {{- if .Values.controller.extraModules }} + {{- range .Values.controller.extraModules }} + - name: {{ .Name }} + image: {{ .Image }} + command: ['sh', '-c', '/usr/local/bin/init_module.sh'] + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.controller.hostNetwork }} + hostNetwork: {{ .Values.controller.hostNetwork }} + {{- end }} + {{- if .Values.controller.nodeSelector }} + nodeSelector: {{ toYaml .Values.controller.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.controller.tolerations }} + tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.controller.affinity }} + affinity: {{ toYaml .Values.controller.affinity | nindent 8 }} + {{- end }} + {{- if .Values.controller.topologySpreadConstraints }} + topologySpreadConstraints: {{ toYaml .Values.controller.topologySpreadConstraints | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }} + terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }} + volumes: + {{- if .Values.controller.extraModules }} + - name: modules + emptyDir: {} + {{- end }} + {{- if .Values.controller.customTemplate.configMapName }} + - name: nginx-template-volume + configMap: + name: {{ .Values.controller.customTemplate.configMapName }} + items: + - key: {{ .Values.controller.customTemplate.configMapKey }} + path: nginx.tmpl + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook-cert + secret: + secretName: {{ include "ingress-nginx.fullname" . }}-admission + {{- end }} + {{- if .Values.controller.extraVolumes }} + {{ toYaml .Values.controller.extraVolumes | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-deployment.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-deployment.yaml new file mode 100644 index 000000000..a1943cd91 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-deployment.yaml @@ -0,0 +1,225 @@ +{{- if or (eq .Values.controller.kind "Deployment") (eq .Values.controller.kind "Both") -}} +{{- include "isControllerTagValid" . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} + {{- if .Values.controller.annotations }} + annotations: {{ toYaml .Values.controller.annotations | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller + {{- if not .Values.controller.autoscaling.enabled }} + replicas: {{ .Values.controller.replicaCount }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.controller.updateStrategy }} + strategy: + {{ toYaml .Values.controller.updateStrategy | nindent 4 }} + {{- end }} + minReadySeconds: {{ .Values.controller.minReadySeconds }} + template: + metadata: + {{- if .Values.controller.podAnnotations }} + annotations: + {{- range $key, $value := .Values.controller.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + labels: + {{- include "ingress-nginx.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.controller.podLabels }} + {{- toYaml .Values.controller.podLabels | nindent 8 }} + {{- end }} + spec: + {{- if .Values.controller.dnsConfig }} + dnsConfig: {{ toYaml .Values.controller.dnsConfig | nindent 8 }} + {{- end }} + {{- if .Values.controller.hostname }} + hostname: {{ toYaml .Values.controller.hostname | nindent 8 }} + {{- end }} + dnsPolicy: {{ .Values.controller.dnsPolicy }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if .Values.controller.priorityClassName }} + priorityClassName: {{ .Values.controller.priorityClassName | quote }} + {{- end }} + {{- if or .Values.controller.podSecurityContext .Values.controller.sysctls }} + securityContext: + {{- end }} + {{- if .Values.controller.podSecurityContext }} + {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + {{- end }} + {{- if .Values.controller.sysctls }} + sysctls: + {{- range $sysctl, $value := .Values.controller.sysctls }} + - name: {{ $sysctl | quote }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + containers: + - name: {{ .Values.controller.containerName }} + {{- with .Values.controller.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + {{- end }} + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} + {{- if .Values.controller.lifecycle }} + lifecycle: {{ toYaml .Values.controller.lifecycle | nindent 12 }} + {{- end }} + args: + {{- include "ingress-nginx.params" . | nindent 12 }} + securityContext: {{ include "controller.containerSecurityContext" . | nindent 12 }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.controller.enableMimalloc }} + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + {{- end }} + {{- if .Values.controller.extraEnvs }} + {{- toYaml .Values.controller.extraEnvs | nindent 12 }} + {{- end }} + {{- if .Values.controller.startupProbe }} + startupProbe: {{ toYaml .Values.controller.startupProbe | nindent 12 }} + {{- end }} + livenessProbe: {{ toYaml .Values.controller.livenessProbe | nindent 12 }} + readinessProbe: {{ toYaml .Values.controller.readinessProbe | nindent 12 }} + ports: + {{- range $key, $value := .Values.controller.containerPort }} + - name: {{ $key }} + containerPort: {{ $value }} + protocol: TCP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ index $.Values.controller.hostPort.ports $key | default $value }} + {{- end }} + {{- end }} + {{- if .Values.controller.metrics.enabled }} + - name: metrics + containerPort: {{ .Values.controller.metrics.port }} + protocol: TCP + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook + containerPort: {{ .Values.controller.admissionWebhooks.port }} + protocol: TCP + {{- end }} + {{- range $key, $value := .Values.tcp }} + - name: {{ $key }}-tcp + containerPort: {{ $key }} + protocol: TCP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ $key }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.udp }} + - name: {{ $key }}-udp + containerPort: {{ $key }} + protocol: UDP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ $key }} + {{- end }} + {{- end }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }} + volumeMounts: + {{- if .Values.controller.extraModules }} + - name: modules + mountPath: /modules_mount + {{- end }} + {{- if .Values.controller.customTemplate.configMapName }} + - mountPath: /etc/nginx/template + name: nginx-template-volume + readOnly: true + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + {{- end }} + {{- if .Values.controller.extraVolumeMounts }} + {{- toYaml .Values.controller.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.controller.resources }} + resources: {{ toYaml .Values.controller.resources | nindent 12 }} + {{- end }} + {{- if .Values.controller.extraContainers }} + {{ toYaml .Values.controller.extraContainers | nindent 8 }} + {{- end }} + {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }} + initContainers: + {{- if .Values.controller.extraInitContainers }} + {{ toYaml .Values.controller.extraInitContainers | nindent 8 }} + {{- end }} + {{- if .Values.controller.extraModules }} + {{- range .Values.controller.extraModules }} + - name: {{ .name }} + image: {{ .image }} + command: ['sh', '-c', '/usr/local/bin/init_module.sh'] + volumeMounts: + - name: modules + mountPath: /modules_mount + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.controller.hostNetwork }} + hostNetwork: {{ .Values.controller.hostNetwork }} + {{- end }} + {{- if .Values.controller.nodeSelector }} + nodeSelector: {{ toYaml .Values.controller.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.controller.tolerations }} + tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.controller.affinity }} + affinity: {{ toYaml .Values.controller.affinity | nindent 8 }} + {{- end }} + {{- if .Values.controller.topologySpreadConstraints }} + topologySpreadConstraints: {{ toYaml .Values.controller.topologySpreadConstraints | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }} + terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }} + volumes: + {{- if .Values.controller.extraModules }} + - name: modules + emptyDir: {} + {{- end }} + {{- if .Values.controller.customTemplate.configMapName }} + - name: nginx-template-volume + configMap: + name: {{ .Values.controller.customTemplate.configMapName }} + items: + - key: {{ .Values.controller.customTemplate.configMapKey }} + path: nginx.tmpl + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook-cert + secret: + secretName: {{ include "ingress-nginx.fullname" . }}-admission + {{- end }} + {{- if .Values.controller.extraVolumes }} + {{ toYaml .Values.controller.extraVolumes | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-hpa.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-hpa.yaml new file mode 100644 index 000000000..e0979f14b --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-hpa.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.controller.autoscaling.enabled (or (eq .Values.controller.kind "Deployment") (eq .Values.controller.kind "Both")) -}} +{{- if not .Values.controller.keda.enabled }} + +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + annotations: + {{- with .Values.controller.autoscaling.annotations }} + {{- toYaml . | trimSuffix "\n" | nindent 4 }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "ingress-nginx.controller.fullname" . }} + minReplicas: {{ .Values.controller.autoscaling.minReplicas }} + maxReplicas: {{ .Values.controller.autoscaling.maxReplicas }} + metrics: + {{- with .Values.controller.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.controller.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.controller.autoscalingTemplate }} + {{- toYaml . | nindent 2 }} + {{- end }} + {{- with .Values.controller.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} + diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-ingressclass.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-ingressclass.yaml new file mode 100644 index 000000000..9492784a2 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-ingressclass.yaml @@ -0,0 +1,21 @@ +{{- if .Values.controller.ingressClassResource.enabled -}} +# We don't support namespaced ingressClass yet +# So a ClusterRole and a ClusterRoleBinding is required +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ .Values.controller.ingressClassResource.name }} +{{- if .Values.controller.ingressClassResource.default }} + annotations: + ingressclass.kubernetes.io/is-default-class: "true" +{{- end }} +spec: + controller: {{ .Values.controller.ingressClassResource.controllerValue }} + {{ template "ingressClass.parameters" . }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-keda.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-keda.yaml new file mode 100644 index 000000000..875157ea4 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-keda.yaml @@ -0,0 +1,42 @@ +{{- if and .Values.controller.keda.enabled (or (eq .Values.controller.kind "Deployment") (eq .Values.controller.kind "Both")) -}} +# https://keda.sh/docs/ + +apiVersion: {{ .Values.controller.keda.apiVersion }} +kind: ScaledObject +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + {{- if .Values.controller.keda.scaledObject.annotations }} + annotations: {{ toYaml .Values.controller.keda.scaledObject.annotations | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: +{{- if eq .Values.controller.keda.apiVersion "keda.k8s.io/v1alpha1" }} + deploymentName: {{ include "ingress-nginx.controller.fullname" . }} +{{- else if eq .Values.controller.keda.apiVersion "keda.sh/v1alpha1" }} + name: {{ include "ingress-nginx.controller.fullname" . }} +{{- end }} + pollingInterval: {{ .Values.controller.keda.pollingInterval }} + cooldownPeriod: {{ .Values.controller.keda.cooldownPeriod }} + minReplicaCount: {{ .Values.controller.keda.minReplicas }} + maxReplicaCount: {{ .Values.controller.keda.maxReplicas }} + triggers: +{{- with .Values.controller.keda.triggers }} +{{ toYaml . | indent 2 }} +{{ end }} + advanced: + restoreToOriginalReplicaCount: {{ .Values.controller.keda.restoreToOriginalReplicaCount }} +{{- if .Values.controller.keda.behavior }} + horizontalPodAutoscalerConfig: + behavior: +{{ with .Values.controller.keda.behavior -}} +{{ toYaml . | indent 8 }} +{{ end }} + +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-poddisruptionbudget.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-poddisruptionbudget.yaml new file mode 100644 index 000000000..8dfbe9891 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-poddisruptionbudget.yaml @@ -0,0 +1,19 @@ +{{- if or (and .Values.controller.autoscaling.enabled (gt (.Values.controller.autoscaling.minReplicas | int) 1)) (and (not .Values.controller.autoscaling.enabled) (gt (.Values.controller.replicaCount | int) 1)) }} +apiVersion: {{ ternary "policy/v1" "policy/v1beta1" (semverCompare ">=1.21.0-0" .Capabilities.KubeVersion.Version) }} +kind: PodDisruptionBudget +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller + minAvailable: {{ .Values.controller.minAvailable }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-prometheusrules.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-prometheusrules.yaml new file mode 100644 index 000000000..ca5427523 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-prometheusrules.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.controller.metrics.enabled .Values.controller.metrics.prometheusRule.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "ingress-nginx.controller.fullname" . }} +{{- if .Values.controller.metrics.prometheusRule.namespace }} + namespace: {{ .Values.controller.metrics.prometheusRule.namespace | quote }} +{{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- if .Values.controller.metrics.prometheusRule.additionalLabels }} + {{- toYaml .Values.controller.metrics.prometheusRule.additionalLabels | nindent 4 }} + {{- end }} +spec: +{{- if .Values.controller.metrics.prometheusRule.rules }} + groups: + - name: {{ template "ingress-nginx.name" . }} + rules: {{- toYaml .Values.controller.metrics.prometheusRule.rules | nindent 4 }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-psp.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-psp.yaml new file mode 100644 index 000000000..a859594d1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-psp.yaml @@ -0,0 +1,89 @@ +{{- if and .Values.podSecurityPolicy.enabled (empty .Values.controller.existingPsp) -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ingress-nginx.fullname" . }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + allowedCapabilities: + - NET_BIND_SERVICE +{{- if .Values.controller.sysctls }} + allowedUnsafeSysctls: + {{- range $sysctl, $value := .Values.controller.sysctls }} + - {{ $sysctl }} + {{- end }} +{{- end }} + privileged: false + allowPrivilegeEscalation: true + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + #- 'projected' + - 'secret' + #- 'downwardAPI' +{{- if .Values.controller.hostNetwork }} + hostNetwork: {{ .Values.controller.hostNetwork }} +{{- end }} +{{- if or .Values.controller.hostNetwork .Values.controller.hostPort.enabled }} + hostPorts: +{{- if .Values.controller.hostNetwork }} +{{- range $key, $value := .Values.controller.containerPort }} + # {{ $key }} + - min: {{ $value }} + max: {{ $value }} +{{- end }} +{{- else if .Values.controller.hostPort.enabled }} +{{- range $key, $value := .Values.controller.hostPort.ports }} + # {{ $key }} + - min: {{ $value }} + max: {{ $value }} +{{- end }} +{{- end }} +{{- if .Values.controller.metrics.enabled }} + # metrics + - min: {{ .Values.controller.metrics.port }} + max: {{ .Values.controller.metrics.port }} +{{- end }} +{{- if .Values.controller.admissionWebhooks.enabled }} + # admission webhooks + - min: {{ .Values.controller.admissionWebhooks.port }} + max: {{ .Values.controller.admissionWebhooks.port }} +{{- end }} +{{- range $key, $value := .Values.tcp }} + # {{ $key }}-tcp + - min: {{ $key }} + max: {{ $key }} +{{- end }} +{{- range $key, $value := .Values.udp }} + # {{ $key }}-udp + - min: {{ $key }} + max: {{ $key }} +{{- end }} +{{- end }} + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'MustRunAsNonRoot' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + seLinux: + rule: 'RunAsAny' +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-role.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-role.yaml new file mode 100644 index 000000000..47bbc32d0 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-role.yaml @@ -0,0 +1,93 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }} + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - {{ .Values.controller.electionID }} + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +{{- if .Values.podSecurityPolicy.enabled }} + - apiGroups: [{{ template "podSecurityPolicy.apiGroup" . }}] + resources: ['podsecuritypolicies'] + verbs: ['use'] + {{- with .Values.controller.existingPsp }} + resourceNames: [{{ . }}] + {{- else }} + resourceNames: [{{ include "ingress-nginx.fullname" . }}] + {{- end }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-rolebinding.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-rolebinding.yaml new file mode 100644 index 000000000..e846a1183 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "ingress-nginx.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "ingress-nginx.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-internal.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-internal.yaml new file mode 100644 index 000000000..599449836 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-internal.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.controller.service.enabled .Values.controller.service.internal.enabled .Values.controller.service.internal.annotations}} +apiVersion: v1 +kind: Service +metadata: + annotations: + {{- range $key, $value := .Values.controller.service.internal.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- if .Values.controller.service.labels }} + {{- toYaml .Values.controller.service.labels | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }}-internal + namespace: {{ .Release.Namespace }} +spec: + type: "{{ .Values.controller.service.type }}" +{{- if .Values.controller.service.internal.loadBalancerIP }} + loadBalancerIP: {{ .Values.controller.service.internal.loadBalancerIP }} +{{- end }} +{{- if .Values.controller.service.internal.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ toYaml .Values.controller.service.internal.loadBalancerSourceRanges | nindent 4 }} +{{- end }} +{{- if .Values.controller.service.internal.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.controller.service.internal.externalTrafficPolicy }} +{{- end }} + ports: + {{- $setNodePorts := (or (eq .Values.controller.service.type "NodePort") (eq .Values.controller.service.type "LoadBalancer")) }} + {{- if .Values.controller.service.enableHttp }} + - name: http + port: {{ .Values.controller.service.ports.http }} + protocol: TCP + targetPort: {{ .Values.controller.service.targetPorts.http }} + {{- if semverCompare ">=1.20" .Capabilities.KubeVersion.Version }} + appProtocol: http + {{- end }} + {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.http))) }} + nodePort: {{ .Values.controller.service.nodePorts.http }} + {{- end }} + {{- end }} + {{- if .Values.controller.service.enableHttps }} + - name: https + port: {{ .Values.controller.service.ports.https }} + protocol: TCP + targetPort: {{ .Values.controller.service.targetPorts.https }} + {{- if semverCompare ">=1.20" .Capabilities.KubeVersion.Version }} + appProtocol: https + {{- end }} + {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.https))) }} + nodePort: {{ .Values.controller.service.nodePorts.https }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.tcp }} + - name: {{ $key }}-tcp + port: {{ $key }} + protocol: TCP + targetPort: {{ $key }}-tcp + {{- if $.Values.controller.service.nodePorts.tcp }} + {{- if index $.Values.controller.service.nodePorts.tcp $key }} + nodePort: {{ index $.Values.controller.service.nodePorts.tcp $key }} + {{- end }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.udp }} + - name: {{ $key }}-udp + port: {{ $key }} + protocol: UDP + targetPort: {{ $key }}-udp + {{- if $.Values.controller.service.nodePorts.udp }} + {{- if index $.Values.controller.service.nodePorts.udp $key }} + nodePort: {{ index $.Values.controller.service.nodePorts.udp $key }} + {{- end }} + {{- end }} + {{- end }} + selector: + {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-metrics.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-metrics.yaml new file mode 100644 index 000000000..0aaf41473 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-metrics.yaml @@ -0,0 +1,45 @@ +{{- if .Values.controller.metrics.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.controller.metrics.service.annotations }} + annotations: {{ toYaml .Values.controller.metrics.service.annotations | nindent 4 }} +{{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- if .Values.controller.metrics.service.labels }} + {{- toYaml .Values.controller.metrics.service.labels | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }}-metrics + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.controller.metrics.service.type }} +{{- if .Values.controller.metrics.service.clusterIP }} + clusterIP: {{ .Values.controller.metrics.service.clusterIP }} +{{- end }} +{{- if .Values.controller.metrics.service.externalIPs }} + externalIPs: {{ toYaml .Values.controller.metrics.service.externalIPs | nindent 4 }} +{{- end }} +{{- if .Values.controller.metrics.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.controller.metrics.service.loadBalancerIP }} +{{- end }} +{{- if .Values.controller.metrics.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ toYaml .Values.controller.metrics.service.loadBalancerSourceRanges | nindent 4 }} +{{- end }} +{{- if .Values.controller.metrics.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.controller.metrics.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: metrics + port: {{ .Values.controller.metrics.service.servicePort }} + protocol: TCP + targetPort: metrics + {{- $setNodePorts := (or (eq .Values.controller.metrics.service.type "NodePort") (eq .Values.controller.metrics.service.type "LoadBalancer")) }} + {{- if (and $setNodePorts (not (empty .Values.controller.metrics.service.nodePort))) }} + nodePort: {{ .Values.controller.metrics.service.nodePort }} + {{- end }} + selector: + {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-webhook.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-webhook.yaml new file mode 100644 index 000000000..2aae24fcf --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service-webhook.yaml @@ -0,0 +1,40 @@ +{{- if .Values.controller.admissionWebhooks.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.controller.admissionWebhooks.service.annotations }} + annotations: {{ toYaml .Values.controller.admissionWebhooks.service.annotations | nindent 4 }} +{{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }}-admission + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.controller.admissionWebhooks.service.type }} +{{- if .Values.controller.admissionWebhooks.service.clusterIP }} + clusterIP: {{ .Values.controller.admissionWebhooks.service.clusterIP }} +{{- end }} +{{- if .Values.controller.admissionWebhooks.service.externalIPs }} + externalIPs: {{ toYaml .Values.controller.admissionWebhooks.service.externalIPs | nindent 4 }} +{{- end }} +{{- if .Values.controller.admissionWebhooks.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.controller.admissionWebhooks.service.loadBalancerIP }} +{{- end }} +{{- if .Values.controller.admissionWebhooks.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ toYaml .Values.controller.admissionWebhooks.service.loadBalancerSourceRanges | nindent 4 }} +{{- end }} + ports: + - name: https-webhook + port: 443 + targetPort: webhook + {{- if semverCompare ">=1.20" .Capabilities.KubeVersion.Version }} + appProtocol: https + {{- end }} + selector: + {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service.yaml new file mode 100644 index 000000000..05fb2041e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-service.yaml @@ -0,0 +1,101 @@ +{{- if and .Values.controller.service.enabled .Values.controller.service.external.enabled -}} +apiVersion: v1 +kind: Service +metadata: + annotations: + {{- range $key, $value := .Values.controller.service.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- if .Values.controller.service.labels }} + {{- toYaml .Values.controller.service.labels | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.controller.service.type }} +{{- if .Values.controller.service.clusterIP }} + clusterIP: {{ .Values.controller.service.clusterIP }} +{{- end }} +{{- if .Values.controller.service.externalIPs }} + externalIPs: {{ toYaml .Values.controller.service.externalIPs | nindent 4 }} +{{- end }} +{{- if .Values.controller.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.controller.service.loadBalancerIP }} +{{- end }} +{{- if .Values.controller.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ toYaml .Values.controller.service.loadBalancerSourceRanges | nindent 4 }} +{{- end }} +{{- if .Values.controller.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.controller.service.externalTrafficPolicy }} +{{- end }} +{{- if .Values.controller.service.sessionAffinity }} + sessionAffinity: {{ .Values.controller.service.sessionAffinity }} +{{- end }} +{{- if .Values.controller.service.healthCheckNodePort }} + healthCheckNodePort: {{ .Values.controller.service.healthCheckNodePort }} +{{- end }} +{{- if semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version -}} +{{- if .Values.controller.service.ipFamilyPolicy }} + ipFamilyPolicy: {{ .Values.controller.service.ipFamilyPolicy }} +{{- end }} +{{- end }} +{{- if semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version -}} +{{- if .Values.controller.service.ipFamilies }} + ipFamilies: {{ toYaml .Values.controller.service.ipFamilies | nindent 4 }} +{{- end }} +{{- end }} + ports: + {{- $setNodePorts := (or (eq .Values.controller.service.type "NodePort") (eq .Values.controller.service.type "LoadBalancer")) }} + {{- if .Values.controller.service.enableHttp }} + - name: http + port: {{ .Values.controller.service.ports.http }} + protocol: TCP + targetPort: {{ .Values.controller.service.targetPorts.http }} + {{- if and (semverCompare ">=1.20" .Capabilities.KubeVersion.Version) (.Values.controller.service.appProtocol) }} + appProtocol: http + {{- end }} + {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.http))) }} + nodePort: {{ .Values.controller.service.nodePorts.http }} + {{- end }} + {{- end }} + {{- if .Values.controller.service.enableHttps }} + - name: https + port: {{ .Values.controller.service.ports.https }} + protocol: TCP + targetPort: {{ .Values.controller.service.targetPorts.https }} + {{- if and (semverCompare ">=1.20" .Capabilities.KubeVersion.Version) (.Values.controller.service.appProtocol) }} + appProtocol: https + {{- end }} + {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.https))) }} + nodePort: {{ .Values.controller.service.nodePorts.https }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.tcp }} + - name: {{ $key }}-tcp + port: {{ $key }} + protocol: TCP + targetPort: {{ $key }}-tcp + {{- if $.Values.controller.service.nodePorts.tcp }} + {{- if index $.Values.controller.service.nodePorts.tcp $key }} + nodePort: {{ index $.Values.controller.service.nodePorts.tcp $key }} + {{- end }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.udp }} + - name: {{ $key }}-udp + port: {{ $key }} + protocol: UDP + targetPort: {{ $key }}-udp + {{- if $.Values.controller.service.nodePorts.udp }} + {{- if index $.Values.controller.service.nodePorts.udp $key }} + nodePort: {{ index $.Values.controller.service.nodePorts.udp $key }} + {{- end }} + {{- end }} + {{- end }} + selector: + {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-serviceaccount.yaml new file mode 100644 index 000000000..824b2a124 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if or .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "ingress-nginx.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + {{- if .Values.serviceAccount.annotations }} + annotations: + {{ toYaml .Values.serviceAccount.annotations | indent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-servicemonitor.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-servicemonitor.yaml new file mode 100644 index 000000000..4dbc6da9f --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/controller-servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.controller.metrics.enabled .Values.controller.metrics.serviceMonitor.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "ingress-nginx.controller.fullname" . }} +{{- if .Values.controller.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.controller.metrics.serviceMonitor.namespace | quote }} +{{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- if .Values.controller.metrics.serviceMonitor.additionalLabels }} + {{- toYaml .Values.controller.metrics.serviceMonitor.additionalLabels | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: metrics + interval: {{ .Values.controller.metrics.serviceMonitor.scrapeInterval }} + {{- if .Values.controller.metrics.serviceMonitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if .Values.controller.metrics.serviceMonitor.relabelings }} + relabelings: {{ toYaml .Values.controller.metrics.serviceMonitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.controller.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: {{ toYaml .Values.controller.metrics.serviceMonitor.metricRelabelings | nindent 8 }} + {{- end }} +{{- if .Values.controller.metrics.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.controller.metrics.serviceMonitor.jobLabel | quote }} +{{- end }} +{{- if .Values.controller.metrics.serviceMonitor.namespaceSelector }} + namespaceSelector: {{ toYaml .Values.controller.metrics.serviceMonitor.namespaceSelector | nindent 4 }} +{{- else }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} +{{- if .Values.controller.metrics.serviceMonitor.targetLabels }} + targetLabels: + {{- range .Values.controller.metrics.serviceMonitor.targetLabels }} + - {{ . }} + {{- end }} +{{- end }} + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-deployment.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-deployment.yaml new file mode 100644 index 000000000..fd3e96e9e --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-deployment.yaml @@ -0,0 +1,118 @@ +{{- if .Values.defaultBackend.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.defaultBackend.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: default-backend +{{- if not .Values.defaultBackend.autoscaling.enabled }} + replicas: {{ .Values.defaultBackend.replicaCount }} +{{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + template: + metadata: + {{- if .Values.defaultBackend.podAnnotations }} + annotations: {{ toYaml .Values.defaultBackend.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "ingress-nginx.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.defaultBackend.podLabels }} + {{- toYaml .Values.defaultBackend.podLabels | nindent 8 }} + {{- end }} + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if .Values.defaultBackend.priorityClassName }} + priorityClassName: {{ .Values.defaultBackend.priorityClassName }} + {{- end }} + {{- if .Values.defaultBackend.podSecurityContext }} + securityContext: {{ toYaml .Values.defaultBackend.podSecurityContext | nindent 8 }} + {{- end }} + containers: + - name: {{ template "ingress-nginx.name" . }}-default-backend + {{- with .Values.defaultBackend.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" + {{- end }} + imagePullPolicy: {{ .Values.defaultBackend.image.pullPolicy }} + {{- if .Values.defaultBackend.extraArgs }} + args: + {{- range $key, $value := .Values.defaultBackend.extraArgs }} + {{- /* Accept keys without values or with false as value */}} + {{- if eq ($value | quote | len) 2 }} + - --{{ $key }} + {{- else }} + - --{{ $key }}={{ $value }} + {{- end }} + {{- end }} + {{- end }} + securityContext: + capabilities: + drop: + - ALL + runAsUser: {{ .Values.defaultBackend.image.runAsUser }} + runAsNonRoot: {{ .Values.defaultBackend.image.runAsNonRoot }} + allowPrivilegeEscalation: {{ .Values.defaultBackend.image.allowPrivilegeEscalation }} + readOnlyRootFilesystem: {{ .Values.defaultBackend.image.readOnlyRootFilesystem}} + {{- if .Values.defaultBackend.extraEnvs }} + env: {{ toYaml .Values.defaultBackend.extraEnvs | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.defaultBackend.port }} + scheme: HTTP + initialDelaySeconds: {{ .Values.defaultBackend.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.defaultBackend.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.defaultBackend.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.defaultBackend.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.defaultBackend.livenessProbe.failureThreshold }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.defaultBackend.port }} + scheme: HTTP + initialDelaySeconds: {{ .Values.defaultBackend.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.defaultBackend.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.defaultBackend.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.defaultBackend.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.defaultBackend.readinessProbe.failureThreshold }} + ports: + - name: http + containerPort: {{ .Values.defaultBackend.port }} + protocol: TCP + {{- if .Values.defaultBackend.extraVolumeMounts }} + volumeMounts: {{- toYaml .Values.defaultBackend.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.defaultBackend.resources }} + resources: {{ toYaml .Values.defaultBackend.resources | nindent 12 }} + {{- end }} + {{- if .Values.defaultBackend.nodeSelector }} + nodeSelector: {{ toYaml .Values.defaultBackend.nodeSelector | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "ingress-nginx.defaultBackend.serviceAccountName" . }} + {{- if .Values.defaultBackend.tolerations }} + tolerations: {{ toYaml .Values.defaultBackend.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.defaultBackend.affinity }} + affinity: {{ toYaml .Values.defaultBackend.affinity | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: 60 + {{- if .Values.defaultBackend.extraVolumes }} + volumes: {{ toYaml .Values.defaultBackend.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-hpa.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-hpa.yaml new file mode 100644 index 000000000..594d26525 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-hpa.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.defaultBackend.enabled .Values.defaultBackend.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "ingress-nginx.defaultBackend.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "ingress-nginx.defaultBackend.fullname" . }} + minReplicas: {{ .Values.defaultBackend.autoscaling.minReplicas }} + maxReplicas: {{ .Values.defaultBackend.autoscaling.maxReplicas }} + metrics: +{{- with .Values.defaultBackend.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ . }} +{{- end }} +{{- with .Values.defaultBackend.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ . }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-poddisruptionbudget.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-poddisruptionbudget.yaml new file mode 100644 index 000000000..00891cee5 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-poddisruptionbudget.yaml @@ -0,0 +1,21 @@ +{{- if .Values.defaultBackend.enabled -}} +{{- if or (gt (.Values.defaultBackend.replicaCount | int) 1) (gt (.Values.defaultBackend.autoscaling.minReplicas | int) 1) }} +apiVersion: {{ ternary "policy/v1" "policy/v1beta1" (semverCompare ">=1.21.0-0" .Capabilities.KubeVersion.Version) }} +kind: PodDisruptionBudget +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.defaultBackend.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: default-backend + minAvailable: {{ .Values.defaultBackend.minAvailable }} +{{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-psp.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-psp.yaml new file mode 100644 index 000000000..42061c5d3 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-psp.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.podSecurityPolicy.enabled .Values.defaultBackend.enabled (empty .Values.defaultBackend.existingPsp) -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ingress-nginx.fullname" . }}-backend + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + allowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + requiredDropCapabilities: + - ALL + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - emptyDir + - projected + - secret + - downwardAPI +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-role.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-role.yaml new file mode 100644 index 000000000..a2b457c36 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-role.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.rbac.create .Values.podSecurityPolicy.enabled .Values.defaultBackend.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }}-backend + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: [{{ template "podSecurityPolicy.apiGroup" . }}] + resources: ['podsecuritypolicies'] + verbs: ['use'] + {{- with .Values.defaultBackend.existingPsp }} + resourceNames: [{{ . }}] + {{- else }} + resourceNames: [{{ include "ingress-nginx.fullname" . }}-backend] + {{- end }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-rolebinding.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-rolebinding.yaml new file mode 100644 index 000000000..dbaa516b9 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.podSecurityPolicy.enabled .Values.defaultBackend.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.fullname" . }}-backend + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "ingress-nginx.fullname" . }}-backend +subjects: + - kind: ServiceAccount + name: {{ template "ingress-nginx.defaultBackend.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-service.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-service.yaml new file mode 100644 index 000000000..5f1d09a95 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-service.yaml @@ -0,0 +1,41 @@ +{{- if .Values.defaultBackend.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.defaultBackend.service.annotations }} + annotations: {{ toYaml .Values.defaultBackend.service.annotations | nindent 4 }} +{{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.defaultBackend.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.defaultBackend.service.type }} +{{- if .Values.defaultBackend.service.clusterIP }} + clusterIP: {{ .Values.defaultBackend.service.clusterIP }} +{{- end }} +{{- if .Values.defaultBackend.service.externalIPs }} + externalIPs: {{ toYaml .Values.defaultBackend.service.externalIPs | nindent 4 }} +{{- end }} +{{- if .Values.defaultBackend.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.defaultBackend.service.loadBalancerIP }} +{{- end }} +{{- if .Values.defaultBackend.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ toYaml .Values.defaultBackend.service.loadBalancerSourceRanges | nindent 4 }} +{{- end }} + ports: + - name: http + port: {{ .Values.defaultBackend.service.servicePort }} + protocol: TCP + targetPort: http + {{- if semverCompare ">=1.20" .Capabilities.KubeVersion.Version }} + appProtocol: http + {{- end }} + selector: + {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: default-backend +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml new file mode 100644 index 000000000..b45a95ad2 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.defaultBackend.enabled .Values.defaultBackend.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + {{- with .Values.defaultBackend.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "ingress-nginx.defaultBackend.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +automountServiceAccountToken: {{ .Values.defaultBackend.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/dh-param-secret.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/dh-param-secret.yaml new file mode 100644 index 000000000..12e7a4f63 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/templates/dh-param-secret.yaml @@ -0,0 +1,10 @@ +{{- with .Values.dhParam -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "ingress-nginx.controller.fullname" $ }} + labels: + {{- include "ingress-nginx.labels" $ | nindent 4 }} +data: + dhparam.pem: {{ . }} +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/ingress-nginx/values.yaml b/scripts/helmcharts/openreplay/charts/ingress-nginx/values.yaml new file mode 100644 index 000000000..c887dc051 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/ingress-nginx/values.yaml @@ -0,0 +1,919 @@ +## nginx configuration +## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/index.md +## + +## Overrides for generated resource names +# See templates/_helpers.tpl +# nameOverride: +# fullnameOverride: + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +controller: + name: controller + image: + registry: k8s.gcr.io + image: ingress-nginx/controller + ## for backwards compatibility consider setting the full image url via the repository value below + ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + ## repository: + tag: "v1.1.2" + digest: sha256:28b11ce69e57843de44e3db6413e98d09de0f6688e33d4bd384002a44f78405c + pullPolicy: IfNotPresent + # www-data -> uid 101 + runAsUser: 101 + allowPrivilegeEscalation: true + + # -- Use an existing PSP instead of creating one + existingPsp: "" + + # -- Configures the controller container name + containerName: controller + + # -- Configures the ports that the nginx-controller listens on + containerPort: + http: 80 + https: 443 + + # -- Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/ + config: {} + + # -- Annotations to be added to the controller config configuration configmap. + configAnnotations: {} + + # -- Will add custom headers before sending traffic to backends according to https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/custom-headers + proxySetHeaders: {} + + # -- Will add custom headers before sending response traffic to the client according to: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#add-headers + addHeaders: {} + + # -- Optionally customize the pod dnsConfig. + dnsConfig: {} + + # -- Optionally customize the pod hostname. + hostname: {} + + # -- Optionally change this to ClusterFirstWithHostNet in case you have 'hostNetwork: true'. + # By default, while using host network, name resolution uses the host's DNS. If you wish nginx-controller + # to keep resolving names inside the k8s network, use ClusterFirstWithHostNet. + dnsPolicy: ClusterFirst + + # -- Bare-metal considerations via the host network https://kubernetes.github.io/ingress-nginx/deploy/baremetal/#via-the-host-network + # Ingress status was blank because there is no Service exposing the NGINX Ingress controller in a configuration using the host network, the default --publish-service flag used in standard cloud setups does not apply + reportNodeInternalIp: false + + # -- Process Ingress objects without ingressClass annotation/ingressClassName field + # Overrides value for --watch-ingress-without-class flag of the controller binary + # Defaults to false + watchIngressWithoutClass: false + + # -- Process IngressClass per name (additionally as per spec.controller). + ingressClassByName: false + + # -- This configuration defines if Ingress Controller should allow users to set + # their own *-snippet annotations, otherwise this is forbidden / dropped + # when users add those annotations. + # Global snippets in ConfigMap are still respected + allowSnippetAnnotations: true + + # -- Required for use with CNI based kubernetes installations (such as ones set up by kubeadm), + # since CNI and hostport don't mix yet. Can be deprecated once https://github.com/kubernetes/kubernetes/issues/23920 + # is merged + hostNetwork: false + + ## Use host ports 80 and 443 + ## Disabled by default + hostPort: + # -- Enable 'hostPort' or not + enabled: false + ports: + # -- 'hostPort' http port + http: 80 + # -- 'hostPort' https port + https: 443 + + # -- Election ID to use for status update + electionID: ingress-controller-leader + + ## This section refers to the creation of the IngressClass resource + ## IngressClass resources are supported since k8s >= 1.18 and required since k8s >= 1.19 + ingressClassResource: + # -- Name of the ingressClass + name: nginx + # -- Is this ingressClass enabled or not + enabled: true + # -- Is this the default ingressClass for the cluster + default: false + # -- Controller-value of the controller that is processing this ingressClass + controllerValue: "k8s.io/ingress-nginx" + + # -- Parameters is a link to a custom resource containing additional + # configuration for the controller. This is optional if the controller + # does not require extra parameters. + parameters: {} + + # -- For backwards compatibility with ingress.class annotation, use ingressClass. + # Algorithm is as follows, first ingressClassName is considered, if not present, controller looks for ingress.class annotation + ingressClass: nginx + + # -- Labels to add to the pod container metadata + podLabels: {} + # key: value + + # -- Security Context policies for controller pods + podSecurityContext: {} + + # -- See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for notes on enabling and using sysctls + sysctls: {} + # sysctls: + # "net.core.somaxconn": "8192" + + # -- Allows customization of the source of the IP address or FQDN to report + # in the ingress status field. By default, it reads the information provided + # by the service. If disable, the status field reports the IP address of the + # node or nodes where an ingress controller pod is running. + publishService: + # -- Enable 'publishService' or not + enabled: true + # -- Allows overriding of the publish service to bind to + # Must be / + pathOverride: "" + + # Limit the scope of the controller to a specific namespace + scope: + # -- Enable 'scope' or not + enabled: false + # -- Namespace to limit the controller to; defaults to $(POD_NAMESPACE) + namespace: "" + # -- When scope.enabled == false, instead of watching all namespaces, we watching namespaces whose labels + # only match with namespaceSelector. Format like foo=bar. Defaults to empty, means watching all namespaces. + namespaceSelector: "" + + # -- Allows customization of the configmap / nginx-configmap namespace; defaults to $(POD_NAMESPACE) + configMapNamespace: "" + + tcp: + # -- Allows customization of the tcp-services-configmap; defaults to $(POD_NAMESPACE) + configMapNamespace: "" + # -- Annotations to be added to the tcp config configmap + annotations: {} + + udp: + # -- Allows customization of the udp-services-configmap; defaults to $(POD_NAMESPACE) + configMapNamespace: "" + # -- Annotations to be added to the udp config configmap + annotations: {} + + # -- Maxmind license key to download GeoLite2 Databases. + ## https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases + maxmindLicenseKey: "" + + # -- Additional command line arguments to pass to nginx-ingress-controller + # E.g. to specify the default SSL certificate you can use + extraArgs: {} + ## extraArgs: + ## default-ssl-certificate: "/" + + # -- Additional environment variables to set + extraEnvs: [] + # extraEnvs: + # - name: FOO + # valueFrom: + # secretKeyRef: + # key: FOO + # name: secret-resource + + # -- Use a `DaemonSet` or `Deployment` + kind: Deployment + + # -- Annotations to be added to the controller Deployment or DaemonSet + ## + annotations: {} + # keel.sh/pollSchedule: "@every 60m" + + # -- Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels + ## + labels: {} + # keel.sh/policy: patch + # keel.sh/trigger: poll + + + # -- The update strategy to apply to the Deployment or DaemonSet + ## + updateStrategy: {} + # rollingUpdate: + # maxUnavailable: 1 + # type: RollingUpdate + + # -- `minReadySeconds` to avoid killing pods before we are ready + ## + minReadySeconds: 0 + + + # -- Node tolerations for server scheduling to nodes with taints + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- Affinity and anti-affinity rules for server scheduling to nodes + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + # # An example of preferred pod anti-affinity, weight is in the range 1-100 + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight: 100 + # podAffinityTerm: + # labelSelector: + # matchExpressions: + # - key: app.kubernetes.io/name + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/instance + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/component + # operator: In + # values: + # - controller + # topologyKey: kubernetes.io/hostname + + # # An example of required pod anti-affinity + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: app.kubernetes.io/name + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/instance + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/component + # operator: In + # values: + # - controller + # topologyKey: "kubernetes.io/hostname" + + # -- Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: failure-domain.beta.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app.kubernetes.io/instance: ingress-nginx-internal + + # -- `terminationGracePeriodSeconds` to avoid killing pods before we are ready + ## wait up to five minutes for the drain of connections + ## + terminationGracePeriodSeconds: 300 + + # -- Node labels for controller pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: + kubernetes.io/os: linux + + ## Liveness and readiness probe values + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes + ## + ## startupProbe: + ## httpGet: + ## # should match container.healthCheckPath + ## path: "/healthz" + ## port: 10254 + ## scheme: HTTP + ## initialDelaySeconds: 5 + ## periodSeconds: 5 + ## timeoutSeconds: 2 + ## successThreshold: 1 + ## failureThreshold: 5 + livenessProbe: + httpGet: + # should match container.healthCheckPath + path: "/healthz" + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + # should match container.healthCheckPath + path: "/healthz" + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + + + # -- Path of the health check endpoint. All requests received on the port defined by + # the healthz-port parameter are forwarded internally to this path. + healthCheckPath: "/healthz" + + # -- Address to bind the health check endpoint. + # It is better to set this option to the internal node address + # if the ingress nginx controller is running in the `hostNetwork: true` mode. + healthCheckHost: "" + + # -- Annotations to be added to controller pods + ## + podAnnotations: {} + + replicaCount: 1 + + minAvailable: 1 + + ## Define requests resources to avoid probe issues due to CPU utilization in busy nodes + ## ref: https://github.com/kubernetes/ingress-nginx/issues/4735#issuecomment-551204903 + ## Ideally, there should be no limits. + ## https://engineering.indeedblog.com/blog/2019/12/cpu-throttling-regression-fix/ + resources: + ## limits: + ## cpu: 100m + ## memory: 90Mi + requests: + cpu: 100m + memory: 90Mi + + # Mutually exclusive with keda autoscaling + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 11 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 + behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + + autoscalingTemplate: [] + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + # - type: Pods + # pods: + # metric: + # name: nginx_ingress_controller_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m + + # Mutually exclusive with hpa autoscaling + keda: + apiVersion: "keda.sh/v1alpha1" + ## apiVersion changes with keda 1.x vs 2.x + ## 2.x = keda.sh/v1alpha1 + ## 1.x = keda.k8s.io/v1alpha1 + enabled: false + minReplicas: 1 + maxReplicas: 11 + pollingInterval: 30 + cooldownPeriod: 300 + restoreToOriginalReplicaCount: false + scaledObject: + annotations: {} + # Custom annotations for ScaledObject resource + # annotations: + # key: value + triggers: [] + # - type: prometheus + # metadata: + # serverAddress: http://:9090 + # metricName: http_requests_total + # threshold: '100' + # query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + + behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + + # -- Enable mimalloc as a drop-in replacement for malloc. + ## ref: https://github.com/microsoft/mimalloc + ## + enableMimalloc: true + + ## Override NGINX template + customTemplate: + configMapName: "" + configMapKey: "" + + service: + enabled: true + + # -- If enabled is adding an appProtocol option for Kubernetes service. An appProtocol field replacing annotations that were + # using for setting a backend protocol. Here is an example for AWS: service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http + # It allows choosing the protocol for each backend specified in the Kubernetes service. + # See the following GitHub issue for more details about the purpose: https://github.com/kubernetes/kubernetes/issues/40244 + # Will be ignored for Kubernetes versions older than 1.20 + ## + appProtocol: true + + annotations: {} + labels: {} + # clusterIP: "" + + # -- List of IP addresses at which the controller services are available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + + enableHttp: true + enableHttps: true + + ## Set external traffic policy to: "Local" to preserve source IP on providers supporting it. + ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer + # externalTrafficPolicy: "" + + ## Must be either "None" or "ClientIP" if set. Kubernetes will default to "None". + ## Ref: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + # sessionAffinity: "" + + ## Specifies the health check node port (numeric port number) for the service. If healthCheckNodePort isn’t specified, + ## the service controller allocates a port from your cluster’s NodePort range. + ## Ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + # healthCheckNodePort: 0 + + # -- Represents the dual-stack-ness requested or required by this Service. Possible values are + # SingleStack, PreferDualStack or RequireDualStack. + # The ipFamilies and clusterIPs fields depend on the value of this field. + ## Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/ + ipFamilyPolicy: "SingleStack" + + # -- List of IP families (e.g. IPv4, IPv6) assigned to the service. This field is usually assigned automatically + # based on cluster configuration and the ipFamilyPolicy field. + ## Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/ + ipFamilies: + - IPv4 + + ports: + http: 80 + https: 443 + + targetPorts: + http: http + https: https + + type: LoadBalancer + + ## type: NodePort + ## nodePorts: + ## http: 32080 + ## https: 32443 + ## tcp: + ## 8080: 32808 + nodePorts: + http: "" + https: "" + tcp: {} + udp: {} + + external: + enabled: true + + internal: + # -- Enables an additional internal load balancer (besides the external one). + enabled: false + # -- Annotations are mandatory for the load balancer to come up. Varies with the cloud service. + annotations: {} + + # loadBalancerIP: "" + + # -- Restrict access For LoadBalancer service. Defaults to 0.0.0.0/0. + loadBalancerSourceRanges: [] + + ## Set external traffic policy to: "Local" to preserve source IP on + ## providers supporting it + ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer + # externalTrafficPolicy: "" + + # -- Additional containers to be added to the controller pod. + # See https://github.com/lemonldap-ng-controller/lemonldap-ng-controller as example. + extraContainers: [] + # - name: my-sidecar + # image: nginx:latest + # - name: lemonldap-ng-controller + # image: lemonldapng/lemonldap-ng-controller:0.2.0 + # args: + # - /lemonldap-ng-controller + # - --alsologtostderr + # - --configmap=$(POD_NAMESPACE)/lemonldap-ng-configuration + # env: + # - name: POD_NAME + # valueFrom: + # fieldRef: + # fieldPath: metadata.name + # - name: POD_NAMESPACE + # valueFrom: + # fieldRef: + # fieldPath: metadata.namespace + # volumeMounts: + # - name: copy-portal-skins + # mountPath: /srv/var/lib/lemonldap-ng/portal/skins + + # -- Additional volumeMounts to the controller main container. + extraVolumeMounts: [] + # - name: copy-portal-skins + # mountPath: /var/lib/lemonldap-ng/portal/skins + + # -- Additional volumes to the controller pod. + extraVolumes: [] + # - name: copy-portal-skins + # emptyDir: {} + + # -- Containers, which are run before the app containers are started. + extraInitContainers: [] + # - name: init-myservice + # image: busybox + # command: ['sh', '-c', 'until nslookup myservice; do echo waiting for myservice; sleep 2; done;'] + + extraModules: [] + ## Modules, which are mounted into the core nginx image + # - name: opentelemetry + # image: busybox + # + # The image must contain a `/usr/local/bin/init_module.sh` executable, which + # will be executed as initContainers, to move its config files within the + # mounted volume. + + admissionWebhooks: + annotations: {} + # ignore-check.kube-linter.io/no-read-only-rootfs: "This deployment needs write access to root filesystem". + + ## Additional annotations to the admission webhooks. + ## These annotations will be added to the ValidatingWebhookConfiguration and + ## the Jobs Spec of the admission webhooks. + enabled: true + failurePolicy: Fail + # timeoutSeconds: 10 + port: 8443 + certificate: "/usr/local/certificates/cert" + key: "/usr/local/certificates/key" + namespaceSelector: {} + objectSelector: {} + # -- Labels to be added to admission webhooks + labels: {} + + # -- Use an existing PSP instead of creating one + existingPsp: "" + + service: + annotations: {} + # clusterIP: "" + externalIPs: [] + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 443 + type: ClusterIP + + createSecretJob: + resources: {} + # limits: + # cpu: 10m + # memory: 20Mi + # requests: + # cpu: 10m + # memory: 20Mi + + patchWebhookJob: + resources: {} + + patch: + enabled: true + image: + registry: k8s.gcr.io + image: ingress-nginx/kube-webhook-certgen + ## for backwards compatibility consider setting the full image url via the repository value below + ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + ## repository: + tag: v1.1.1 + digest: sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660 + pullPolicy: IfNotPresent + # -- Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + podAnnotations: {} + nodeSelector: + kubernetes.io/os: linux + tolerations: [] + # -- Labels to be added to patch job resources + labels: {} + runAsUser: 2000 + fsGroup: 2000 + + metrics: + port: 10254 + # if this port is changed, change healthz-port: in extraArgs: accordingly + enabled: false + + service: + annotations: {} + # prometheus.io/scrape: "true" + # prometheus.io/port: "10254" + + # clusterIP: "" + + # -- List of IP addresses at which the stats-exporter service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 10254 + type: ClusterIP + # externalTrafficPolicy: "" + # nodePort: "" + + serviceMonitor: + enabled: false + additionalLabels: {} + ## The label to use to retrieve the job name from. + ## jobLabel: "app.kubernetes.io/name" + namespace: "" + namespaceSelector: {} + ## Default: scrape .Release.Namespace only + ## To scrape all, use the following: + ## namespaceSelector: + ## any: true + scrapeInterval: 30s + # honorLabels: true + targetLabels: [] + relabelings: [] + metricRelabelings: [] + + prometheusRule: + enabled: false + additionalLabels: {} + # namespace: "" + rules: [] + # # These are just examples rules, please adapt them to your needs + # - alert: NGINXConfigFailed + # expr: count(nginx_ingress_controller_config_last_reload_successful == 0) > 0 + # for: 1s + # labels: + # severity: critical + # annotations: + # description: bad ingress config - nginx config test failed + # summary: uninstall the latest ingress changes to allow config reloads to resume + # - alert: NGINXCertificateExpiry + # expr: (avg(nginx_ingress_controller_ssl_expire_time_seconds) by (host) - time()) < 604800 + # for: 1s + # labels: + # severity: critical + # annotations: + # description: ssl certificate(s) will expire in less then a week + # summary: renew expiring certificates to avoid downtime + # - alert: NGINXTooMany500s + # expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5 + # for: 1m + # labels: + # severity: warning + # annotations: + # description: Too many 5XXs + # summary: More than 5% of all requests returned 5XX, this requires your attention + # - alert: NGINXTooMany400s + # expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"4.+"} ) / sum(nginx_ingress_controller_requests) ) > 5 + # for: 1m + # labels: + # severity: warning + # annotations: + # description: Too many 4XXs + # summary: More than 5% of all requests returned 4XX, this requires your attention + + # -- Improve connection draining when ingress controller pod is deleted using a lifecycle hook: + # With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds + # to 300, allowing the draining of connections up to five minutes. + # If the active connections end before that, the pod will terminate gracefully at that time. + # To effectively take advantage of this feature, the Configmap feature + # worker-shutdown-timeout new value is 240s instead of 10s. + ## + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + + priorityClassName: "" + +# -- Rollback limit +## +revisionHistoryLimit: 10 + +## Default 404 backend +## +defaultBackend: + ## + enabled: false + + name: defaultbackend + image: + registry: k8s.gcr.io + image: defaultbackend-amd64 + ## for backwards compatibility consider setting the full image url via the repository value below + ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + ## repository: + tag: "1.5" + pullPolicy: IfNotPresent + # nobody user -> uid 65534 + runAsUser: 65534 + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + + # -- Use an existing PSP instead of creating one + existingPsp: "" + + extraArgs: {} + + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + # -- Additional environment variables to set for defaultBackend pods + extraEnvs: [] + + port: 8080 + + ## Readiness and liveness probes for default backend + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ + ## + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + failureThreshold: 6 + initialDelaySeconds: 0 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + + # -- Node tolerations for server scheduling to nodes with taints + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + affinity: {} + + # -- Security Context policies for controller pods + # See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for + # notes on enabling and using sysctls + ## + podSecurityContext: {} + + # -- Security Context policies for controller main container. + # See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for + # notes on enabling and using sysctls + ## + containerSecurityContext: {} + + # -- Labels to add to the pod container metadata + podLabels: {} + # key: value + + # -- Node labels for default backend pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: + kubernetes.io/os: linux + + # -- Annotations to be added to default backend pods + ## + podAnnotations: {} + + replicaCount: 1 + + minAvailable: 1 + + resources: {} + # limits: + # cpu: 10m + # memory: 20Mi + # requests: + # cpu: 10m + # memory: 20Mi + + extraVolumeMounts: [] + ## Additional volumeMounts to the default backend container. + # - name: copy-portal-skins + # mountPath: /var/lib/lemonldap-ng/portal/skins + + extraVolumes: [] + ## Additional volumes to the default backend pod. + # - name: copy-portal-skins + # emptyDir: {} + + autoscaling: + annotations: {} + enabled: false + minReplicas: 1 + maxReplicas: 2 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 50 + + service: + annotations: {} + + # clusterIP: "" + + # -- List of IP addresses at which the default backend service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 80 + type: ClusterIP + + priorityClassName: "" + # -- Labels to be added to the default backend resources + labels: {} + +## Enable RBAC as per https://github.com/kubernetes/ingress-nginx/blob/main/docs/deploy/rbac.md and https://github.com/kubernetes/ingress-nginx/issues/266 +rbac: + create: true + scope: false + +## If true, create & use Pod Security Policy resources +## https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +podSecurityPolicy: + enabled: false + +serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + # -- Annotations for the controller service account + annotations: {} + +# -- Optional array of imagePullSecrets containing private registry credentials +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# - name: secretName + +# -- TCP service key:value pairs +## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md +## +tcp: {} +# 8080: "default/example-tcp-svc:9000" + +# -- UDP service key:value pairs +## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md +## +udp: {} +# 53: "kube-system/kube-dns:53" + +# -- (string) A base64-encoded Diffie-Hellman parameter. +# This can be generated with: `openssl dhparam 4096 2> /dev/null | base64` +## Ref: https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/ssl-dh-param +dhParam: diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt deleted file mode 120000 index 12e23824a..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt +++ /dev/null @@ -1 +0,0 @@ -../../../files/site.crt \ No newline at end of file diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key deleted file mode 120000 index 3805a27d1..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key +++ /dev/null @@ -1 +0,0 @@ -../../../files/site.key \ No newline at end of file diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml deleted file mode 100644 index 6322318e3..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml +++ /dev/null @@ -1,190 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx - namespace: {{ .Release.Namespace }} -data: - location.list: |- - location ~* /general_stats { - deny all; - } - location /healthz { - return 200 'OK'; - } - location ~ ^/(mobs|sessions-assets|frontend|static|sourcemaps|ios-images)/ { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $origin_forwarded_ip; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - - proxy_connect_timeout 300; - # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 - proxy_http_version 1.1; - proxy_set_header Connection ""; - chunked_transfer_encoding off; - - proxy_pass http://minio.db.svc.cluster.local:9000; - } - - location /minio/ { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_pass http://minio.db.svc.cluster.local:9000; - } - location /ingest/ { - rewrite ^/ingest/(.*) /$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header X-Forwarded-For $origin_forwarded_ip; - proxy_set_header X-Forwarded-Host $origin_forwarded_ip; - proxy_set_header X-Real-IP $origin_forwarded_ip; - proxy_set_header Host $host; - proxy_pass http://http-openreplay.app.svc.cluster.local; - proxy_read_timeout 300; - proxy_connect_timeout 120; - proxy_send_timeout 300; - } - location /grafana { - set $target http://monitoring-grafana.monitoring.svc.cluster.local; - rewrite ^/grafana/(.*) /$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_pass $target; - } - location /api/ { - rewrite ^/api/(.*) /$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $origin_proto; - proxy_pass http://chalice-openreplay.app.svc.cluster.local:8000; - } - location /assist/ { - rewrite ^/assist/(.*) /$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $origin_forwarded_ip; - proxy_pass http://utilities-openreplay.app.svc.cluster.local:9000; - } - location /ws-assist/ { - rewrite ^/ws-assist/(.*) /$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $origin_forwarded_ip; - proxy_set_header X-Real-IP $origin_forwarded_ip; - proxy_pass http://utilities-openreplay.app.svc.cluster.local:9001; - } - location /assets/ { - rewrite ^/assets/(.*) /sessions-assets/$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_pass http://minio.db.svc.cluster.local:9000; - } - location / { - index /index.html; - rewrite ^((?!.(js|css|png|svg|jpg|woff|woff2)).)*$ /frontend/index.html break; - proxy_http_version 1.1; - proxy_set_header Connection ""; - include /etc/nginx/conf.d/compression.conf; - proxy_set_header Host $http_host; - proxy_pass http://minio.db.svc.cluster.local:9000/frontend/; - proxy_intercept_errors on; # see http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_intercept_errors - error_page 404 =200 /index.html; - } - compression.conf: |- - # Compression - gzip on; - gzip_comp_level 5; - gzip_min_length 256; # 256Bytes - gzip_proxied any; - gzip_vary on; - # Content types for compression - gzip_types - application/atom+xml - application/javascript - application/json - application/ld+json - application/manifest+json - application/rss+xml - application/vnd.geo+json - application/vnd.ms-fontobject - application/x-font-ttf - application/x-web-app-manifest+json - application/xhtml+xml - application/xml - font/opentype - image/bmp - image/svg+xml - image/x-icon - text/cache-manifest - text/css - text/plain - ; - - sites.conf: |- - # Ref: https://github.com/openresty/openresty/#resolvconf-parsing - resolver local=on; - # Need real ip address for flags in replay. - # Some LBs will forward real ips as x-forwarded-for - # So making that as priority - map $http_x_forwarded_for $origin_forwarded_ip { - ~^(\d+\.\d+\.\d+\.\d+) $1; - default $remote_addr; - } - map $http_upgrade $connection_upgrade { - default upgrade; - '' close; - } - map $http_x_forwarded_proto $origin_proto { - default $http_x_forwarded_proto; - '' $scheme; - } - # Default server for helath check - server { - listen 80 default_server; - listen [::]:80; - location /healthz { - return 200 'OK'; - } - } - - upstream utilities-pool { - # comment this line if you have multiple pods - server utilities-openreplay-headless.app:9001; - # uncomment and change these lines if you have multiple pods - # server POD1_IP:9001; - # server POD2_IP:9001; - hash $origin_forwarded_ip consistent; - } - - server { - listen 80; - listen [::]:80; - server_name {{ .Values.global.domainName }}; - {{ .Values.customServerConfigs }} - include /etc/nginx/conf.d/location.list; - client_max_body_size 10M; - } - server { - listen 443 ssl; - server_name {{ .Values.global.domainName }}; - ssl_certificate /etc/secrets/site.crt; - ssl_certificate_key /etc/secrets/site.key; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA HIGH !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; - include /etc/nginx/conf.d/location.list; - client_max_body_size 10M; - } - diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml deleted file mode 100644 index 63cfce077..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "nginx-ingress.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml deleted file mode 100644 index 91b7cc09c..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: ssl -data: - ca.crt: '' - site.crt: '{{ .Files.Get "files/site.crt" | b64enc }}' - site.key: '{{ .Files.Get "files/site.key" | b64enc }}' diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml deleted file mode 100644 index f20d4fc38..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "nginx-ingress.fullname" . }} - labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} - annotations: - {{- with .Values.service.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.service.type }} - {{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort")}} - # Make sure to get client ip - externalTrafficPolicy: Local - {{- end}} - ports: - {{- range .Values.service.ports }} - - port: {{ .port }} - targetPort: {{ .targetPort }} - protocol: TCP - name: {{ .name }} - {{- end }} - selector: - {{- include "nginx-ingress.selectorLabels" . | nindent 4 }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "nginx-ingress.fullname" . }}-metrics - labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} - openreplay/monitoring: nginx-ingress-metrics -spec: - selector: - {{- include "nginx-ingress.selectorLabels" . | nindent 4 }} - ports: - - name: metrics - port: 9145 - targetPort: 9145 - protocol: TCP diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml deleted file mode 100644 index 074cec518..000000000 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "nginx-ingress.fullname" . }}-test-connection" - labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "nginx-ingress.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/scripts/helmcharts/openreplay/charts/utilities/.helmignore b/scripts/helmcharts/openreplay/charts/peers/.helmignore similarity index 100% rename from scripts/helmcharts/openreplay/charts/utilities/.helmignore rename to scripts/helmcharts/openreplay/charts/peers/.helmignore diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml b/scripts/helmcharts/openreplay/charts/peers/Chart.yaml similarity index 81% rename from scripts/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml rename to scripts/helmcharts/openreplay/charts/peers/Chart.yaml index 8aa1dcf82..b482f5986 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: nginx-ingress +name: peers description: A Helm chart for Kubernetes # A chart can be either an 'application' or a 'library' chart. @@ -7,8 +7,8 @@ description: A Helm chart for Kubernetes # Application charts are a collection of templates that can be packaged into versioned archives # to be deployed. # -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering +# Library charts provide useful peers or functions for the chart developer. They're included as +# a dependency of application charts to inject those peers and functions into the rendering # pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/peers/templates/NOTES.txt similarity index 76% rename from scripts/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/peers/templates/NOTES.txt index 8125afe9c..fbe8dd585 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt +++ b/scripts/helmcharts/openreplay/charts/peers/templates/NOTES.txt @@ -6,16 +6,16 @@ {{- end }} {{- end }} {{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "nginx-ingress.fullname" . }}) + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "peers.fullname" . }}) export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo http://$NODE_IP:$NODE_PORT {{- else if contains "LoadBalancer" .Values.service.type }} NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "nginx-ingress.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "nginx-ingress.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "peers.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "peers.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") echo http://$SERVICE_IP:{{ .Values.service.port }} {{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "nginx-ingress.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "peers.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") echo "Visit http://127.0.0.1:8080 to use your application" kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/peers/templates/_helpers.tpl similarity index 73% rename from scripts/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/peers/templates/_helpers.tpl index b0f2808f1..9dce56cdd 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl +++ b/scripts/helmcharts/openreplay/charts/peers/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "nginx-ingress.name" -}} +{{- define "peers.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "nginx-ingress.fullname" -}} +{{- define "peers.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "nginx-ingress.chart" -}} +{{- define "peers.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "nginx-ingress.labels" -}} -helm.sh/chart: {{ include "nginx-ingress.chart" . }} -{{ include "nginx-ingress.selectorLabels" . }} +{{- define "peers.labels" -}} +helm.sh/chart: {{ include "peers.chart" . }} +{{ include "peers.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "nginx-ingress.selectorLabels" -}} -app.kubernetes.io/name: {{ include "nginx-ingress.name" . }} +{{- define "peers.selectorLabels" -}} +app.kubernetes.io/name: {{ include "peers.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Create the name of the service account to use */}} -{{- define "nginx-ingress.serviceAccountName" -}} +{{- define "peers.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} -{{- default (include "nginx-ingress.fullname" .) .Values.serviceAccount.name }} +{{- default (include "peers.fullname" .) .Values.serviceAccount.name }} {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml similarity index 53% rename from scripts/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml index 95c05aa67..8ad4364d8 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml @@ -1,71 +1,57 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "nginx-ingress.fullname" . }} + name: {{ include "peers.fullname" . }} labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} + {{- include "peers.labels" . | nindent 4 }} spec: {{- if not .Values.autoscaling.enabled }} replicas: {{ .Values.replicaCount }} {{- end }} selector: matchLabels: - {{- include "nginx-ingress.selectorLabels" . | nindent 6 }} + {{- include "peers.selectorLabels" . | nindent 6 }} template: metadata: - annotations: - nginxRolloutID: {{ randAlphaNum 5 | quote }} # Restart nginx after every deployment {{- with .Values.podAnnotations }} + annotations: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "nginx-ingress.selectorLabels" . | nindent 8 }} + {{- include "peers.selectorLabels" . | nindent 8 }} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "nginx-ingress.serviceAccountName" . }} + serviceAccountName: {{ include "peers.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: S3_KEY + value: {{ .Values.global.s3.accessKey }} + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} ports: - - name: https - containerPort: 443 + {{- range $key, $val := .Values.service.ports }} + - name: {{ $key }} + containerPort: {{ $val }} protocol: TCP - - name: http - containerPort: 80 - protocol: TCP - - name: metrics - containerPort: 9145 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: http - readinessProbe: - httpGet: - path: /healthz - port: http + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: nginx - mountPath: /etc/nginx/conf.d/ - - name: ssl - mountPath: /etc/secrets/ - volumes: - - name: nginx - configMap: - name: nginx - - name: ssl - secret: - secretName: ssl {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/hpa.yaml similarity index 83% rename from scripts/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/peers/templates/hpa.yaml index 348f8f95b..7191631ee 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/templates/hpa.yaml @@ -2,14 +2,14 @@ apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: - name: {{ include "nginx-ingress.fullname" . }} + name: {{ include "peers.fullname" . }} labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} + {{- include "peers.labels" . | nindent 4 }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ include "nginx-ingress.fullname" . }} + name: {{ include "peers.fullname" . }} minReplicas: {{ .Values.autoscaling.minReplicas }} maxReplicas: {{ .Values.autoscaling.maxReplicas }} metrics: diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/ingress.yaml new file mode 100644 index 000000000..0da5eca0d --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/templates/ingress.yaml @@ -0,0 +1,34 @@ +{{- if .Values.ingress.enabled }} +{{- $fullName := include "peers.fullname" . -}} +{{- $peerjsSvcPort := .Values.service.ports.peerjs -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "peers.labels" . | nindent 4 }} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: "{{ tpl .Values.ingress.className . }}" + tls: + - hosts: + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} + rules: + - host: {{ .Values.global.domainName }} + http: + paths: + - pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ $peerjsSvcPort }} + path: /assist/(.*) +{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/service.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/service.yaml new file mode 100644 index 000000000..3a069c2c2 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/templates/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "peers.fullname" . }} + labels: + {{- include "peers.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + {{- range $key, $val := .Values.service.ports }} + - port: {{ $val }} + targetPort: {{ $key }} + protocol: TCP + name: {{ $key }} + {{- end}} + selector: + {{- include "peers.selectorLabels" . | nindent 4 }} diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/serviceaccount.yaml similarity index 65% rename from scripts/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/peers/templates/serviceaccount.yaml index bc0091029..4b6cc04fd 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/templates/serviceaccount.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "nginx-ingress.serviceAccountName" . }} + name: {{ include "peers.serviceAccountName" . }} labels: - {{- include "nginx-ingress.labels" . | nindent 4 }} + {{- include "peers.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/tests/test-connection.yaml new file mode 100644 index 000000000..e8dd6a063 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/peers/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "peers.fullname" . }}-test-connection" + labels: + {{- include "peers.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "peers.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml b/scripts/helmcharts/openreplay/charts/peers/values.yaml similarity index 63% rename from scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml rename to scripts/helmcharts/openreplay/charts/peers/values.yaml index 1f7169c0d..ab74348fc 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/values.yaml @@ -1,18 +1,18 @@ -# Default values for nginx-ingress. +# Default values for openreplay. # This is a YAML-formatted file. # Declare variables to be passed into your templates. replicaCount: 1 image: - repository: nginx + repository: rg.fr-par.scw.cloud/foss/peers pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "" imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" +nameOverride: "peers" +fullnameOverride: "peers" serviceAccount: # Specifies whether a service account should be created @@ -36,32 +36,23 @@ securityContext: {} # runAsNonRoot: true # runAsUser: 1000 +#service: +# type: ClusterIP +# port: 9000 + service: - annotations: {} - type: LoadBalancer + type: ClusterIP ports: - - port: 80 - targetPort: http - name: http - - port: 443 - targetPort: https - name: https + peerjs: 9000 ingress: - enabled: false - className: "" + enabled: true + className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}" annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local + tls: + secretName: openreplay-ssl resources: {} # We usually recommend not to specify default resources and to leave this as a conscious @@ -78,32 +69,15 @@ resources: {} autoscaling: enabled: false minReplicas: 1 - maxReplicas: 100 + maxReplicas: 5 targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 +env: + debug: 0 + nodeSelector: {} tolerations: [] affinity: {} - -healthProbes: - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: http - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: http - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml deleted file mode 100644 index 567cac846..000000000 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "utilities.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "utilities.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/service.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/service.yaml deleted file mode 100644 index 8b1dc8753..000000000 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/service.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "utilities.fullname" . }} - labels: - {{- include "utilities.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - {{- range $key, $val := .Values.service.ports }} - - port: {{ $val }} - targetPort: {{ $key }} - protocol: TCP - name: {{ $key }} - {{- end}} - selector: - {{- include "utilities.selectorLabels" . | nindent 4 }} - ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "utilities.fullname" . }}-headless - labels: - {{- include "utilities.labels" . | nindent 4 }} -spec: - type: ClusterIP - clusterIP: None - ports: - {{- range $key, $val := .Values.service.ports }} - - port: {{ $val }} - targetPort: {{ $key }} - protocol: TCP - name: {{ $key }} - {{- end}} - selector: - {{- include "utilities.selectorLabels" . | nindent 4 }} diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml deleted file mode 100644 index 44b72f68d..000000000 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "utilities.fullname" . }}-test-connection" - labels: - {{- include "utilities.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "utilities.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/scripts/helmcharts/openreplay/values.yaml b/scripts/helmcharts/openreplay/values.yaml index 7fcd4121e..f702c1a49 100644 --- a/scripts/helmcharts/openreplay/values.yaml +++ b/scripts/helmcharts/openreplay/values.yaml @@ -110,5 +110,7 @@ sink: fullnameOverride: sink-openreplay storage: fullnameOverride: storage-openreplay -utilities: - fullnameOverride: utilities-openreplay +assist: + fullnameOverride: assist-openreplay +peers: + fullnameOverride: peers-openreplay diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 24ea9344d..7f5aaf424 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -41,8 +41,31 @@ minio: accessKey: "changeMeMinioAccessKey" secretKey: "changeMeMinioPassword" +ingress-nginx: &ingress-nginx + controller: + ingressClassResource: + # -- Name of the ingressClass + name: openreplay + # -- For backwards compatibility with ingress.class annotation, use ingressClass. + # Algorithm is as follows, first ingressClassName is considered, if not present, controller looks for ingress.class annotation + ingressClass: openreplay + service: + externalTrafficPolicy: "Local" + extraArgs: + default-ssl-certificate: "app/openreplay-ssl" + config: + enable-real-ip: true + forwarded-for-header: "proxy_protocol" + # Ref: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#max-worker-connections + max-worker-connections: 0 + # If you want the ssl redirection + # Comment the following line + ssl-redirect: false + force-ssl-redirect: false + # Application specific variables global: + ingress: *ingress-nginx postgresql: *postgres kafka: *kafka redis: *redis @@ -55,7 +78,7 @@ global: # if you're using one node installation, where # you're using local s3, make sure these variables # are same as minio.global.minio.accesskey and secretKey - accessKey: "changeMeMinioAccessKey" + accessKey: "YkkPAPYjogRlicqvCuNSHkfsdGtCCq" secretKey: "changeMeMinioPassword" email: emailHost: '' @@ -69,7 +92,7 @@ global: emailFrom: 'OpenReplay' enterpriseEditionLicense: "" - domainName: "" + domainName: "foss.openreplay.com" # If there is multiple nodes in the kubernetes cluster, # we'll have to create a NFS share PVC for both the containers to share data. @@ -109,14 +132,3 @@ chalice: # requests: # cpu: 512m # memory: 2056Mi - -## Changes to nginx -# -# nginx-ingress: -# # Key and certificate files must be named site.key and site.crt -# # and copied to ../openreplay/files/ -# sslKey: site.key -# sslCert: site.crt -# # Redirecting http to https -# customServerConfigs: | -# return 301 https://$host$request_uri; diff --git a/sourcemap-reader/.gitignore b/sourcemap-reader/.gitignore new file mode 100644 index 000000000..a4b05b411 --- /dev/null +++ b/sourcemap-reader/.gitignore @@ -0,0 +1,6 @@ +.idea +node_modules +npm-debug.log +.cache +test.html +/utils/ diff --git a/sourcemap-reader/package-lock.json b/sourcemap-reader/package-lock.json new file mode 100644 index 000000000..e2871acb2 --- /dev/null +++ b/sourcemap-reader/package-lock.json @@ -0,0 +1,1113 @@ +{ + "name": "sourcemaps-reader", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "sourcemaps-reader", + "version": "1.0.0", + "license": "Elastic License 2.0 (ELv2)", + "dependencies": { + "aws-sdk": "^2.992.0", + "express": "^4.17.1", + "source-map": "^0.7.3" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "node_modules/aws-sdk": { + "version": "2.1087.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", + "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.16.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.9.7", + "raw-body": "2.4.3", + "type-is": "~1.6.18" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/express": { + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", + "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.19.2", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.4.2", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.9.7", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/jmespath": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", + "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", + "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.34", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", + "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", + "dependencies": { + "mime-db": "1.51.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "node_modules/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==", + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", + "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "node_modules/send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "dependencies": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "1.8.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/serve-static": { + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "node_modules/source-map": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", + "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "node_modules/xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", + "engines": { + "node": ">=4.0" + } + } + }, + "dependencies": { + "accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "requires": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + } + }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "aws-sdk": { + "version": "2.1087.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", + "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.16.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + } + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", + "requires": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.9.7", + "raw-body": "2.4.3", + "type-is": "~1.6.18" + } + }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "requires": { + "safe-buffer": "5.2.1" + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "express": { + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", + "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", + "requires": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.19.2", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.4.2", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.9.7", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + } + }, + "finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + } + }, + "forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + }, + "http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "jmespath": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", + "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==" + }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + }, + "mime-db": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", + "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==" + }, + "mime-types": { + "version": "2.1.34", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", + "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", + "requires": { + "mime-db": "1.51.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" + }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "requires": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + } + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" + }, + "raw-body": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", + "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", + "requires": { + "bytes": "3.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "1.8.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + }, + "serve-static": { + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.2" + } + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "source-map": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", + "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==" + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" + }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } +} diff --git a/sourcemap-reader/package.json b/sourcemap-reader/package.json new file mode 100644 index 000000000..8ed3189bb --- /dev/null +++ b/sourcemap-reader/package.json @@ -0,0 +1,25 @@ +{ + "name": "sourcemaps-reader", + "version": "1.0.0", + "description": "assist server to get live sessions & sourcemaps reader to get stack trace", + "main": "peerjs-server.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node server.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/openreplay/openreplay.git" + }, + "author": "KRAIEM Taha Yassine ", + "license": "Elastic License 2.0 (ELv2)", + "bugs": { + "url": "https://github.com/openreplay/openreplay/issues" + }, + "homepage": "https://github.com/openreplay/openreplay#readme", + "dependencies": { + "aws-sdk": "^2.992.0", + "express": "^4.17.1", + "source-map": "^0.7.3" + } +} diff --git a/sourcemap-reader/server.js b/sourcemap-reader/server.js new file mode 100644 index 000000000..073cb4cfc --- /dev/null +++ b/sourcemap-reader/server.js @@ -0,0 +1,19 @@ +const dumps = require('./utils/HeapSnapshot'); +const sourcemapsReaderServer = require('./servers/sourcemaps-server'); +const express = require('express'); +const {request_logger} = require("./utils/helper"); + +const HOST = '0.0.0.0'; +const PORT = 9000; + +const app = express(); +app.use(request_logger("[wsapp]")); + +app.use('/sourcemaps', sourcemapsReaderServer); +app.use('/heapdump', dumps.router); + +const server = app.listen(PORT, HOST, () => { + console.log(`WS App listening on http://${HOST}:${PORT}`); + console.log('Press Ctrl+C to quit.'); +}); +module.exports = {server}; \ No newline at end of file diff --git a/utilities/servers/sourcemaps-handler.js b/sourcemap-reader/servers/sourcemaps-handler.js similarity index 100% rename from utilities/servers/sourcemaps-handler.js rename to sourcemap-reader/servers/sourcemaps-handler.js diff --git a/utilities/servers/sourcemaps-server.js b/sourcemap-reader/servers/sourcemaps-server.js similarity index 100% rename from utilities/servers/sourcemaps-server.js rename to sourcemap-reader/servers/sourcemaps-server.js diff --git a/tracker/tracker-fetch/README.md b/tracker/tracker-fetch/README.md index b7fca2e4b..53fddcf8e 100644 --- a/tracker/tracker-fetch/README.md +++ b/tracker/tracker-fetch/README.md @@ -1,7 +1,6 @@ -# OpenReplay Tracker Fetch plugin +# Fetch plugin for OpenReplay -Tracker plugin to support tracking of the `fetch` requests payload. -Additionally it populates the requests with `sessionToken` header for backend logging. +This plugin allows you to capture `fetch` payloads and inspect them later on while replaying session recordings. This is very useful for understanding and fixing issues. ## Installation @@ -11,36 +10,120 @@ npm i @openreplay/tracker-fetch ## Usage -Initialize the `@openreplay/tracker` package as usual and load the plugin into it. -Then you can use the provided `fetch` method from the plugin instead of built-in. +Use the provided `fetch` method from the plugin instead of the one built-in. + +### If your website is a Single Page Application (SPA) ```js -import Tracker from '@openreplay/tracker'; +import tracker from '@openreplay/tracker'; import trackerFetch from '@openreplay/tracker-fetch'; -const tracker = new Tracker({ - projectKey: YOUR_PROJECT_KEY, +const tracker = new OpenReplay({ + projectKey: PROJECT_KEY }); +const fetch = tracker.use(trackerFetch(options)); // check list of available options below + tracker.start(); -export const fetch = tracker.use(trackerFetch({ /* options here*/ })); - -fetch('https://my.api.io/resource').then(response => response.json()).then(body => console.log(body)); +fetch('https://myapi.com/').then(response => console.log(response.json())); ``` -Options: -```ts -{ - failuresOnly: boolean, // default false - sessionTokenHeader: string | undefined, // default undefined - ignoreHeaders: Array | boolean, // default [ 'Cookie', 'Set-Cookie', 'Authorization' ] +### If your web app is Server-Side-Rendered (SSR) + +Follow the below example if your app is SSR. Ensure `tracker.start()` is called once the app is started (in `useEffect` or `componentDidMount`). + +```js +import OpenReplay from '@openreplay/tracker/cjs'; +import trackerFetch from '@openreplay/tracker-fetch/cjs'; + +const tracker = new OpenReplay({ + projectKey: PROJECT_KEY +}); +const fetch = tracker.use(trackerFetch(options)); // check list of available options below + +//... +function MyApp() { + useEffect(() => { // use componentDidMount in case of React Class Component + tracker.start(); + + fetch('https://myapi.com/').then(response => console.log(response.json())); + }, []) +//... } ``` -Set `failuresOnly` option to `true` if you want to record only requests with the status code >= 400. +## Options -In case you use [OpenReplay integrations (sentry, bugsnag or others)](https://docs.openreplay.com/integrations), you can use `sessionTokenHeader` option to specify the header name. This header will be appended automatically to the each fetch request and will contain OpenReplay session identificator value. +```js +trackerFetch({ + overrideGlobal: boolean; + failuresOnly: boolean; + sessionTokenHeader: string; + ignoreHeaders: Array | boolean; + sanitiser: (RequestResponseData) => RequestResponseData | null; +}) +``` -You can define list of headers that you don't want to capture with the `ignoreHeaders` options. Set its value to `false` if you want to catch them all (`true` if opposite). By default plugin ignores the list of headers that might be sensetive such as `[ 'Cookie', 'Set-Cookie', 'Authorization' ]`. +- `overrideGlobal`: Overrides the default `window.fetch`. Default: `false`. +- `failuresOnly`: Captures requests having 4xx-5xx HTTP status code. Default: `false`. +- `sessionTokenHeader`: In case you have enabled some of our backend [integrations](/integrations) (i.e. Sentry), you can use this option to specify the header name (i.e. 'X-OpenReplay-SessionToken'). This latter gets appended automatically to each fetch request to contain the OpenReplay sessionToken's value. Default: `undefined`. +- `ignoreHeaders`: Helps define a list of headers you don't wish to capture. Set its value to `false` to capture all of them (`true` if none). Default: `['Cookie', 'Set-Cookie', 'Authorization']` so sensitive headers won't be captured. +- `sanitiser`: Sanitise sensitive data from fetch request/response or ignore request comletely. You can redact fields on the request object by modifying then returning it from the function: +```typescript +interface RequestData { + body: BodyInit | null | undefined; // whatewer you've put in the init.body in fetch(url, init) + headers: Record; +} + +interface ResponseData { + body: string | Object | null; // Object if response is of JSON type + headers: Record; +} + +interface RequestResponseData { + readonly status: number; + readonly method: string; + url: string; + request: RequestData; + response: ResponseData; +} + +sanitiser: (data: RequestResponseData) => { // sanitise the body or headers + if (data.url === "/auth") { + data.request.body = null + } + + if (data.request.headers['x-auth-token']) { // can also use ignoreHeaders option instead + data.request.headers['x-auth-token'] = 'SANITISED'; + } + + // Sanitise response + if (data.status < 400 && data.response.body.token) { + data.response.body.token = "" + } + + return data +} + +// OR + +sanitiser: data => { // ignore requests that start with /secure + if (data.url.startsWith("/secure")) { + return null + } + return data +} + +// OR + +sanitiser: data => { // sanitise request url: replace all numbers + data.url = data.url.replace(/\d/g, "*") + return data +} +``` + +## Troubleshooting + +Having trouble setting up this plugin? please connect to our [Discord](https://discord.openreplay.com) and get help from our community. \ No newline at end of file diff --git a/tracker/tracker-fetch/package.json b/tracker/tracker-fetch/package.json index 0b1373edc..c13b1a28b 100644 --- a/tracker/tracker-fetch/package.json +++ b/tracker/tracker-fetch/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-fetch", "description": "Tracker plugin for fetch requests recording ", - "version": "3.5.2", + "version": "3.5.3", "keywords": [ "fetch", "logging", diff --git a/tracker/tracker-fetch/src/index.ts b/tracker/tracker-fetch/src/index.ts index 93e39cce5..922913923 100644 --- a/tracker/tracker-fetch/src/index.ts +++ b/tracker/tracker-fetch/src/index.ts @@ -1,38 +1,48 @@ import { App, Messages } from '@openreplay/tracker'; -interface Request { - url: string, - body: string | Object, - headers: Record, +interface RequestData { + body: BodyInit | null | undefined + headers: Record } -interface Response { - url: string, - status: number, - body: string, - headers: Record, +interface ResponseData { + body: string | Object | null + headers: Record } +interface RequestResponseData { + readonly status: number + readonly method: string + url: string + request: RequestData + response: ResponseData +} + + export interface Options { - sessionTokenHeader?: string; - replaceDefault: boolean; // overrideDefault ? - failuresOnly: boolean; - ignoreHeaders: Array | boolean; - requestSanitizer: ((Request) => Request | null) | null; - responseSanitizer: ((Response) => Response | null) | null; + sessionTokenHeader?: string + failuresOnly: boolean + overrideGlobal: boolean + ignoreHeaders: Array | boolean + sanitiser?: (RequestResponseData) => RequestResponseData | null + + requestSanitizer?: any + responseSanitizer?: any } export default function(opts: Partial = {}) { const options: Options = Object.assign( { - replaceDefault: false, + overrideGlobal: false, failuresOnly: false, ignoreHeaders: [ 'Cookie', 'Set-Cookie', 'Authorization' ], - requestSanitizer: null, - responseSanitizer: null, }, opts, ); + if (options.requestSanitizer && options.responseSanitizer) { + console.warn("OpenReplay fetch plugin: `requestSanitizer` and `responseSanitizer` options are depricated. Please, use `sanitiser` instead (check out documentation at https://docs.openreplay.com/plugins/fetch).") + } + const origFetch = window.fetch return (app: App | null) => { if (app === null) { @@ -90,56 +100,55 @@ export default function(opts: Partial = {}) { r.headers.forEach((v, n) => { if (!isHIgnoring(n)) resHs[n] = v }) } - // Request forming - let reqBody = '' - if (typeof init.body === 'string') { - reqBody = init.body - } else if (typeof init.body === 'object') { - try { - reqBody = JSON.stringify(init.body) - } catch {} - } - let req: Request | null = { - url: input, + const req: RequestData = { headers: reqHs, - body: reqBody, - } - if (options.requestSanitizer !== null) { - req = options.requestSanitizer(req) - if (!req) { - return - } + body: init.body, } // Response forming - let res: Response | null = { - url: input, - status: r.status, + const res: ResponseData = { headers: resHs, body: text, } - if (options.responseSanitizer !== null) { - res = options.responseSanitizer(res) - if (!res) { + + const method = typeof init.method === 'string' + ? init.method.toUpperCase() + : 'GET' + let reqResInfo: RequestResponseData | null = { + url: input, + method, + status: r.status, + request: req, + response: res, + } + if (options.sanitiser) { + try { + reqResInfo.response.body = JSON.parse(text) as Object // Why the returning type is "any"? + } catch {} + reqResInfo = options.sanitiser(reqResInfo) + if (!reqResInfo) { return } } - const reqStr = JSON.stringify({ - headers: req.headers, - body: req.body, - }) - const resStr = JSON.stringify({ - headers: res.headers, - body: res.body, - }) + const getStj = (r: RequestData | ResponseData): string => { + if (r && typeof r.body !== 'string') { + try { + r.body = JSON.stringify(r.body) + } catch { + r.body = "" + //app.log.warn("Openreplay fetch") // TODO: version check + } + } + return JSON.stringify(r) + } app.send( Messages.Fetch( - typeof init.method === 'string' ? init.method.toUpperCase() : 'GET', - input, - reqStr, - resStr, + method, + String(reqResInfo.url), + getStj(reqResInfo.request), + getStj(reqResInfo.response), r.status, startTime + performance.timing.navigationStart, duration, @@ -147,8 +156,9 @@ export default function(opts: Partial = {}) { ) }); return response; - }; - if (options.replaceDefault) { + } + + if (options.overrideGlobal) { window.fetch = fetch } return fetch; diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 6df50033b..9c7920ad7 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "3.5.4", + "version": "3.5.5", "keywords": [ "logging", "replay" diff --git a/tracker/tracker/src/main/app/context.ts b/tracker/tracker/src/main/app/context.ts index 781f91ea8..fd7ec11dd 100644 --- a/tracker/tracker/src/main/app/context.ts +++ b/tracker/tracker/src/main/app/context.ts @@ -41,13 +41,13 @@ export function isInstance(node: Node, constr: Cons // @ts-ignore (for EI, Safary) doc.parentWindow || doc.defaultView; // TODO: smart global typing for Window object - while((context.parent || context.top) && context.parent !== context) { + while(context !== window) { // @ts-ignore if (node instanceof context[constr.name]) { return true } // @ts-ignore - context = context.parent || context.top + context = context.parent || window } // @ts-ignore return node instanceof context[constr.name] diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 59129cbec..de9518af6 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -234,10 +234,11 @@ export default class App { ); } + // TODO: full correct semantic checkRequiredVersion(version: string): boolean { const reqVer = version.split('.') const ver = this.version.split('.') - for (let i = 0; i < ver.length; i++) { + for (let i = 0; i < 3; i++) { if (Number(ver[i]) < Number(reqVer[i]) || isNaN(Number(ver[i])) || isNaN(Number(reqVer[i]))) { return false } diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index ad8cda673..0cad3c58b 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -1,4 +1,9 @@ -import { normSpaces, IN_BROWSER, getLabelAttribute, hasOpenreplayAttribute } from "../utils.js"; +import { + normSpaces, + IN_BROWSER, + getLabelAttribute, + hasOpenreplayAttribute, +} from "../utils.js"; import App from "../app/index.js"; import { SetInputTarget, SetInputValue, SetInputChecked } from "../../messages/index.js"; @@ -31,14 +36,14 @@ function isCheckable(node: any): node is HTMLInputElement { } const labelElementFor: ( - node: TextEditableElement, + element: TextEditableElement, ) => HTMLLabelElement | undefined = IN_BROWSER && 'labels' in HTMLInputElement.prototype - ? (node): HTMLLabelElement | undefined => { + ? (node) => { let p: Node | null = node; while ((p = p.parentNode) !== null) { - if (p.nodeName === 'LABEL') { - return p as HTMLLabelElement; + if (p instanceof HTMLLabelElement) { + return p } } const labels = node.labels; @@ -46,10 +51,10 @@ const labelElementFor: ( return labels[0]; } } - : (node): HTMLLabelElement | undefined => { + : (node) => { let p: Node | null = node; while ((p = p.parentNode) !== null) { - if (p.nodeName === 'LABEL') { + if (p instanceof HTMLLabelElement) { return p as HTMLLabelElement; } } @@ -66,10 +71,12 @@ export function getInputLabel(node: TextEditableElement): string { let label = getLabelAttribute(node); if (label === null) { const labelElement = labelElementFor(node); - label = - labelElement === undefined - ? node.placeholder || node.name - : labelElement.innerText; + label = (labelElement && labelElement.innerText) + || node.placeholder + || node.name + || node.id + || node.className + || node.type } return normSpaces(label).slice(0, 100); } @@ -101,7 +108,7 @@ export default function (app: App, opts: Partial): void { app.send(new SetInputTarget(id, label)); } } - function sendInputValue(id: number, node: TextEditableElement): void { + function sendInputValue(id: number, node: TextEditableElement | HTMLSelectElement): void { let value = node.value; let inputMode: InputMode = options.defaultInputMode; if (node.type === 'password' || hasOpenreplayAttribute(node, 'hidden')) { @@ -175,6 +182,13 @@ export default function (app: App, opts: Partial): void { if (id === undefined) { return; } + // TODO: support multiple select (?): use selectedOptions; Need send target? + if (node instanceof HTMLSelectElement) { + sendInputValue(id, node) + app.attachEventListener(node, "change", () => { + sendInputValue(id, node) + }) + } if (isTextEditable(node)) { inputValues.set(id, node.value); sendInputValue(id, node); diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index 90cf17908..196a89653 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -1,4 +1,8 @@ -import { normSpaces, hasOpenreplayAttribute, getLabelAttribute } from "../utils.js"; +import { + normSpaces, + hasOpenreplayAttribute, + getLabelAttribute, +} from "../utils.js"; import App from "../app/index.js"; import { MouseMove, MouseClick } from "../../messages/index.js"; import { getInputLabel } from "./input.js"; @@ -24,6 +28,18 @@ function _getSelector(target: Element): string { return selector } +function isClickable(element: Element): boolean { + const tag = element.tagName.toUpperCase() + return tag === 'BUTTON' || + tag === 'A' || + tag === 'LI' || + tag === 'SELECT' || + (element as HTMLElement).onclick != null || + element.getAttribute('role') === 'button' + //|| element.className.includes("btn") + // MBTODO: intersect addEventListener +} + //TODO: fix (typescript doesn't allow work when the guard is inside the function) function getTarget(target: EventTarget | null): Element | null { if (target instanceof Element) { @@ -56,13 +72,7 @@ function _getTarget(target: Element): Element | null { if (tag === 'INPUT') { return element; } - if ( - tag === 'BUTTON' || - tag === 'A' || - tag === 'LI' || - tag === 'SELECT' || - (element as HTMLElement).onclick != null || - element.getAttribute('role') === 'button' || + if (isClickable(element) || getLabelAttribute(element) !== null ) { return element; @@ -83,19 +93,16 @@ export default function (app: App): void { if (dl !== null) { return dl; } - const tag = target.tagName.toUpperCase(); - if (tag === 'INPUT') { - return getInputLabel(target as HTMLInputElement) + if (target instanceof HTMLInputElement) { + return getInputLabel(target) } - if (tag === 'BUTTON' || - tag === 'A' || - tag === 'LI' || - tag === 'SELECT' || - (target as HTMLElement).onclick != null || - target.getAttribute('role') === 'button' - ) { - const label: string = app.sanitizer.getInnerTextSecure(target as HTMLElement); - return normSpaces(label).slice(0, 100); + if (isClickable(target)) { + let label = '' + if (target instanceof HTMLElement) { + label = app.sanitizer.getInnerTextSecure(target) + } + label = label || target.id || target.className + return normSpaces(label).slice(0, 100) } return ''; } @@ -126,7 +133,7 @@ export default function (app: App): void { } app.attachEventListener( - document.documentElement, + document.documentElement, 'mouseover', (e: MouseEvent): void => { const target = getTarget(e.target); diff --git a/tracker/tracker/src/main/modules/scroll.ts b/tracker/tracker/src/main/modules/scroll.ts index 0f54ba8f9..f9c80e6d9 100644 --- a/tracker/tracker/src/main/modules/scroll.ts +++ b/tracker/tracker/src/main/modules/scroll.ts @@ -20,7 +20,7 @@ export default function (app: App): void { ), ); - const sendSetNodeScroll = app.safe((s, node): void => { + const sendSetNodeScroll = app.safe((s: [number, number], node: Node): void => { const id = app.nodes.getID(node); if (id !== undefined) { app.send(new SetNodeScroll(id, s[0], s[1])); @@ -34,6 +34,12 @@ export default function (app: App): void { nodeScroll.clear(); }); + app.nodes.attachNodeCallback(node => { + if (node instanceof Element && node.scrollLeft + node.scrollTop > 0) { + nodeScroll.set(node, [node.scrollLeft, node.scrollTop]); + } + }) + app.attachEventListener(window, 'scroll', (e: Event): void => { const target = e.target; if (target === document) { @@ -41,9 +47,7 @@ export default function (app: App): void { return; } if (target instanceof Element) { - { - nodeScroll.set(target, [target.scrollLeft, target.scrollTop]); - } + nodeScroll.set(target, [target.scrollLeft, target.scrollTop]); } }); diff --git a/tracker/tracker/src/webworker/index.ts b/tracker/tracker/src/webworker/index.ts index cf0d1586a..f598ac0a4 100644 --- a/tracker/tracker/src/webworker/index.ts +++ b/tracker/tracker/src/webworker/index.ts @@ -31,42 +31,18 @@ let sendIntervalID: ReturnType | null = null; const sendQueue: Array = []; let busy = false; let attemptsCount = 0; -let ATTEMPT_TIMEOUT = 8000; +let ATTEMPT_TIMEOUT = 3000; let MAX_ATTEMPTS_COUNT = 10; // TODO?: exploit https://developer.mozilla.org/en-US/docs/Web/API/Navigator/sendBeacon function sendBatch(batch: Uint8Array):void { - const req = new XMLHttpRequest(); + const xhr = new XMLHttpRequest(); // TODO: async=false (3d param) instead of sendQueue array ? - req.open("POST", ingestPoint + "/v1/web/i", false); // TODO opaque request? - req.setRequestHeader("Authorization", "Bearer " + token); - // req.setRequestHeader("Content-Type", ""); - req.onreadystatechange = function() { - if (this.readyState === 4) { - if (this.status == 0) { - return; // happens simultaneously with onerror TODO: clear codeflow - } - if (this.status >= 400) { // TODO: test workflow. After 400+ it calls /start for some reason - busy = false; - reset(); - sendQueue.length = 0; - if (this.status === 401) { // Unauthorised (Token expired) - self.postMessage("restart") - return - } - self.postMessage(null); - return - } - //if (this.response == null) - const nextBatch = sendQueue.shift(); - if (nextBatch) { - sendBatch(nextBatch); - } else { - busy = false; - } - } - }; - req.onerror = function(e) { + xhr.open("POST", ingestPoint + "/v1/web/i", false); + xhr.setRequestHeader("Authorization", "Bearer " + token); + // xhr.setRequestHeader("Content-Type", ""); + + function retry() { if (attemptsCount >= MAX_ATTEMPTS_COUNT) { reset(); self.postMessage(null); @@ -75,8 +51,32 @@ function sendBatch(batch: Uint8Array):void { attemptsCount++; setTimeout(() => sendBatch(batch), ATTEMPT_TIMEOUT); } - // TODO: handle offline exception - req.send(batch.buffer); + xhr.onreadystatechange = function() { + if (this.readyState === 4) { + if (this.status == 0) { + return; // happens simultaneously with onerror TODO: clear codeflow + } + if (this.status === 401) { // Unauthorised (Token expired) + busy = false + self.postMessage("restart") + return + } else if (this.status >= 400) { // TODO: test workflow. After 400+ it calls /start for some reason + retry() + return + } + // Success + attemptsCount = 0 + const nextBatch = sendQueue.shift(); + if (nextBatch) { + sendBatch(nextBatch); + } else { + busy = false; + } + } + }; + xhr.onerror = retry // TODO: when in Offline mode it doesn't handle the error + // TODO: handle offline exception (?) + xhr.send(batch.buffer); } function send(): void { @@ -101,6 +101,7 @@ function reset() { clearInterval(sendIntervalID); sendIntervalID = null; } + sendQueue.length = 0; writer.reset(); } diff --git a/utilities/build.sh b/utilities/build.sh index 99be144fe..f7d003ed3 100644 --- a/utilities/build.sh +++ b/utilities/build.sh @@ -20,11 +20,11 @@ function build_api(){ [[ $1 == "ee" ]] && { cp -rf ../ee/utilities/* ./ } - docker build -f ./Dockerfile -t ${DOCKER_REPO:-'local'}/utilities:${git_sha1} . + docker build -f ./Dockerfile -t ${DOCKER_REPO:-'local'}/assist:${git_sha1} . [[ $PUSH_IMAGE -eq 1 ]] && { - docker push ${DOCKER_REPO:-'local'}/utilities:${git_sha1} - docker tag ${DOCKER_REPO:-'local'}/utilities:${git_sha1} ${DOCKER_REPO:-'local'}/utilities:latest - docker push ${DOCKER_REPO:-'local'}/utilities:latest + docker push ${DOCKER_REPO:-'local'}/assist:${git_sha1} + docker tag ${DOCKER_REPO:-'local'}/assist:${git_sha1} ${DOCKER_REPO:-'local'}/assist:latest + docker push ${DOCKER_REPO:-'local'}/assist:latest } } diff --git a/utilities/package-lock.json b/utilities/package-lock.json index 724f7341a..df0b8c9ef 100644 --- a/utilities/package-lock.json +++ b/utilities/package-lock.json @@ -10,11 +10,8 @@ "license": "MIT", "dependencies": { "@maxmind/geoip2-node": "^3.4.0", - "aws-sdk": "^2.992.0", "express": "^4.17.1", - "peer": "^0.6.1", "socket.io": "^4.4.1", - "source-map": "^0.7.3", "ua-parser-js": "^1.0.2" } }, @@ -37,28 +34,11 @@ "node": ">= 0.6.0" } }, - "node_modules/@types/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, "node_modules/@types/component-emitter": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.11.tgz", "integrity": "sha512-SRXjM+tfsSlA9VuG8hGO2nft2p8zjXCK1VcC6N4NXbBbYbSia9kzCChYQajIjzIqOOOuh5Ock6MmV2oux4jDZQ==" }, - "node_modules/@types/connect": { - "version": "3.4.35", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", - "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/cookie": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz", @@ -69,64 +49,11 @@ "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" }, - "node_modules/@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.18", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.17.28", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", - "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "node_modules/@types/mime": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", - "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" - }, "node_modules/@types/node": { "version": "17.0.21", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" }, - "node_modules/@types/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" - }, - "node_modules/@types/range-parser": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "node_modules/@types/serve-static": { - "version": "1.13.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", - "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", - "dependencies": { - "@types/mime": "^1", - "@types/node": "*" - } - }, - "node_modules/@types/ws": { - "version": "7.4.7", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", - "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -139,28 +66,6 @@ "node": ">= 0.6" } }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -174,44 +79,6 @@ "node": ">=0.8" } }, - "node_modules/aws-sdk": { - "version": "2.1087.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", - "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", - "dependencies": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "uuid": "3.3.2", - "xml2js": "0.4.19" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/base64id": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz", @@ -240,16 +107,6 @@ "node": ">= 0.8" } }, - "node_modules/buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "dependencies": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -286,32 +143,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, "node_modules/component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", @@ -374,14 +205,6 @@ "ms": "2.0.0" } }, - "node_modules/decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -400,11 +223,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -498,14 +316,6 @@ "node": ">= 0.6" } }, - "node_modules/events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/express": { "version": "4.17.3", "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", @@ -571,18 +381,6 @@ "node": ">= 0.8" } }, - "node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -599,14 +397,6 @@ "node": ">= 0.6" } }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, "node_modules/http-errors": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", @@ -633,11 +423,6 @@ "node": ">=0.10.0" } }, - "node_modules/ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -660,27 +445,6 @@ "node": ">= 0.10" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "node_modules/jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", - "engines": { - "node": ">= 0.6.0" - } - }, "node_modules/json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", @@ -700,17 +464,6 @@ "verror": "1.10.0" } }, - "node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", @@ -832,39 +585,6 @@ "node": ">= 0.8" } }, - "node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "engines": { - "node": ">=6" - } - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -873,50 +593,11 @@ "node": ">= 0.8" } }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "engines": { - "node": ">=8" - } - }, "node_modules/path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "node_modules/peer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", - "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", - "dependencies": { - "@types/cors": "^2.8.6", - "@types/express": "^4.17.3", - "@types/ws": "^7.2.3", - "body-parser": "^1.19.0", - "cors": "^2.8.5", - "express": "^4.17.1", - "uuid": "^3.4.0", - "ws": "^7.2.3", - "yargs": "^15.3.1" - }, - "bin": { - "peerjs": "bin/peerjs" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/peer/node_modules/uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -929,11 +610,6 @@ "node": ">= 0.10" } }, - "node_modules/punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" - }, "node_modules/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", @@ -945,15 +621,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", - "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -987,19 +654,6 @@ "node": ">= 0.8" } }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1024,11 +678,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "node_modules/sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" - }, "node_modules/send": { "version": "0.17.2", "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", @@ -1071,11 +720,6 @@ "node": ">= 0.8.0" } }, - "node_modules/set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -1157,14 +801,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", - "engines": { - "node": ">= 8" - } - }, "node_modules/statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -1173,30 +809,6 @@ "node": ">= 0.6" } }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/tiny-lru": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-7.0.6.tgz", @@ -1262,15 +874,6 @@ "node": ">= 0.8" } }, - "node_modules/url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", - "dependencies": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -1279,15 +882,6 @@ "node": ">= 0.4.0" } }, - "node_modules/uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -1308,107 +902,6 @@ "core-util-is": "1.0.2", "extsprintf": "^1.2.0" } - }, - "node_modules/which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" - }, - "node_modules/wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/ws": { - "version": "7.5.7", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", - "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", - "engines": { - "node": ">=8.3.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/xml2js": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~9.0.1" - } - }, - "node_modules/xmlbuilder": { - "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/y18n": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" - }, - "node_modules/yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", - "dependencies": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "dependencies": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/yargs-parser/node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "engines": { - "node": ">=6" - } } }, "dependencies": { @@ -1428,28 +921,11 @@ "resolved": "https://registry.npmjs.org/@socket.io/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz", "integrity": "sha512-dOlCBKnDw4iShaIsH/bxujKTM18+2TOAsYz+KSc11Am38H4q5Xw8Bbz97ZYdrVNM+um3p7w86Bvvmcn9q+5+eQ==" }, - "@types/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", - "requires": { - "@types/connect": "*", - "@types/node": "*" - } - }, "@types/component-emitter": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.11.tgz", "integrity": "sha512-SRXjM+tfsSlA9VuG8hGO2nft2p8zjXCK1VcC6N4NXbBbYbSia9kzCChYQajIjzIqOOOuh5Ock6MmV2oux4jDZQ==" }, - "@types/connect": { - "version": "3.4.35", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", - "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", - "requires": { - "@types/node": "*" - } - }, "@types/cookie": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz", @@ -1460,64 +936,11 @@ "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" }, - "@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", - "requires": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.18", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "@types/express-serve-static-core": { - "version": "4.17.28", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", - "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", - "requires": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "@types/mime": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", - "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" - }, "@types/node": { "version": "17.0.21", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" }, - "@types/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" - }, - "@types/range-parser": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "@types/serve-static": { - "version": "1.13.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", - "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", - "requires": { - "@types/mime": "^1", - "@types/node": "*" - } - }, - "@types/ws": { - "version": "7.4.7", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", - "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", - "requires": { - "@types/node": "*" - } - }, "accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -1527,19 +950,6 @@ "negotiator": "0.6.3" } }, - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -1550,27 +960,6 @@ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, - "aws-sdk": { - "version": "2.1087.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1087.0.tgz", - "integrity": "sha512-m5EERT29Fwh2cv3SaSdygeAjJBXnjSaXRRERy70bf6PQ7KgmASJouBxY11g5G7LTEPK/yfB0TGshujKh3hEtPA==", - "requires": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "uuid": "3.3.2", - "xml2js": "0.4.19" - } - }, - "base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" - }, "base64id": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz", @@ -1593,16 +982,6 @@ "type-is": "~1.6.18" } }, - "buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, "bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -1624,29 +1003,6 @@ "type-fest": "^1.2.1" } }, - "cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, "component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", @@ -1697,11 +1053,6 @@ "ms": "2.0.0" } }, - "decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" - }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -1717,11 +1068,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -1783,11 +1129,6 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, - "events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" - }, "express": { "version": "4.17.3", "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", @@ -1844,15 +1185,6 @@ "unpipe": "~1.0.0" } }, - "find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1863,11 +1195,6 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, - "get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" - }, "http-errors": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", @@ -1888,11 +1215,6 @@ "safer-buffer": ">= 2.1.2 < 3" } }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -1912,21 +1234,6 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==" - }, "json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", @@ -1943,14 +1250,6 @@ "verror": "1.10.0" } }, - "locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "requires": { - "p-locate": "^4.1.0" - } - }, "lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", @@ -2031,65 +1330,16 @@ "ee-first": "1.1.1" } }, - "p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "requires": { - "p-limit": "^2.2.0" - } - }, - "p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" - }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, - "path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" - }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "peer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", - "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", - "requires": { - "@types/cors": "^2.8.6", - "@types/express": "^4.17.3", - "@types/ws": "^7.2.3", - "body-parser": "^1.19.0", - "cors": "^2.8.5", - "express": "^4.17.1", - "uuid": "^3.4.0", - "ws": "^7.2.3", - "yargs": "^15.3.1" - }, - "dependencies": { - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - } - } - }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -2099,21 +1349,11 @@ "ipaddr.js": "1.9.1" } }, - "punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" - }, "qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==" }, - "querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" - }, "quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -2135,16 +1375,6 @@ "unpipe": "1.0.0" } }, - "require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" - }, - "require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2155,11 +1385,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" - }, "send": { "version": "0.17.2", "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", @@ -2198,11 +1423,6 @@ "send": "0.17.2" } }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, "setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -2266,34 +1486,11 @@ } } }, - "source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==" - }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - }, "tiny-lru": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-7.0.6.tgz", @@ -2328,25 +1525,11 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, - "url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", - "requires": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" - }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -2361,80 +1544,6 @@ "core-util-is": "1.0.2", "extsprintf": "^1.2.0" } - }, - "which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" - }, - "wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, - "ws": { - "version": "7.5.7", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", - "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", - "requires": {} - }, - "xml2js": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", - "requires": { - "sax": ">=0.6.0", - "xmlbuilder": "~9.0.1" - } - }, - "xmlbuilder": { - "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" - }, - "y18n": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" - }, - "yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", - "requires": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - } - }, - "yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "dependencies": { - "camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" - } - } } } } diff --git a/utilities/package.json b/utilities/package.json index 452c3fe00..73b92d0f2 100644 --- a/utilities/package.json +++ b/utilities/package.json @@ -1,5 +1,5 @@ { - "name": "utilities_server", + "name": "utilities-server", "version": "1.0.0", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", @@ -12,18 +12,15 @@ "url": "git+https://github.com/openreplay/openreplay.git" }, "author": "KRAIEM Taha Yassine ", - "license": "MIT", + "license": "Elastic License 2.0 (ELv2)", "bugs": { "url": "https://github.com/openreplay/openreplay/issues" }, "homepage": "https://github.com/openreplay/openreplay#readme", "dependencies": { "@maxmind/geoip2-node": "^3.4.0", - "aws-sdk": "^2.992.0", "express": "^4.17.1", - "peer": "^0.6.1", "socket.io": "^4.4.1", - "source-map": "^0.7.3", "ua-parser-js": "^1.0.2" } } diff --git a/utilities/server.js b/utilities/server.js index 661ef081c..b0eadcccd 100644 --- a/utilities/server.js +++ b/utilities/server.js @@ -1,53 +1,21 @@ -var sourcemapsReaderServer = require('./servers/sourcemaps-server'); -var {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); -var express = require('express'); -const {ExpressPeerServer} = require('peer'); +const dumps = require('./utils/HeapSnapshot'); +const express = require('express'); const socket = require("./servers/websocket"); +const {request_logger} = require("./utils/helper"); const HOST = '0.0.0.0'; -const PORT = 9000; +const PORT = 9001; -var app = express(); -var wsapp = express(); -let debug = process.env.debug === "1" || false; -const request_logger = (identity) => { - return (req, res, next) => { - debug && console.log(identity, new Date().toTimeString(), 'REQUEST', req.method, req.originalUrl); - res.on('finish', function () { - if (this.statusCode !== 200 || debug) { - console.log(new Date().toTimeString(), 'RESPONSE', req.method, req.originalUrl, this.statusCode); - } - }) - - next(); - } -}; -app.use(request_logger("[app]")); +const wsapp = express(); wsapp.use(request_logger("[wsapp]")); -app.use('/sourcemaps', sourcemapsReaderServer); -app.use('/assist', peerRouter); -wsapp.use('/assist', socket.wsRouter); +wsapp.use(`/assist/${process.env.S3_KEY}`, socket.wsRouter); +wsapp.use(`/heapdump/${process.env.S3_KEY}`, dumps.router); -const server = app.listen(PORT, HOST, () => { - console.log(`App listening on http://${HOST}:${PORT}`); +const wsserver = wsapp.listen(PORT, HOST, () => { + console.log(`WS App listening on http://${HOST}:${PORT}`); console.log('Press Ctrl+C to quit.'); }); -const wsserver = wsapp.listen(PORT + 1, HOST, () => { - console.log(`WS App listening on http://${HOST}:${PORT + 1}`); - console.log('Press Ctrl+C to quit.'); -}); -const peerServer = ExpressPeerServer(server, { - debug: true, - path: '/', - proxied: true, - allow_discovery: false -}); -peerServer.on('connection', peerConnection); -peerServer.on('disconnect', peerDisconnect); -peerServer.on('error', peerError); -app.use('/', peerServer); -app.enable('trust proxy'); wsapp.enable('trust proxy'); socket.start(wsserver); -module.exports = {wsserver, server}; +module.exports = {wsserver}; \ No newline at end of file diff --git a/utilities/servers/websocket.js b/utilities/servers/websocket.js index 772bd7315..2c4d22342 100644 --- a/utilities/servers/websocket.js +++ b/utilities/servers/websocket.js @@ -1,8 +1,8 @@ const _io = require('socket.io'); const express = require('express'); const uaParser = require('ua-parser-js'); -const geoip2Reader = require('@maxmind/geoip2-node').Reader; -const {extractPeerId} = require('./peerjs-server'); +const {extractPeerId} = require('../utils/helper'); +const {geoip} = require('../utils/geoIP'); const wsRouter = express.Router(); const UPDATE_EVENT = "UPDATE_SESSION"; const IDENTITIES = {agent: 'agent', session: 'session'}; @@ -12,6 +12,7 @@ const AGENT_DISCONNECT = "AGENT_DISCONNECTED"; const AGENTS_CONNECTED = "AGENTS_CONNECTED"; const NO_SESSIONS = "SESSION_DISCONNECTED"; const SESSION_ALREADY_CONNECTED = "SESSION_ALREADY_CONNECTED"; +const SESSION_RECONNECTED = "SESSION_RECONNECTED"; let io; const debug = process.env.debug === "1" || false; @@ -78,7 +79,7 @@ const socketsList = async function (req, res) { } respond(res, liveSessions); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-list`, socketsList); +wsRouter.get(`/sockets-list`, socketsList); const socketsListByProject = async function (req, res) { debug && console.log("[WS]looking for available sessions"); @@ -104,7 +105,7 @@ const socketsListByProject = async function (req, res) { } respond(res, liveSessions[_projectKey] || []); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-list/:projectKey`, socketsListByProject); +wsRouter.get(`/sockets-list/:projectKey`, socketsListByProject); const socketsLive = async function (req, res) { debug && console.log("[WS]looking for all available LIVE sessions"); @@ -112,7 +113,7 @@ const socketsLive = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey !== undefined) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -131,7 +132,7 @@ const socketsLive = async function (req, res) { } respond(res, liveSessions); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-live`, socketsLive); +wsRouter.get(`/sockets-live`, socketsLive); const socketsLiveByProject = async function (req, res) { debug && console.log("[WS]looking for available LIVE sessions"); @@ -140,7 +141,7 @@ const socketsLiveByProject = async function (req, res) { let liveSessions = {}; let rooms = await getAvailableRooms(); for (let peerId of rooms) { - let {projectKey, sessionId} = extractPeerId(peerId); + let {projectKey} = extractPeerId(peerId); if (projectKey === _projectKey) { let connected_sockets = await io.in(peerId).fetchSockets(); for (let item of connected_sockets) { @@ -159,7 +160,7 @@ const socketsLiveByProject = async function (req, res) { } respond(res, liveSessions[_projectKey] || []); } -wsRouter.get(`/${process.env.S3_KEY}/sockets-live/:projectKey`, socketsLiveByProject); +wsRouter.get(`/sockets-live/:projectKey`, socketsLiveByProject); const findSessionSocketId = async (io, peerId) => { const connected_sockets = await io.in(peerId).fetchSockets(); @@ -203,6 +204,7 @@ async function get_all_agents_ids(io, socket) { return agents; } + function extractSessionInfo(socket) { if (socket.handshake.query.sessionInfo !== undefined) { debug && console.log("received headers"); @@ -216,21 +218,11 @@ function extractSessionInfo(socket) { socket.handshake.query.sessionInfo.userDevice = ua.device.model || null; socket.handshake.query.sessionInfo.userDeviceType = ua.device.type || 'desktop'; socket.handshake.query.sessionInfo.userCountry = null; - - const options = { - // you can use options like `cache` or `watchForUpdates` - }; - // console.log("Looking for MMDB file in " + process.env.MAXMINDDB_FILE); - geoip2Reader.open(process.env.MAXMINDDB_FILE, options) - .then(reader => { - debug && console.log("looking for location of "); - debug && console.log(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - let country = reader.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); - socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; - }) - .catch(error => { - console.error(error); - }); + if (geoip !== null) { + debug && console.log(`looking for location of ${socket.handshake.headers['x-forwarded-for'] || socket.handshake.address}`); + let country = geoip.country(socket.handshake.headers['x-forwarded-for'] || socket.handshake.address); + socket.handshake.query.sessionInfo.userCountry = country.country.isoCode; + } } } @@ -242,10 +234,6 @@ module.exports = { debug && console.log(`WS started:${socket.id}, Query:${JSON.stringify(socket.handshake.query)}`); socket.peerId = socket.handshake.query.peerId; socket.identity = socket.handshake.query.identity; - const {projectKey, sessionId} = extractPeerId(socket.peerId); - socket.sessionId = sessionId; - socket.projectKey = projectKey; - socket.lastMessageReceivedAt = Date.now(); let {c_sessions, c_agents} = await sessions_agents_count(io, socket); if (socket.identity === IDENTITIES.session) { if (c_sessions > 0) { @@ -258,6 +246,7 @@ module.exports = { debug && console.log(`notifying new session about agent-existence`); let agents_ids = await get_all_agents_ids(io, socket); io.to(socket.id).emit(AGENTS_CONNECTED, agents_ids); + socket.to(socket.peerId).emit(SESSION_RECONNECTED, socket.id); } } else if (c_sessions <= 0) { @@ -306,7 +295,6 @@ module.exports = { }); socket.onAny(async (eventName, ...args) => { - socket.lastMessageReceivedAt = Date.now(); if (socket.identity === IDENTITIES.session) { debug && console.log(`received event:${eventName}, from:${socket.identity}, sending message to room:${socket.peerId}`); socket.to(socket.peerId).emit(eventName, args[0]); diff --git a/utilities/utils/HeapSnapshot.js b/utilities/utils/HeapSnapshot.js new file mode 100644 index 000000000..818589fcc --- /dev/null +++ b/utilities/utils/HeapSnapshot.js @@ -0,0 +1,68 @@ +const fs = require('fs'); +const v8 = require('v8'); +const express = require('express'); +const router = express.Router(); + +const location = '/tmp/'; +let creationStatus = null; +let fileName = null; + +async function createHeapSnapshot() { + if (creationStatus) { + return console.log(`In progress ${fileName}`); + } + if (fileName === null) { + fileName = `${Date.now()}.heapsnapshot`; + } + console.log(`Creating ${fileName}`); + await fs.promises.writeFile( + location + fileName, + v8.getHeapSnapshot() + ); + console.log(`Created ${fileName}`); + creationStatus = true; +} + + +async function downloadHeapSnapshot(req, res) { + if (creationStatus === null) { + return res.end("should call /new first"); + } else if (!creationStatus) { + return res.end("should wait for done status"); + } + res.download(location + fileName, function (err) { + if (err) { + return console.error("error while uploading HeapSnapshot file"); + } + try { + fs.unlinkSync(location + fileName) + } catch (err) { + console.error("error while deleting heapsnapshot file"); + console.error(err); + } + }); +} + +function getHeapSnapshotStatus(req, res) { + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({path: location + fileName, 'done': creationStatus})); +} + +function createNewHeapSnapshot(req, res) { + creationStatus = false; + fileName = `${Date.now()}.heapsnapshot`; + setTimeout(() => { + createHeapSnapshot() + }, 0); + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({path: location + fileName, 'done': creationStatus})); +} + +router.get(`/status`, getHeapSnapshotStatus); +router.get(`/new`, createNewHeapSnapshot); +router.get(`/download`, downloadHeapSnapshot); +module.exports = {router} + +console.log(`HeapSnapshot enabled. Send a request to "/heapdump/new" to generate a heapdump.`); \ No newline at end of file diff --git a/utilities/utils/geoIP.js b/utilities/utils/geoIP.js new file mode 100644 index 000000000..5bfcfd0e8 --- /dev/null +++ b/utilities/utils/geoIP.js @@ -0,0 +1,16 @@ +const geoip2Reader = require('@maxmind/geoip2-node').Reader; +let geoip = null; +if (process.env.MAXMINDDB_FILE !== undefined) { + geoip2Reader.open(process.env.MAXMINDDB_FILE, {}) + .then(reader => { + geoip = reader; + }) + .catch(error => { + console.log("Error while opening the MAXMINDDB_FILE.") + console.error(error); + }); +} else { + console.error("!!! please provide a valid value for MAXMINDDB_FILE env var."); +} + +module.exports = {geoip} \ No newline at end of file diff --git a/utilities/utils/helper.js b/utilities/utils/helper.js new file mode 100644 index 000000000..98322c417 --- /dev/null +++ b/utilities/utils/helper.js @@ -0,0 +1,30 @@ +let PROJECT_KEY_LENGTH = parseInt(process.env.PROJECT_KEY_LENGTH) || 20; +let debug = process.env.debug === "1" || false; +const extractPeerId = (peerId) => { + let splited = peerId.split("-"); + if (splited.length !== 2) { + debug && console.error(`cannot split peerId: ${peerId}`); + return {}; + } + if (PROJECT_KEY_LENGTH > 0 && splited[0].length !== PROJECT_KEY_LENGTH) { + debug && console.error(`wrong project key length for peerId: ${peerId}`); + return {}; + } + return {projectKey: splited[0], sessionId: splited[1]}; +}; +const request_logger = (identity) => { + return (req, res, next) => { + debug && console.log(identity, new Date().toTimeString(), 'REQUEST', req.method, req.originalUrl); + res.on('finish', function () { + if (this.statusCode !== 200 || debug) { + console.log(new Date().toTimeString(), 'RESPONSE', req.method, req.originalUrl, this.statusCode); + } + }) + + next(); + } +}; + +module.exports = { + extractPeerId, request_logger +}; \ No newline at end of file