From c64454ec94ea8e17501f4e287eebafbe3e4a9763 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 19 Apr 2023 16:39:38 +0100 Subject: [PATCH] feat(chalice): changed release version feat(assist): changed release version feat(peers): changed release version feat(sourcemaps-reader): changed release version feat(chalice): enhanced health-check feat(DB): sessions_count table to keep status --- api/chalicelib/core/health.py | 54 +++++++++++++--- assist/package.json | 2 +- ee/api/.gitignore | 1 - ee/api/chalicelib/core/health.py | 64 +++++++++++++++---- ee/api/clean-dev.sh | 1 - ee/api/routers/subs/health.py | 23 +++++++ ee/assist/package.json | 2 +- .../{1.12.0/1.12.0.sql => 1.11.5/1.11.5.sql} | 0 .../clickhouse/create/init_schema.sql | 2 +- .../{1.12.0/1.12.0.sql => 1.11.5/1.11.5.sql} | 16 ++++- .../db/init_dbs/postgresql/init_schema.sql | 42 ++++++++---- peers/package.json | 2 +- .../{1.12.0/1.12.0.sql => 1.11.5/1.11.5.sql} | 14 +++- .../db/init_dbs/postgresql/init_schema.sql | 12 +++- sourcemap-reader/package.json | 2 +- 15 files changed, 192 insertions(+), 45 deletions(-) create mode 100644 ee/api/routers/subs/health.py rename ee/scripts/schema/db/init_dbs/clickhouse/{1.12.0/1.12.0.sql => 1.11.5/1.11.5.sql} (100%) rename ee/scripts/schema/db/init_dbs/postgresql/{1.12.0/1.12.0.sql => 1.11.5/1.11.5.sql} (57%) rename scripts/schema/db/init_dbs/postgresql/{1.12.0/1.12.0.sql => 1.11.5/1.11.5.sql} (62%) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index f1fec41fa..8832f7be2 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -1,3 +1,4 @@ +from ssl import SSLCertVerificationError from urllib.parse import urlparse import redis @@ -33,7 +34,7 @@ HEALTH_ENDPOINTS = { } -def __check_database_pg(): +def __check_database_pg(*_): fail_response = { "health": False, "details": { @@ -64,11 +65,11 @@ def __check_database_pg(): } -def __not_supported(): +def __not_supported(*_): return {"errors": ["not supported"]} -def __always_healthy(): +def __always_healthy(*_): return { "health": True, "details": {} @@ -76,7 +77,7 @@ def __always_healthy(): def __check_be_service(service_name): - def fn(): + def fn(*_): fail_response = { "health": False, "details": { @@ -112,7 +113,7 @@ def __check_be_service(service_name): return fn -def __check_redis(): +def __check_redis(*_): fail_response = { "health": False, "details": {"errors": ["server health-check failed"]} @@ -139,6 +140,38 @@ def __check_redis(): } +def __check_SSL(*_): + fail_response = { + "health": False, + "details": { + "errors": ["SSL Certificate health-check failed"] + } + } + try: + requests.get(config("SITE_URL"), verify=True, allow_redirects=True) + except Exception as e: + print("!! health failed: SSL Certificate") + print(str(e)) + return fail_response + return { + "health": True, + "details": {} + } + + +def __get_sessions_stats(*_): + with pg_client.PostgresClient() as cur: + query = cur.mogrify("""SELECT COALESCE(SUM(sessions_count),0) AS s_c, + COALESCE(SUM(events_count),0) AS e_c + FROM public.sessions_count;""") + cur.execute(query) + row = cur.fetchone() + return { + "numberOfSessionsCaptured": row["s_c"], + "numberOfEventCaptured": row["e_c"] + } + + def get_health(): health_map = { "databases": { @@ -163,9 +196,14 @@ def get_health(): "sink": __check_be_service("sink"), "sourcemaps-reader": __check_be_service("sourcemaps-reader"), "storage": __check_be_service("storage") - } + }, + "details": __get_sessions_stats, + "ssl": __check_SSL } for parent_key in health_map.keys(): - for element_key in health_map[parent_key]: - health_map[parent_key][element_key] = health_map[parent_key][element_key]() + if isinstance(health_map[parent_key], dict): + for element_key in health_map[parent_key]: + health_map[parent_key][element_key] = health_map[parent_key][element_key]() + else: + health_map[parent_key] = health_map[parent_key]() return health_map diff --git a/assist/package.json b/assist/package.json index 7a0026de7..35ade8ffd 100644 --- a/assist/package.json +++ b/assist/package.json @@ -1,6 +1,6 @@ { "name": "assist-server", - "version": "v1.12.0", + "version": "v1.11.5", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", "scripts": { diff --git a/ee/api/.gitignore b/ee/api/.gitignore index e41c75ab5..9484964ea 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -267,6 +267,5 @@ Pipfile.lock /build_crons.sh /run-dev.sh /run-alerts-dev.sh -/routers/subs/health.py /routers/subs/v1_api.py #exp /chalicelib/core/dashboards.py \ No newline at end of file diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 70ecdd674..b92530213 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -34,7 +34,7 @@ HEALTH_ENDPOINTS = { } -def __check_database_pg(): +def __check_database_pg(*_): fail_response = { "health": False, "details": { @@ -65,11 +65,11 @@ def __check_database_pg(): } -def __not_supported(): +def __not_supported(*_): return {"errors": ["not supported"]} -def __always_healthy(): +def __always_healthy(*_): return { "health": True, "details": {} @@ -77,7 +77,7 @@ def __always_healthy(): def __check_be_service(service_name): - def fn(): + def fn(*_): fail_response = { "health": False, "details": { @@ -113,7 +113,7 @@ def __check_be_service(service_name): return fn -def __check_redis(): +def __check_redis(*_): fail_response = { "health": False, "details": {"errors": ["server health-check failed"]} @@ -140,7 +140,44 @@ def __check_redis(): } -def get_health(): +def __check_SSL(*_): + fail_response = { + "health": False, + "details": { + "errors": ["SSL Certificate health-check failed"] + } + } + try: + requests.get(config("SITE_URL"), verify=True, allow_redirects=True) + except Exception as e: + print("!! health failed: SSL Certificate") + print(str(e)) + return fail_response + return { + "health": True, + "details": {} + } + + +def __get_sessions_stats(tenant_id, *_): + with pg_client.PostgresClient() as cur: + extra_query = "" + if tenant_id: + extra_query = """INNER JOIN public.projects USING(project_id) + WHERE tenant_id=%(tenant_id)s""" + query = cur.mogrify(f"""SELECT COALESCE(SUM(sessions_count),0) AS s_c, + COALESCE(SUM(events_count),0) AS e_c + FROM public.sessions_count {extra_query};""", + {"tenant_id": tenant_id}) + cur.execute(query) + row = cur.fetchone() + return { + "numberOfSessionsCaptured": row["s_c"], + "numberOfEventCaptured": row["e_c"] + } + + +def get_health(tenant_id=None): health_map = { "databases": { "postgres": __check_database_pg, @@ -169,15 +206,20 @@ def get_health(): "sink": __check_be_service("sink"), "sourcemaps-reader": __check_be_service("sourcemaps-reader"), "storage": __check_be_service("storage") - } + }, + "details": __get_sessions_stats, + "ssl": __check_SSL } for parent_key in health_map.keys(): - for element_key in health_map[parent_key]: - health_map[parent_key][element_key] = health_map[parent_key][element_key]() + if isinstance(health_map[parent_key], dict): + for element_key in health_map[parent_key]: + health_map[parent_key][element_key] = health_map[parent_key][element_key](tenant_id) + else: + health_map[parent_key] = health_map[parent_key](tenant_id) return health_map -def __check_database_ch(): +def __check_database_ch(*_): fail_response = { "health": False, "details": {"errors": ["server health-check failed"]} @@ -210,7 +252,7 @@ def __check_database_ch(): } } -# def __check_kafka(): +# def __check_kafka(*_): # fail_response = { # "health": False, # "details": {"errors": ["server health-check failed"]} diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index 510ae9b4c..22e30a1c1 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -80,7 +80,6 @@ rm -rf ./Dockerfile_bundle rm -rf ./entrypoint.bundle.sh rm -rf ./chalicelib/core/heatmaps.py rm -rf ./schemas.py -rm -rf ./routers/subs/health.py rm -rf ./routers/subs/v1_api.py #exp rm -rf ./chalicelib/core/custom_metrics.py rm -rf ./chalicelib/core/performance_event.py diff --git a/ee/api/routers/subs/health.py b/ee/api/routers/subs/health.py new file mode 100644 index 000000000..be583503b --- /dev/null +++ b/ee/api/routers/subs/health.py @@ -0,0 +1,23 @@ +from fastapi import Depends +from fastapi import HTTPException, status + +import schemas +from chalicelib.core import health, tenants +from or_dependencies import OR_context +from routers.base import get_routers + +public_app, app, app_apikey = get_routers() + + +@app.get('/healthz', tags=["health-check"]) +async def get_global_health_status(context: schemas.CurrentContext = Depends(OR_context)): + return {"data": health.get_health(tenant_id=context.tenant_id)} + + +if not tenants.tenants_exists(use_pool=False): + @public_app.get('/health', tags=["health-check"]) + async def get_public_health_status(): + if tenants.tenants_exists(): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Not Found") + + return await get_global_health_status() diff --git a/ee/assist/package.json b/ee/assist/package.json index 50d1fcbd4..8f579e78a 100644 --- a/ee/assist/package.json +++ b/ee/assist/package.json @@ -1,6 +1,6 @@ { "name": "assist-server", - "version": "v1.12.0-ee", + "version": "v1.11.5-ee", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", "scripts": { diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/1.12.0/1.12.0.sql b/ee/scripts/schema/db/init_dbs/clickhouse/1.11.5/1.11.5.sql similarity index 100% rename from ee/scripts/schema/db/init_dbs/clickhouse/1.12.0/1.12.0.sql rename to ee/scripts/schema/db/init_dbs/clickhouse/1.11.5/1.11.5.sql diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index 1db327ce0..2a07d5560 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.12.0-ee'; +CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.11.5-ee'; CREATE DATABASE IF NOT EXISTS experimental; CREATE TABLE IF NOT EXISTS experimental.autocomplete diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.12.0/1.12.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.11.5/1.11.5.sql similarity index 57% rename from ee/scripts/schema/db/init_dbs/postgresql/1.12.0/1.12.0.sql rename to ee/scripts/schema/db/init_dbs/postgresql/1.11.5/1.11.5.sql index e614d5517..73170bec6 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.12.0/1.12.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.11.5/1.11.5.sql @@ -2,7 +2,7 @@ DO $$ DECLARE previous_version CONSTANT text := 'v1.11.0-ee'; - next_version CONSTANT text := 'v1.12.0-ee'; + next_version CONSTANT text := 'v1.11.5-ee'; BEGIN IF (SELECT openreplay_version()) = previous_version THEN raise notice 'valid previous DB version'; @@ -19,9 +19,21 @@ BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.12.0-ee' +SELECT 'v1.11.5-ee' $$ LANGUAGE sql IMMUTABLE; ALTER TYPE issue_type ADD VALUE IF NOT EXISTS 'app_crash'; +CREATE TABLE IF NOT EXISTS public.sessions_count +( + project_id integer NOT NULL, + created_at bigint default (EXTRACT(epoch FROM date_trunc('day'::text, now())) * 1000)::bigint, + sessions_count integer, + events_count bigint, + _timestamp timestamp default (now() AT TIME ZONE 'utc'::text), + primary key (project_id, created_at) +); + +CREATE INDEX IF NOT EXISTS sessions_count_project_id_idx ON public.sessions_count (project_id); + COMMIT; \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index 902a471b2..07b4a6e33 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -7,7 +7,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.12.0-ee' +SELECT 'v1.11.5-ee' $$ LANGUAGE sql IMMUTABLE; @@ -131,7 +131,8 @@ $$ ('users'), ('webhooks'), ('sessions_notes'), - ('assist_records')) + ('assist_records'), + ('sessions_count')) select bool_and(exists(select * from information_schema.tables t where table_schema = 'public' @@ -863,6 +864,31 @@ $$ is_public boolean NOT NULL DEFAULT FALSE ); + CREATE TABLE IF NOT EXISTS public.assist_records + ( + record_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE SET NULL, + created_at bigint NOT NULL DEFAULT (EXTRACT(EPOCH FROM now() at time zone 'utc') * 1000)::bigint, + deleted_at timestamp without time zone NULL DEFAULT NULL, + name text NOT NULL, + file_key text NOT NULL, + duration integer NOT NULL + ); + + CREATE TABLE IF NOT EXISTS public.sessions_count + ( + project_id integer NOT NULL, + created_at bigint default (EXTRACT(epoch FROM date_trunc('day'::text, now())) * 1000)::bigint, + sessions_count integer, + events_count bigint, + _timestamp timestamp default (now() AT TIME ZONE 'utc'::text), + primary key (project_id, created_at) + ); + + CREATE INDEX IF NOT EXISTS sessions_count_project_id_idx ON public.sessions_count (project_id); + RAISE NOTICE 'Created missing public schema tables'; END IF; END; @@ -1213,18 +1239,6 @@ $$ CREATE INDEX IF NOT EXISTS requests_path_nn_gin_idx ON events_common.requests USING GIN (path gin_trgm_ops) WHERE path IS NOT NULL; CREATE INDEX IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL; - CREATE TABLE IF NOT EXISTS assist_records - ( - record_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE SET NULL, - created_at bigint NOT NULL DEFAULT (EXTRACT(EPOCH FROM now() at time zone 'utc') * 1000)::bigint, - deleted_at timestamp without time zone NULL DEFAULT NULL, - name text NOT NULL, - file_key text NOT NULL, - duration integer NOT NULL - ); END IF; END; $$ diff --git a/peers/package.json b/peers/package.json index aef95ffe4..4e8158565 100644 --- a/peers/package.json +++ b/peers/package.json @@ -1,6 +1,6 @@ { "name": "peers-server", - "version": "v1.12.0", + "version": "v1.11.5", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", "scripts": { diff --git a/scripts/schema/db/init_dbs/postgresql/1.12.0/1.12.0.sql b/scripts/schema/db/init_dbs/postgresql/1.11.5/1.11.5.sql similarity index 62% rename from scripts/schema/db/init_dbs/postgresql/1.12.0/1.12.0.sql rename to scripts/schema/db/init_dbs/postgresql/1.11.5/1.11.5.sql index 9161447d7..84560819f 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.12.0/1.12.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.11.5/1.11.5.sql @@ -2,7 +2,7 @@ DO $$ DECLARE previous_version CONSTANT text := 'v1.11.0'; - next_version CONSTANT text := 'v1.12.0'; + next_version CONSTANT text := 'v1.11.5'; BEGIN IF (SELECT openreplay_version()) = previous_version THEN raise notice 'valid previous DB version'; @@ -19,9 +19,19 @@ BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.12.0' +SELECT 'v1.11.5' $$ LANGUAGE sql IMMUTABLE; ALTER TYPE issue_type ADD VALUE IF NOT EXISTS 'app_crash'; +CREATE TABLE IF NOT EXISTS public.sessions_count +( + project_id integer NOT NULL, + created_at bigint default (EXTRACT(epoch FROM date_trunc('day'::text, now())) * 1000)::bigint, + sessions_count integer, + events_count bigint, + _timestamp timestamp default (now() AT TIME ZONE 'utc'::text), + primary key (project_id, created_at) +); + COMMIT; \ No newline at end of file diff --git a/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/scripts/schema/db/init_dbs/postgresql/init_schema.sql index dcd60bc86..495ab3070 100644 --- a/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS events; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.12.0' +SELECT 'v1.11.5' $$ LANGUAGE sql IMMUTABLE; @@ -962,6 +962,16 @@ $$ is_public boolean NOT NULL DEFAULT FALSE ); + CREATE TABLE public.sessions_count + ( + project_id integer NOT NULL, + created_at bigint default (EXTRACT(epoch FROM date_trunc('day'::text, now())) * 1000)::bigint, + sessions_count integer, + events_count bigint, + _timestamp timestamp default (now() AT TIME ZONE 'utc'::text), + primary key (project_id, created_at) + ); + raise notice 'DB created'; END IF; END; diff --git a/sourcemap-reader/package.json b/sourcemap-reader/package.json index 8faee593e..c3821ea5e 100644 --- a/sourcemap-reader/package.json +++ b/sourcemap-reader/package.json @@ -1,6 +1,6 @@ { "name": "sourcemaps-reader", - "version": "v1.12.0", + "version": "v1.11.5", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", "scripts": {