From 193030657a3ba002a89a02a62cdf9e0bcd846dfc Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 7 Apr 2023 16:14:33 +0100 Subject: [PATCH 01/28] feat(chalice): skip mob existence verification --- api/chalicelib/core/sessions_replay.py | 16 ++++++++++------ ee/api/chalicelib/core/sessions_replay.py | 14 ++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/api/chalicelib/core/sessions_replay.py b/api/chalicelib/core/sessions_replay.py index 94e3cc504..68991ceca 100644 --- a/api/chalicelib/core/sessions_replay.py +++ b/api/chalicelib/core/sessions_replay.py @@ -69,9 +69,11 @@ def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_ if e['source'] == "js_exception"][:500] data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, session_id=session_id) - data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) - data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) - data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id) + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False) + data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, start_ts=data["startTs"], duration=data["duration"]) @@ -126,9 +128,11 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat if data["platform"] == 'ios': data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) else: - data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) - data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) - data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id) + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False) + data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, diff --git a/ee/api/chalicelib/core/sessions_replay.py b/ee/api/chalicelib/core/sessions_replay.py index fcbce92eb..c3ecbca9d 100644 --- a/ee/api/chalicelib/core/sessions_replay.py +++ b/ee/api/chalicelib/core/sessions_replay.py @@ -72,10 +72,11 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu if e['source'] == "js_exception"][:500] data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, session_id=session_id) - data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) - data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False) data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, - context=context) + context=context, check_existence=False) data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, start_ts=data["startTs"], duration=data["duration"]) @@ -132,10 +133,11 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat if data["platform"] == 'ios': data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) else: - data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) - data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False) data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, - context=context) + context=context, check_existence=False) data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, From a71a7086476e6d85a008f1a81bea8b0501189b38 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 8 Apr 2023 10:14:10 +0200 Subject: [PATCH 02/28] fix redis endpoint and chalice health endpoints (#1138) * chore(helm): Adding redis string from global config Signed-off-by: rjshrjndrn * fix(chalice): health check url for alerts and assist Signed-off-by: rjshrjndrn --------- Signed-off-by: rjshrjndrn --- api/chalicelib/core/health.py | 4 ++-- .../openreplay/charts/chalice/templates/deployment.yaml | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index f1fec41fa..155a73272 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -14,9 +14,9 @@ def app_connection_string(name, port, path): HEALTH_ENDPOINTS = { - "alerts": app_connection_string("alerts-openreplay", 8888, "metrics"), + "alerts": app_connection_string("alerts-openreplay", 8888, "health"), "assets": app_connection_string("assets-openreplay", 8888, "metrics"), - "assist": app_connection_string("assist-openreplay", 8888, "metrics"), + "assist": app_connection_string("assist-openreplay", 8888, "health"), "chalice": app_connection_string("chalice-openreplay", 8888, "metrics"), "db": app_connection_string("db-openreplay", 8888, "metrics"), "ender": app_connection_string("ender-openreplay", 8888, "metrics"), diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index 1efa5bccf..a5b6dce7b 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -43,10 +43,9 @@ spec: {{- .Values.healthCheck | toYaml | nindent 10}} {{- end}} env: + {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} - name: KAFKA_SERVERS value: "{{ .Values.global.kafka.kafkaHost }}" - - name: REDIS_STRING - value: "{{ .Values.global.redis.redisHost }}" - name: ch_username value: "{{ .Values.global.clickhouse.username }}" - name: ch_password From eef776a13df893fba0f8a8532ea3f7603b947e95 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sun, 9 Apr 2023 15:43:38 +0200 Subject: [PATCH 03/28] fix(ee): chalice health check (#1142) Signed-off-by: rjshrjndrn --- ee/api/chalicelib/core/health.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 70ecdd674..9b47720ee 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -15,9 +15,9 @@ def app_connection_string(name, port, path): HEALTH_ENDPOINTS = { - "alerts": app_connection_string("alerts-openreplay", 8888, "metrics"), + "alerts": app_connection_string("alerts-openreplay", 8888, "health"), "assets": app_connection_string("assets-openreplay", 8888, "metrics"), - "assist": app_connection_string("assist-openreplay", 8888, "metrics"), + "assist": app_connection_string("assist-openreplay", 8888, "health"), "chalice": app_connection_string("chalice-openreplay", 8888, "metrics"), "db": app_connection_string("db-openreplay", 8888, "metrics"), "ender": app_connection_string("ender-openreplay", 8888, "metrics"), From c724b57fc711e68da424be588dca660f08409375 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 10 Apr 2023 11:38:42 +0200 Subject: [PATCH 04/28] chore(cli): Adding verbose logging (#1144) Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay-cli | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index addb7ec3a..82caf9dfa 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -190,6 +190,12 @@ function or_helm_upgrade() { function upgrade_old() { old_vars_path="$1" + [[ -f $old_vars_path ]] || log err "No configuration file ${BWHITE}$old_vars_path${RED}. + If you're updating from version older than ${BWHITE}v1.10.0${RED}, for example ${BWHITE}v1.9.0${RED}: + ${BWHITE}openreplay --deprecated-upgrade ~/openreplay_v1.9.0/scripts/helmcharts/vars.yaml${RED}. + If you're having a custom installation, + ${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED}. + " or_version=$(busybox awk '/fromVersion/{print $2}' < "${old_vars_path}") sudo cp "${old_vars_path}" ${OR_DIR}/vars.yaml.backup."${or_version//\"}"_"$(date +%Y%m%d-%H%M%S)" || log err "Not able to copy old vars.yaml" sudo cp "${old_vars_path}" ${OR_DIR}/vars.yaml || log err "Not able to copy old vars.yaml" @@ -271,6 +277,12 @@ function upgrade() { # 3. How to update package. Because openreplay -u will be done from old update script # 4. Update from Version exists git || log err "Git not found. Please install" + [[ -f ${OR_DIR}/vars.yaml ]] || log err "No configuration file ${BWHITE}${OR_DIR}/vars.yaml${RED}. + If you're updating from version older than ${BWHITE}v1.10.0${RED}, for example ${BWHITE}v1.9.0${RED}: + ${BWHITE}openreplay --deprecated-upgrade ~/openreplay_v1.9.0/scripts/helmcharts/vars.yaml${RED}. + If you're having a custom installation, + ${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED}. + " or_version=$(busybox awk '/fromVersion/{print $2}' < "${OR_DIR}/vars.yaml") || { log err "${BWHITE}${OR_DIR}/vars.yaml${RED} not found. Please do ${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED} From 6317d4b1cb5060760d594f0740b8e2267a01ac14 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Tue, 11 Apr 2023 14:24:29 +0200 Subject: [PATCH 05/28] chore(helm): Adding option for records bucket (#1146) Signed-off-by: rjshrjndrn --- .../openreplay/charts/chalice/templates/deployment.yaml | 2 ++ scripts/helmcharts/vars.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index a5b6dce7b..e75ef91b2 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -113,6 +113,8 @@ spec: value: '{{ .Values.global.s3.region }}' - name: sessions_region value: '{{ .Values.global.s3.region }}' + - name: ASSIST_RECORDS_BUCKET + value: {{ .Values.global.s3.assistRecordsBucket }} - name: sessions_bucket value: {{ .Values.global.s3.recordingsBucket }} - name: sourcemaps_bucket diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 4c48b54bd..6aebf266f 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -117,6 +117,7 @@ global: assetsBucket: "sessions-assets" recordingsBucket: "mobs" sourcemapsBucket: "sourcemaps" + assistRecordsBucket: "records" vaultBucket: "vault-data" # This is only for enterpriseEdition quickwitBucket: "quickwit" From c1625cc9867c2900f82cdd024f2db6989cb6d9c1 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Tue, 11 Apr 2023 16:07:36 +0200 Subject: [PATCH 06/28] chore(build): Bump image version of frontend assets while building (#1149) Signed-off-by: rjshrjndrn --- frontend/build.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/build.sh b/frontend/build.sh index 0eb9cc725..e95aa35ff 100644 --- a/frontend/build.sh +++ b/frontend/build.sh @@ -20,7 +20,9 @@ check_prereq() { chart=frontend [[ $1 == ee ]] && ee=true [[ $PATCH -eq 1 ]] && { - image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + __app_version="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/${chart} | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}' | cut -d 'v' -f2)" + sed -i "s/^VERSION = .*/VERSION = $__app_version/g" .env.sample + image_tag="v${__app_version}" [[ $ee == "true" ]] && { image_tag="${image_tag}-ee" } @@ -31,8 +33,9 @@ update_helm_release() { # Update the chart version sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml # Update image tags - sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + sed -i "s#ppVersion.*#ppVersion: \"v${__app_version}\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml # Commit the changes + git add .env.sample git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml git commit -m "chore(helm): Updating $chart image release" } From 9d6daf09a20dfe856c23bfda5c4b55ddd2b15a83 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 13:06:38 +0100 Subject: [PATCH 07/28] feat(chalice): fixed jobs execution --- .github/workflows/api-ee.yaml | 1 + .github/workflows/api.yaml | 1 + api/chalicelib/core/jobs.py | 98 ++++++++++---------------- api/chalicelib/core/sessions.py | 36 +++------- api/chalicelib/core/sessions_mobs.py | 3 +- ee/api/chalicelib/core/sessions_exp.py | 36 +++------- 6 files changed, 61 insertions(+), 114 deletions(-) diff --git a/.github/workflows/api-ee.yaml b/.github/workflows/api-ee.yaml index f9a1730f1..ed81c8972 100644 --- a/.github/workflows/api-ee.yaml +++ b/.github/workflows/api-ee.yaml @@ -10,6 +10,7 @@ on: branches: - dev - api-* + - v1.11.0-patch paths: - "ee/api/**" - "api/**" diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index 8e2f7fa7b..451ae64b5 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -10,6 +10,7 @@ on: branches: - dev - api-* + - v1.11.0-patch paths: - "api/**" - "!api/.gitignore" diff --git a/api/chalicelib/core/jobs.py b/api/chalicelib/core/jobs.py index 2d244e366..69a777511 100644 --- a/api/chalicelib/core/jobs.py +++ b/api/chalicelib/core/jobs.py @@ -17,11 +17,9 @@ class JobStatus: def get(job_id): with pg_client.PostgresClient() as cur: query = cur.mogrify( - """\ - SELECT - * - FROM public.jobs - WHERE job_id = %(job_id)s;""", + """SELECT * + FROM public.jobs + WHERE job_id = %(job_id)s;""", {"job_id": job_id} ) cur.execute(query=query) @@ -37,11 +35,9 @@ def get(job_id): def get_all(project_id): with pg_client.PostgresClient() as cur: query = cur.mogrify( - """\ - SELECT - * - FROM public.jobs - WHERE project_id = %(project_id)s;""", + """SELECT * + FROM public.jobs + WHERE project_id = %(project_id)s;""", {"project_id": project_id} ) cur.execute(query=query) @@ -59,15 +55,10 @@ def create(project_id, data): **data } - query = cur.mogrify("""\ - INSERT INTO public.jobs( - project_id, description, status, action, - reference_id, start_at - ) - VALUES ( - %(project_id)s, %(description)s, %(status)s, %(action)s, - %(reference_id)s, %(start_at)s - ) RETURNING *;""", job) + query = cur.mogrify( + """INSERT INTO public.jobs(project_id, description, status, action,reference_id, start_at) + VALUES (%(project_id)s, %(description)s, %(status)s, %(action)s,%(reference_id)s, %(start_at)s) + RETURNING *;""", job) cur.execute(query=query) @@ -90,14 +81,13 @@ def update(job_id, job): **job } - query = cur.mogrify("""\ - UPDATE public.jobs - SET - updated_at = timezone('utc'::text, now()), - status = %(status)s, - errors = %(errors)s - WHERE - job_id = %(job_id)s RETURNING *;""", job_data) + query = cur.mogrify( + """UPDATE public.jobs + SET updated_at = timezone('utc'::text, now()), + status = %(status)s, + errors = %(errors)s + WHERE job_id = %(job_id)s + RETURNING *;""", job_data) cur.execute(query=query) @@ -116,11 +106,12 @@ def format_datetime(r): def get_scheduled_jobs(): with pg_client.PostgresClient() as cur: query = cur.mogrify( - """\ - SELECT * FROM public.jobs - WHERE status = %(status)s AND start_at <= (now() at time zone 'utc');""", - {"status": JobStatus.SCHEDULED} - ) + """SELECT * FROM public.jobs + WHERE status = %(status)s AND start_at <= (now() at time zone 'utc');""", + {"status": JobStatus.SCHEDULED}) + print("------------------") + print(query) + print("------------------") cur.execute(query=query) data = cur.fetchall() for record in data: @@ -131,43 +122,28 @@ def get_scheduled_jobs(): def execute_jobs(): jobs = get_scheduled_jobs() - if len(jobs) == 0: - # No jobs to execute - return for job in jobs: - print(f"job can be executed {job['id']}") + print(f"Executing jobId:{job['jobId']}") try: if job["action"] == Actions.DELETE_USER_DATA: - session_ids = sessions.get_session_ids_by_user_ids( - project_id=job["projectId"], user_ids=job["referenceId"] - ) - - sessions.delete_sessions_by_session_ids(session_ids) - sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) + session_ids = sessions.get_session_ids_by_user_ids(project_id=job["projectId"], + user_ids=[job["referenceId"]]) + if len(session_ids) > 0: + print(f"Deleting {len(session_ids)} sessions") + sessions.delete_sessions_by_session_ids(session_ids) + sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) else: - raise Exception(f"The action {job['action']} not supported.") + raise Exception(f"The action '{job['action']}' not supported.") job["status"] = JobStatus.COMPLETED - print(f"job completed {job['id']}") + print(f"Job completed {job['jobId']}") except Exception as e: + print("-----") + print(e) + print("-----") job["status"] = JobStatus.FAILED job["error"] = str(e) - print(f"job failed {job['id']}") + print(f"Job failed {job['jobId']}") - update(job["job_id"], job) - - -def group_user_ids_by_project_id(jobs, now): - project_id_user_ids = {} - for job in jobs: - if job["startAt"] > now: - continue - - project_id = job["projectId"] - if project_id not in project_id_user_ids: - project_id_user_ids[project_id] = [] - - project_id_user_ids[project_id].append(job) - - return project_id_user_ids + update(job["jobId"], job) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 8f98aac83..7de8253da 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1068,23 +1068,21 @@ def get_session_user(project_id, user_id): def get_session_ids_by_user_ids(project_id, user_ids): with pg_client.PostgresClient() as cur: query = cur.mogrify( - """\ - SELECT session_id FROM public.sessions - WHERE - project_id = %(project_id)s AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)} - ) - ids = cur.execute(query=query) - return ids + """SELECT session_id + FROM public.sessions + WHERE project_id = %(project_id)s + AND user_id IN %(userId)s;""", + {"project_id": project_id, "userId": tuple(user_ids)}) + cur.execute(query=query) + ids = cur.fetchall() + return [s["session_id"] for s in ids] def delete_sessions_by_session_ids(session_ids): with pg_client.PostgresClient(unlimited_query=True) as cur: query = cur.mogrify( - """\ - DELETE FROM public.sessions - WHERE - session_id IN %(session_ids)s;""", + """DELETE FROM public.sessions + WHERE session_id IN %(session_ids)s;""", {"session_ids": tuple(session_ids)} ) cur.execute(query=query) @@ -1092,20 +1090,6 @@ def delete_sessions_by_session_ids(session_ids): return True -def delete_sessions_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """\ - DELETE FROM public.sessions - WHERE - project_id = %(project_id)s AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)} - ) - cur.execute(query=query) - - return True - - def count_all(): with pg_client.PostgresClient(unlimited_query=True) as cur: cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions") diff --git a/api/chalicelib/core/sessions_mobs.py b/api/chalicelib/core/sessions_mobs.py index 68c64fd2b..fb9f8fa9e 100644 --- a/api/chalicelib/core/sessions_mobs.py +++ b/api/chalicelib/core/sessions_mobs.py @@ -57,5 +57,6 @@ def get_ios(session_id): def delete_mobs(project_id, session_ids): for session_id in session_ids: - for k in __get_mob_keys(project_id=project_id, session_id=session_id): + for k in __get_mob_keys(project_id=project_id, session_id=session_id) \ + + __get_mob_keys_deprecated(session_id=session_id): s3.schedule_for_deletion(config("sessions_bucket"), k) diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index 888800681..1c4d20883 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -1399,23 +1399,21 @@ def get_session_user(project_id, user_id): def get_session_ids_by_user_ids(project_id, user_ids): with pg_client.PostgresClient() as cur: query = cur.mogrify( - """\ - SELECT session_id FROM public.sessions - WHERE - project_id = %(project_id)s AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)} - ) - ids = cur.execute(query=query) - return ids + """SELECT session_id + FROM public.sessions + WHERE project_id = %(project_id)s + AND user_id IN %(userId)s;""", + {"project_id": project_id, "userId": tuple(user_ids)}) + cur.execute(query=query) + ids = cur.fetchall() + return [s["session_id"] for s in ids] def delete_sessions_by_session_ids(session_ids): with pg_client.PostgresClient(unlimited_query=True) as cur: query = cur.mogrify( - """\ - DELETE FROM public.sessions - WHERE - session_id IN %(session_ids)s;""", + """DELETE FROM public.sessions + WHERE session_id IN %(session_ids)s;""", {"session_ids": tuple(session_ids)} ) cur.execute(query=query) @@ -1423,20 +1421,6 @@ def delete_sessions_by_session_ids(session_ids): return True -def delete_sessions_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """\ - DELETE FROM public.sessions - WHERE - project_id = %(project_id)s AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)} - ) - cur.execute(query=query) - - return True - - def count_all(): with ch_client.ClickHouseClient() as cur: row = cur.execute(query=f"SELECT COUNT(session_id) AS count FROM {exp_ch_helper.get_main_sessions_table()}") From 5b8d0957644017b9a6cca1ea294b5f3697dd57a7 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 13:37:16 +0100 Subject: [PATCH 08/28] feat(chalice): debugging jobs execution --- api/chalicelib/core/jobs.py | 12 ++++-------- api/chalicelib/core/sessions.py | 3 +++ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/api/chalicelib/core/jobs.py b/api/chalicelib/core/jobs.py index 69a777511..783cbad2c 100644 --- a/api/chalicelib/core/jobs.py +++ b/api/chalicelib/core/jobs.py @@ -106,17 +106,13 @@ def format_datetime(r): def get_scheduled_jobs(): with pg_client.PostgresClient() as cur: query = cur.mogrify( - """SELECT * FROM public.jobs - WHERE status = %(status)s AND start_at <= (now() at time zone 'utc');""", + """SELECT * + FROM public.jobs + WHERE status = %(status)s + AND start_at <= (now() at time zone 'utc');""", {"status": JobStatus.SCHEDULED}) - print("------------------") - print(query) - print("------------------") cur.execute(query=query) data = cur.fetchall() - for record in data: - format_datetime(record) - return helper.list_to_camel_case(data) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 7de8253da..9edaa280c 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1073,6 +1073,9 @@ def get_session_ids_by_user_ids(project_id, user_ids): WHERE project_id = %(project_id)s AND user_id IN %(userId)s;""", {"project_id": project_id, "userId": tuple(user_ids)}) + print("----------") + print(query) + print("----------") cur.execute(query=query) ids = cur.fetchall() return [s["session_id"] for s in ids] From 9053719aef30d423cdb96bfb50b1130303524156 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 13:48:52 +0100 Subject: [PATCH 09/28] feat(chalice): debugging jobs execution --- api/chalicelib/core/jobs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/chalicelib/core/jobs.py b/api/chalicelib/core/jobs.py index 783cbad2c..88554e314 100644 --- a/api/chalicelib/core/jobs.py +++ b/api/chalicelib/core/jobs.py @@ -111,13 +111,18 @@ def get_scheduled_jobs(): WHERE status = %(status)s AND start_at <= (now() at time zone 'utc');""", {"status": JobStatus.SCHEDULED}) + print(query) cur.execute(query=query) data = cur.fetchall() + print(">>>") + print(data) return helper.list_to_camel_case(data) def execute_jobs(): + print(">>> looking for jobs to execute") jobs = get_scheduled_jobs() + print(jobs) for job in jobs: print(f"Executing jobId:{job['jobId']}") From cea886b61995b08d2fe2ca8b21f476e10d5eb536 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 14:07:47 +0100 Subject: [PATCH 10/28] feat(chalice): debugging jobs execution --- api/chalicelib/core/jobs.py | 6 ------ api/chalicelib/utils/s3.py | 3 +++ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/api/chalicelib/core/jobs.py b/api/chalicelib/core/jobs.py index 88554e314..0c78859be 100644 --- a/api/chalicelib/core/jobs.py +++ b/api/chalicelib/core/jobs.py @@ -111,19 +111,13 @@ def get_scheduled_jobs(): WHERE status = %(status)s AND start_at <= (now() at time zone 'utc');""", {"status": JobStatus.SCHEDULED}) - print(query) cur.execute(query=query) data = cur.fetchall() - print(">>>") - print(data) return helper.list_to_camel_case(data) def execute_jobs(): - print(">>> looking for jobs to execute") jobs = get_scheduled_jobs() - print(jobs) - for job in jobs: print(f"Executing jobId:{job['jobId']}") try: diff --git a/api/chalicelib/utils/s3.py b/api/chalicelib/utils/s3.py index 655628602..d1acff558 100644 --- a/api/chalicelib/utils/s3.py +++ b/api/chalicelib/utils/s3.py @@ -110,11 +110,14 @@ def rename(source_bucket, source_key, target_bucket, target_key): def schedule_for_deletion(bucket, key): + if not exists(bucket, key): + return False s3 = __get_s3_resource() s3_object = s3.Object(bucket, key) s3_object.copy_from(CopySource={'Bucket': bucket, 'Key': key}, Expires=datetime.now() + timedelta(days=7), MetadataDirective='REPLACE') + return True def generate_file_key(project_id, key): From 59905075c6cc61f76cecf66feb733e81d6fdec8e Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 14:36:51 +0100 Subject: [PATCH 11/28] feat(chalice): fixing jobs execution --- api/chalicelib/core/jobs.py | 8 +++----- api/chalicelib/core/sessions.py | 3 --- api/chalicelib/core/sessions_devtool.py | 6 ++++++ ee/api/chalicelib/core/sessions_devtool.py | 6 ++++++ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/api/chalicelib/core/jobs.py b/api/chalicelib/core/jobs.py index 0c78859be..bc5ce81ab 100644 --- a/api/chalicelib/core/jobs.py +++ b/api/chalicelib/core/jobs.py @@ -1,6 +1,6 @@ from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC -from chalicelib.core import sessions, sessions_mobs +from chalicelib.core import sessions, sessions_mobs, sessions_devtool class Actions: @@ -128,17 +128,15 @@ def execute_jobs(): print(f"Deleting {len(session_ids)} sessions") sessions.delete_sessions_by_session_ids(session_ids) sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) + sessions_devtool.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) else: raise Exception(f"The action '{job['action']}' not supported.") job["status"] = JobStatus.COMPLETED print(f"Job completed {job['jobId']}") except Exception as e: - print("-----") - print(e) - print("-----") job["status"] = JobStatus.FAILED - job["error"] = str(e) + job["errors"] = str(e) print(f"Job failed {job['jobId']}") update(job["jobId"], job) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 9edaa280c..7de8253da 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1073,9 +1073,6 @@ def get_session_ids_by_user_ids(project_id, user_ids): WHERE project_id = %(project_id)s AND user_id IN %(userId)s;""", {"project_id": project_id, "userId": tuple(user_ids)}) - print("----------") - print(query) - print("----------") cur.execute(query=query) ids = cur.fetchall() return [s["session_id"] for s in ids] diff --git a/api/chalicelib/core/sessions_devtool.py b/api/chalicelib/core/sessions_devtool.py index 6aab5a5e2..50af2bb39 100644 --- a/api/chalicelib/core/sessions_devtool.py +++ b/api/chalicelib/core/sessions_devtool.py @@ -24,3 +24,9 @@ def get_urls(session_id, project_id, check_existence: bool = True): ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) )) return results + + +def delete_mobs(project_id, session_ids): + for session_id in session_ids: + for k in __get_devtools_keys(project_id=project_id, session_id=session_id): + s3.schedule_for_deletion(config("sessions_bucket"), k) diff --git a/ee/api/chalicelib/core/sessions_devtool.py b/ee/api/chalicelib/core/sessions_devtool.py index 9961df360..198466f65 100644 --- a/ee/api/chalicelib/core/sessions_devtool.py +++ b/ee/api/chalicelib/core/sessions_devtool.py @@ -31,3 +31,9 @@ def get_urls(session_id, project_id, context: schemas_ee.CurrentContext, check_e ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) )) return results + + +def delete_mobs(project_id, session_ids): + for session_id in session_ids: + for k in __get_devtools_keys(project_id=project_id, session_id=session_id): + s3.schedule_for_deletion(config("sessions_bucket"), k) From fb7902c424578192842fc0a64f94a86bbd2a9fa2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 14:51:15 +0100 Subject: [PATCH 12/28] feat(chalice): configurable mobs expiration --- api/chalicelib/utils/s3.py | 2 +- api/env.default | 3 ++- ee/api/env.default | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/api/chalicelib/utils/s3.py b/api/chalicelib/utils/s3.py index d1acff558..cdd22aa4e 100644 --- a/api/chalicelib/utils/s3.py +++ b/api/chalicelib/utils/s3.py @@ -115,7 +115,7 @@ def schedule_for_deletion(bucket, key): s3 = __get_s3_resource() s3_object = s3.Object(bucket, key) s3_object.copy_from(CopySource={'Bucket': bucket, 'Key': key}, - Expires=datetime.now() + timedelta(days=7), + Expires=datetime.utcnow() + timedelta(days=config("SCH_DELETE_DAYS", cast=int, default=7)), MetadataDirective='REPLACE') return True diff --git a/api/env.default b/api/env.default index 074d9b643..e0560619f 100644 --- a/api/env.default +++ b/api/env.default @@ -53,4 +53,5 @@ PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= PYTHONUNBUFFERED=1 -REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 \ No newline at end of file +REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 +SCH_DELETE_DAYS=7 \ No newline at end of file diff --git a/ee/api/env.default b/ee/api/env.default index 1947e9847..603c291b0 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -73,4 +73,5 @@ PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= KAFKA_SERVERS=kafka.db.svc.cluster.local:9092 -KAFKA_USE_SSL=false \ No newline at end of file +KAFKA_USE_SSL=false +SCH_DELETE_DAYS=7 \ No newline at end of file From 7e2fa078558ec8c6bf8a2ee5a64ffcc54297aa19 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 15:07:23 +0100 Subject: [PATCH 13/28] feat(chalice): changes --- .github/workflows/api-ee.yaml | 1 - .github/workflows/api.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/api-ee.yaml b/.github/workflows/api-ee.yaml index ed81c8972..f9a1730f1 100644 --- a/.github/workflows/api-ee.yaml +++ b/.github/workflows/api-ee.yaml @@ -10,7 +10,6 @@ on: branches: - dev - api-* - - v1.11.0-patch paths: - "ee/api/**" - "api/**" diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index 451ae64b5..8e2f7fa7b 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -10,7 +10,6 @@ on: branches: - dev - api-* - - v1.11.0-patch paths: - "api/**" - "!api/.gitignore" From 5f1b97a3d83203e82192a0cb7eba83b9f4cb870d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 15:23:50 +0100 Subject: [PATCH 14/28] feat(chalice): refactored Jobs feat(chalice): added limits on Jobs --- api/chalicelib/core/jobs.py | 34 +++++++++++++++++++++++--- api/chalicelib/core/sessions.py | 25 ------------------- api/chalicelib/utils/s3.py | 2 +- api/env.default | 2 +- ee/api/chalicelib/core/sessions_exp.py | 25 ------------------- ee/api/env.default | 2 +- 6 files changed, 33 insertions(+), 57 deletions(-) diff --git a/api/chalicelib/core/jobs.py b/api/chalicelib/core/jobs.py index bc5ce81ab..8ca556fdf 100644 --- a/api/chalicelib/core/jobs.py +++ b/api/chalicelib/core/jobs.py @@ -1,6 +1,6 @@ from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC -from chalicelib.core import sessions, sessions_mobs, sessions_devtool +from chalicelib.core import sessions_mobs, sessions_devtool class Actions: @@ -103,6 +103,32 @@ def format_datetime(r): r["start_at"] = TimeUTC.datetime_to_timestamp(r["start_at"]) +def __get_session_ids_by_user_ids(project_id, user_ids): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """SELECT session_id + FROM public.sessions + WHERE project_id = %(project_id)s + AND user_id IN %(userId)s;""", + {"project_id": project_id, "userId": tuple(user_ids)}) + cur.execute(query=query) + ids = cur.fetchall() + return [s["session_id"] for s in ids] + + +def __delete_sessions_by_session_ids(session_ids): + with pg_client.PostgresClient(unlimited_query=True) as cur: + query = cur.mogrify( + """DELETE FROM public.sessions + WHERE session_id IN %(session_ids)s + LIMIT 1000;""", + {"session_ids": tuple(session_ids)} + ) + cur.execute(query=query) + + return True + + def get_scheduled_jobs(): with pg_client.PostgresClient() as cur: query = cur.mogrify( @@ -122,11 +148,11 @@ def execute_jobs(): print(f"Executing jobId:{job['jobId']}") try: if job["action"] == Actions.DELETE_USER_DATA: - session_ids = sessions.get_session_ids_by_user_ids(project_id=job["projectId"], - user_ids=[job["referenceId"]]) + session_ids = __get_session_ids_by_user_ids(project_id=job["projectId"], + user_ids=[job["referenceId"]]) if len(session_ids) > 0: print(f"Deleting {len(session_ids)} sessions") - sessions.delete_sessions_by_session_ids(session_ids) + __delete_sessions_by_session_ids(session_ids) sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) sessions_devtool.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) else: diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 7de8253da..e7283755e 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1065,31 +1065,6 @@ def get_session_user(project_id, user_id): return helper.dict_to_camel_case(data) -def get_session_ids_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient() as cur: - query = cur.mogrify( - """SELECT session_id - FROM public.sessions - WHERE project_id = %(project_id)s - AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)}) - cur.execute(query=query) - ids = cur.fetchall() - return [s["session_id"] for s in ids] - - -def delete_sessions_by_session_ids(session_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """DELETE FROM public.sessions - WHERE session_id IN %(session_ids)s;""", - {"session_ids": tuple(session_ids)} - ) - cur.execute(query=query) - - return True - - def count_all(): with pg_client.PostgresClient(unlimited_query=True) as cur: cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions") diff --git a/api/chalicelib/utils/s3.py b/api/chalicelib/utils/s3.py index cdd22aa4e..6eeacd261 100644 --- a/api/chalicelib/utils/s3.py +++ b/api/chalicelib/utils/s3.py @@ -115,7 +115,7 @@ def schedule_for_deletion(bucket, key): s3 = __get_s3_resource() s3_object = s3.Object(bucket, key) s3_object.copy_from(CopySource={'Bucket': bucket, 'Key': key}, - Expires=datetime.utcnow() + timedelta(days=config("SCH_DELETE_DAYS", cast=int, default=7)), + Expires=datetime.utcnow() + timedelta(days=config("SCH_DELETE_DAYS", cast=int, default=30)), MetadataDirective='REPLACE') return True diff --git a/api/env.default b/api/env.default index e0560619f..7d5266470 100644 --- a/api/env.default +++ b/api/env.default @@ -54,4 +54,4 @@ ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= PYTHONUNBUFFERED=1 REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 -SCH_DELETE_DAYS=7 \ No newline at end of file +SCH_DELETE_DAYS=30 \ No newline at end of file diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index 1c4d20883..b7539c3c2 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -1396,31 +1396,6 @@ def get_session_user(project_id, user_id): return helper.dict_to_camel_case(data) -def get_session_ids_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient() as cur: - query = cur.mogrify( - """SELECT session_id - FROM public.sessions - WHERE project_id = %(project_id)s - AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)}) - cur.execute(query=query) - ids = cur.fetchall() - return [s["session_id"] for s in ids] - - -def delete_sessions_by_session_ids(session_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """DELETE FROM public.sessions - WHERE session_id IN %(session_ids)s;""", - {"session_ids": tuple(session_ids)} - ) - cur.execute(query=query) - - return True - - def count_all(): with ch_client.ClickHouseClient() as cur: row = cur.execute(query=f"SELECT COUNT(session_id) AS count FROM {exp_ch_helper.get_main_sessions_table()}") diff --git a/ee/api/env.default b/ee/api/env.default index 603c291b0..a35f6f451 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -74,4 +74,4 @@ ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= KAFKA_SERVERS=kafka.db.svc.cluster.local:9092 KAFKA_USE_SSL=false -SCH_DELETE_DAYS=7 \ No newline at end of file +SCH_DELETE_DAYS=30 \ No newline at end of file From 7b0fe221d0cbbc57ad234a50a29ddf30509a736e Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 15:24:42 +0100 Subject: [PATCH 15/28] chore(build): test patch branch --- .github/workflows/api-ee.yaml | 1 + .github/workflows/api.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/api-ee.yaml b/.github/workflows/api-ee.yaml index f9a1730f1..ed81c8972 100644 --- a/.github/workflows/api-ee.yaml +++ b/.github/workflows/api-ee.yaml @@ -10,6 +10,7 @@ on: branches: - dev - api-* + - v1.11.0-patch paths: - "ee/api/**" - "api/**" diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index 8e2f7fa7b..451ae64b5 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -10,6 +10,7 @@ on: branches: - dev - api-* + - v1.11.0-patch paths: - "api/**" - "!api/.gitignore" From 172c3072993602e7e82b7df82559ff4168118e55 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 15:59:17 +0100 Subject: [PATCH 16/28] feat(chalice): fixing Jobs --- api/chalicelib/core/jobs.py | 21 ++++++++++----------- api/routers/subs/v1_api.py | 37 +++++++++++-------------------------- 2 files changed, 21 insertions(+), 37 deletions(-) diff --git a/api/chalicelib/core/jobs.py b/api/chalicelib/core/jobs.py index 8ca556fdf..3c812ad33 100644 --- a/api/chalicelib/core/jobs.py +++ b/api/chalicelib/core/jobs.py @@ -47,13 +47,14 @@ def get_all(project_id): return helper.list_to_camel_case(data) -def create(project_id, data): +def create(project_id, user_id): with pg_client.PostgresClient() as cur: - job = { - "status": "scheduled", - "project_id": project_id, - **data - } + job = {"status": "scheduled", + "project_id": project_id, + "action": Actions.DELETE_USER_DATA, + "reference_id": user_id, + "description": f"Delete user sessions of userId = {user_id}", + "start_at": TimeUTC.to_human_readable(TimeUTC.midnight(1))} query = cur.mogrify( """INSERT INTO public.jobs(project_id, description, status, action,reference_id, start_at) @@ -109,7 +110,8 @@ def __get_session_ids_by_user_ids(project_id, user_ids): """SELECT session_id FROM public.sessions WHERE project_id = %(project_id)s - AND user_id IN %(userId)s;""", + AND user_id IN %(userId)s + LIMIT 1000;""", {"project_id": project_id, "userId": tuple(user_ids)}) cur.execute(query=query) ids = cur.fetchall() @@ -120,14 +122,11 @@ def __delete_sessions_by_session_ids(session_ids): with pg_client.PostgresClient(unlimited_query=True) as cur: query = cur.mogrify( """DELETE FROM public.sessions - WHERE session_id IN %(session_ids)s - LIMIT 1000;""", + WHERE session_id IN %(session_ids)s""", {"session_ids": tuple(session_ids)} ) cur.execute(query=query) - return True - def get_scheduled_jobs(): with pg_client.PostgresClient() as cur: diff --git a/api/routers/subs/v1_api.py b/api/routers/subs/v1_api.py index 0759d0c31..ca98aaf5f 100644 --- a/api/routers/subs/v1_api.py +++ b/api/routers/subs/v1_api.py @@ -2,7 +2,6 @@ from fastapi import Depends, Body import schemas from chalicelib.core import sessions, events, jobs, projects -from chalicelib.utils.TimeUTC import TimeUTC from or_dependencies import OR_context from routers.base import get_routers @@ -15,7 +14,7 @@ async def get_user_sessions(projectKey: str, userId: str, start_date: int = None if projectId is None: return {"errors": ["invalid projectKey"]} return { - 'data': sessions.get_user_sessions( + "data": sessions.get_user_sessions( project_id=projectId, user_id=userId, start_date=start_date, @@ -30,7 +29,7 @@ async def get_session_events(projectKey: str, sessionId: int): if projectId is None: return {"errors": ["invalid projectKey"]} return { - 'data': events.get_by_session_id( + "data": events.get_by_session_id( project_id=projectId, session_id=sessionId ) @@ -43,7 +42,7 @@ async def get_user_details(projectKey: str, userId: str): if projectId is None: return {"errors": ["invalid projectKey"]} return { - 'data': sessions.get_session_user( + "data": sessions.get_session_user( project_id=projectId, user_id=userId ) @@ -55,14 +54,8 @@ async def schedule_to_delete_user_data(projectKey: str, userId: str): projectId = projects.get_internal_project_id(projectKey) if projectId is None: return {"errors": ["invalid projectKey"]} - data = {"action": "delete_user_data", - "reference_id": userId, - "description": f"Delete user sessions of userId = {userId}", - "start_at": TimeUTC.to_human_readable(TimeUTC.midnight(1))} - record = jobs.create(project_id=projectId, data=data) - return { - 'data': record - } + record = jobs.create(project_id=projectId, user_id=userId) + return {"data": record} @app_apikey.get('/v1/{projectKey}/jobs', tags=["api"]) @@ -70,16 +63,12 @@ async def get_jobs(projectKey: str): projectId = projects.get_internal_project_id(projectKey) if projectId is None: return {"errors": ["invalid projectKey"]} - return { - 'data': jobs.get_all(project_id=projectId) - } + return {"data": jobs.get_all(project_id=projectId)} @app_apikey.get('/v1/{projectKey}/jobs/{jobId}', tags=["api"]) async def get_job(projectKey: str, jobId: int): - return { - 'data': jobs.get(job_id=jobId) - } + return {"data": jobs.get(job_id=jobId)} @app_apikey.delete('/v1/{projectKey}/jobs/{jobId}', tags=["api"]) @@ -93,9 +82,7 @@ async def cancel_job(projectKey: str, jobId: int): return {"errors": ["The request job has already been canceled/completed."]} job["status"] = "cancelled" - return { - 'data': jobs.update(job_id=jobId, job=job) - } + return {"data": jobs.update(job_id=jobId, job=job)} @app_apikey.get('/v1/projects', tags=["api"]) @@ -104,15 +91,13 @@ async def get_projects(context: schemas.CurrentContext = Depends(OR_context)): for record in records: del record['projectId'] - return { - 'data': records - } + return {"data": records} @app_apikey.get('/v1/projects/{projectKey}', tags=["api"]) async def get_project(projectKey: str, context: schemas.CurrentContext = Depends(OR_context)): return { - 'data': projects.get_project_by_key(tenant_id=context.tenant_id, project_key=projectKey) + "data": projects.get_project_by_key(tenant_id=context.tenant_id, project_key=projectKey) } @@ -125,5 +110,5 @@ async def create_project(data: schemas.CreateProjectSchema = Body(...), data=data, skip_authorization=True ) - del record['data']['projectId'] + del record["data"]['projectId'] return record From 24c822c64eb3a2d70b26fd0eaf32bb31736d110a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 16:19:33 +0100 Subject: [PATCH 17/28] feat(chalice): changed corn-Job execution time --- api/routers/crons/core_dynamic_crons.py | 3 +-- ee/api/routers/crons/core_dynamic_crons.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/api/routers/crons/core_dynamic_crons.py b/api/routers/crons/core_dynamic_crons.py index a79ce705d..0fdeb242a 100644 --- a/api/routers/crons/core_dynamic_crons.py +++ b/api/routers/crons/core_dynamic_crons.py @@ -1,5 +1,4 @@ from apscheduler.triggers.cron import CronTrigger -from apscheduler.triggers.interval import IntervalTrigger from chalicelib.core import telemetry from chalicelib.core import weekly_report, jobs @@ -20,7 +19,7 @@ async def telemetry_cron() -> None: cron_jobs = [ {"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"), "misfire_grace_time": 60 * 60, "max_instances": 1}, - {"func": run_scheduled_jobs, "trigger": IntervalTrigger(minutes=1), + {"func": run_scheduled_jobs, "trigger": CronTrigger(day_of_week="*", hour=0, minute=15), "misfire_grace_time": 20, "max_instances": 1}, {"func": weekly_report2, "trigger": CronTrigger(day_of_week="mon", hour=5), "misfire_grace_time": 60 * 60, "max_instances": 1} diff --git a/ee/api/routers/crons/core_dynamic_crons.py b/ee/api/routers/crons/core_dynamic_crons.py index 5d13c90d1..9febbe18a 100644 --- a/ee/api/routers/crons/core_dynamic_crons.py +++ b/ee/api/routers/crons/core_dynamic_crons.py @@ -1,5 +1,4 @@ from apscheduler.triggers.cron import CronTrigger -from apscheduler.triggers.interval import IntervalTrigger from decouple import config from chalicelib.core import jobs @@ -31,7 +30,7 @@ cron_jobs = [ SINGLE_CRONS = [{"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"), "misfire_grace_time": 60 * 60, "max_instances": 1}, - {"func": run_scheduled_jobs, "trigger": IntervalTrigger(minutes=60), + {"func": run_scheduled_jobs, "trigger": CronTrigger(day_of_week="*", hour=0, minute=15), "misfire_grace_time": 20, "max_instances": 1}, {"func": weekly_report, "trigger": CronTrigger(day_of_week="mon", hour=5), "misfire_grace_time": 60 * 60, "max_instances": 1} From 668b9cb3830ec7b42dca3e23b973a1b68879057e Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Apr 2023 16:36:51 +0100 Subject: [PATCH 18/28] chore(build): testing EE cron-Jobs --- .github/workflows/crons-ee.yaml | 1 + ee/api/routers/crons/core_dynamic_crons.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/crons-ee.yaml b/.github/workflows/crons-ee.yaml index 77c098e4e..b357fea65 100644 --- a/.github/workflows/crons-ee.yaml +++ b/.github/workflows/crons-ee.yaml @@ -10,6 +10,7 @@ on: branches: - dev - api-* + - v1.11.0-patch paths: - "ee/api/**" - "api/**" diff --git a/ee/api/routers/crons/core_dynamic_crons.py b/ee/api/routers/crons/core_dynamic_crons.py index 9febbe18a..89846366b 100644 --- a/ee/api/routers/crons/core_dynamic_crons.py +++ b/ee/api/routers/crons/core_dynamic_crons.py @@ -28,13 +28,14 @@ cron_jobs = [ {"func": unlock_cron, "trigger": CronTrigger(day="*")}, ] -SINGLE_CRONS = [{"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"), - "misfire_grace_time": 60 * 60, "max_instances": 1}, - {"func": run_scheduled_jobs, "trigger": CronTrigger(day_of_week="*", hour=0, minute=15), - "misfire_grace_time": 20, "max_instances": 1}, - {"func": weekly_report, "trigger": CronTrigger(day_of_week="mon", hour=5), - "misfire_grace_time": 60 * 60, "max_instances": 1} - ] +SINGLE_CRONS = [ + {"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"), + "misfire_grace_time": 60 * 60, "max_instances": 1}, + {"func": run_scheduled_jobs, "trigger": CronTrigger(day_of_week="*", hour=0, minute=15), + "misfire_grace_time": 20, "max_instances": 1}, + {"func": weekly_report, "trigger": CronTrigger(day_of_week="mon", hour=5), + "misfire_grace_time": 60 * 60, "max_instances": 1} +] if config("LOCAL_CRONS", default=False, cast=bool): cron_jobs += SINGLE_CRONS From a5bf2531e5db95ebf448b448f18d3c13ae584c29 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 12 Apr 2023 18:47:06 +0200 Subject: [PATCH 19/28] Add files via upload (#1156) --- static/openreplay-git-hero.svg | 500 ++++++++++++++++----------------- 1 file changed, 245 insertions(+), 255 deletions(-) diff --git a/static/openreplay-git-hero.svg b/static/openreplay-git-hero.svg index bf294a353..6a750f4d8 100644 --- a/static/openreplay-git-hero.svg +++ b/static/openreplay-git-hero.svg @@ -1,421 +1,411 @@ - + - - + + - + - + - - - + + + - + - - - - - - - - - + + + + + + + + + - + - - - + + + - - - + + + - + - - + + - + - + - - - + + + - + - - + + - + - - - - + + + + - + - - - + + + - + - - - - - - - - + + + + + + + + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + + - - - + + + - - + + - - - + + - - + + - - + + - - + + - - - + + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - + - + - - - - + + + + - + - - - + + + - + - - - + + + - - + + - - - + + + - + - - - + + + - + - - + + - + - - - - + + + + - + - - - + + + - + - - - + + + - + - - - + + + + + + + + - - - + + + - - - - - + + + + + - + - - + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - + + + - - - - - - - - + + + + + + + + - - - + + + - + - - + - + - + - + - + - + - - + + - - + + - - + + - + - + - + - - - - - - - + + - + + + + + + - - + + - - + + - - + + - - - - - - - - - - - + - + - + - + From bf546e8e3a6c5e122c188a5baa0e15ac29fff05a Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 12 Apr 2023 19:10:50 +0200 Subject: [PATCH 20/28] Add files via upload (#1157) --- static/replay-thumbnail.svg | 148 ++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 static/replay-thumbnail.svg diff --git a/static/replay-thumbnail.svg b/static/replay-thumbnail.svg new file mode 100644 index 000000000..9f550f423 --- /dev/null +++ b/static/replay-thumbnail.svg @@ -0,0 +1,148 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From cb64fcb3f028f2b79b2541ee79bef4503a265f3c Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Thu, 13 Apr 2023 10:10:20 +0200 Subject: [PATCH 21/28] chore(helm): Enabling redis string for helm template variable (#1159) fix #1158 Signed-off-by: rjshrjndrn --- scripts/helmcharts/vars.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 6aebf266f..46bad0f00 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -50,7 +50,7 @@ kafka: &kafka # value: "3000000" redis: &redis - # enabled: false + enabled: true redisHost: "redis-master.db.svc.cluster.local" redisPort: "6379" From 00e136f9cb5b853568a9cfb5f0496f88372500b4 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Thu, 13 Apr 2023 13:07:43 +0200 Subject: [PATCH 22/28] Changing default encryption to false (#1161) --- scripts/helmcharts/databases/charts/clickhouse/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/databases/charts/clickhouse/values.yaml b/scripts/helmcharts/databases/charts/clickhouse/values.yaml index f3f39aefd..05a10c879 100644 --- a/scripts/helmcharts/databases/charts/clickhouse/values.yaml +++ b/scripts/helmcharts/databases/charts/clickhouse/values.yaml @@ -84,4 +84,4 @@ nodeSelector: {} tolerations: [] affinity: {} -storageSize: 100G +storageSize: 100Gi From d55e2fdc7806c9e500f1b2de04183eef934ff02c Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Thu, 13 Apr 2023 19:25:39 +0200 Subject: [PATCH 23/28] Updated hero --- static/openreplay-git-hero.svg | 147 ++++++++++++++------------------- 1 file changed, 63 insertions(+), 84 deletions(-) diff --git a/static/openreplay-git-hero.svg b/static/openreplay-git-hero.svg index 6a750f4d8..079ac5efc 100644 --- a/static/openreplay-git-hero.svg +++ b/static/openreplay-git-hero.svg @@ -85,12 +85,7 @@ - - - - - @@ -103,47 +98,47 @@ - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -163,72 +158,72 @@ - - - - - - + + + + - - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -237,21 +232,21 @@ - + - + - + - + - + - + @@ -260,7 +255,7 @@ - + @@ -291,30 +286,34 @@ - + - + - + - + - + - + + + + + @@ -335,45 +334,22 @@ - + - - - - - - - - - - - - - - - - + + - - - - - - - - - - + - + - + @@ -407,5 +383,8 @@ + + + From 2751334876a5d4f9113fdc73cb26e7c5691cd5b6 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Apr 2023 00:28:16 +0100 Subject: [PATCH 24/28] feat(chalice): return all records if date is not specified --- ee/api/chalicelib/core/assist_records.py | 7 +++++-- ee/api/schemas_ee.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ee/api/chalicelib/core/assist_records.py b/ee/api/chalicelib/core/assist_records.py index 18449dc14..e2caa4d56 100644 --- a/ee/api/chalicelib/core/assist_records.py +++ b/ee/api/chalicelib/core/assist_records.py @@ -41,9 +41,12 @@ def save_record(project_id, data: schemas_ee.AssistRecordSavePayloadSchema, cont def search_records(project_id, data: schemas_ee.AssistRecordSearchPayloadSchema, context: schemas_ee.CurrentContext): conditions = ["projects.tenant_id=%(tenant_id)s", "projects.deleted_at ISNULL", - "assist_records.created_at>=%(startDate)s", - "assist_records.created_at<=%(endDate)s", "assist_records.deleted_at ISNULL"] + if data.startDate: + conditions.append("assist_records.created_at>=%(startDate)s") + if data.endDate: + conditions.append("assist_records.created_at<=%(endDate)s") + params = {"tenant_id": context.tenant_id, "project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, "p_start": (data.page - 1) * data.limit, "p_limit": data.limit, diff --git a/ee/api/schemas_ee.py b/ee/api/schemas_ee.py index f5a3b8de0..50136ac02 100644 --- a/ee/api/schemas_ee.py +++ b/ee/api/schemas_ee.py @@ -137,8 +137,8 @@ class AssistRecordSavePayloadSchema(AssistRecordPayloadSchema): class AssistRecordSearchPayloadSchema(schemas._PaginatedSchema): limit: int = Field(default=200, gt=0) - startDate: int = Field(default=TimeUTC.now(-7)) - endDate: int = Field(default=TimeUTC.now(1)) + startDate: Optional[int] = Field(default=None) + endDate: Optional[int] = Field(default=None) user_id: Optional[int] = Field(default=None) query: Optional[str] = Field(default=None) order: Literal["asc", "desc"] = Field(default="desc") From 961ea9904812838fd309de15d491a00c55db18e8 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Apr 2023 00:44:57 +0100 Subject: [PATCH 25/28] feat(chalice): refactored records list --- ee/api/chalicelib/core/assist_records.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ee/api/chalicelib/core/assist_records.py b/ee/api/chalicelib/core/assist_records.py index e2caa4d56..be63e1524 100644 --- a/ee/api/chalicelib/core/assist_records.py +++ b/ee/api/chalicelib/core/assist_records.py @@ -41,6 +41,7 @@ def save_record(project_id, data: schemas_ee.AssistRecordSavePayloadSchema, cont def search_records(project_id, data: schemas_ee.AssistRecordSearchPayloadSchema, context: schemas_ee.CurrentContext): conditions = ["projects.tenant_id=%(tenant_id)s", "projects.deleted_at ISNULL", + "projects.project_id=%(project_id)s", "assist_records.deleted_at ISNULL"] if data.startDate: conditions.append("assist_records.created_at>=%(startDate)s") From 0316999528fe1685d79ff3b52ddc85dd22310f0a Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 22 Apr 2023 11:37:31 +0200 Subject: [PATCH 26/28] Moving cli to scripts folder (#1196) --- scripts/{helmcharts => }/openreplay-cli | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{helmcharts => }/openreplay-cli (100%) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/openreplay-cli similarity index 100% rename from scripts/helmcharts/openreplay-cli rename to scripts/openreplay-cli From ae8e22709a0e19fe895966e6808eb9d2aec98a57 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 22 Apr 2023 11:41:38 +0200 Subject: [PATCH 27/28] Revert "Moving cli to scripts folder (#1196)" (#1197) This reverts commit c947e48d9915ec3779b14997897d832b3494a209. --- scripts/{ => helmcharts}/openreplay-cli | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{ => helmcharts}/openreplay-cli (100%) diff --git a/scripts/openreplay-cli b/scripts/helmcharts/openreplay-cli similarity index 100% rename from scripts/openreplay-cli rename to scripts/helmcharts/openreplay-cli From be903c80e0fe8beee1a65143481cf7d36e07bf9f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 26 Apr 2023 12:03:09 +0200 Subject: [PATCH 28/28] feat(chalice): support old FilterType --- api/schemas.py | 62 +++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/api/schemas.py b/api/schemas.py index a7f1eb683..81089e2b2 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -649,6 +649,36 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw): value: Union[List[Union[_SessionSearchEventRaw, str]], str] = Field(...) +def transform_old_FilterType(cls, values): + if values.get("type") is None: + return values + values["type"] = { + "USEROS": FilterType.user_os.value, + "USERBROWSER": FilterType.user_browser.value, + "USERDEVICE": FilterType.user_device.value, + "USERCOUNTRY": FilterType.user_country.value, + "USERID": FilterType.user_id.value, + "USERANONYMOUSID": FilterType.user_anonymous_id.value, + "REFERRER": FilterType.referrer.value, + "REVID": FilterType.rev_id.value, + "USEROS_IOS": FilterType.user_os_ios.value, + "USERDEVICE_IOS": FilterType.user_device_ios.value, + "USERCOUNTRY_IOS": FilterType.user_country_ios.value, + "USERID_IOS": FilterType.user_id_ios.value, + "USERANONYMOUSID_IOS": FilterType.user_anonymous_id_ios.value, + "REVID_IOS": FilterType.rev_id_ios.value, + "DURATION": FilterType.duration.value, + "PLATFORM": FilterType.platform.value, + "METADATA": FilterType.metadata.value, + "ISSUE": FilterType.issue.value, + "EVENTS_COUNT": FilterType.events_count.value, + "UTM_SOURCE": FilterType.utm_source.value, + "UTM_MEDIUM": FilterType.utm_medium.value, + "UTM_CAMPAIGN": FilterType.utm_campaign.value + }.get(values["type"], values["type"]) + return values + + class SessionSearchFilterSchema(__MixedSearchFilter): is_event: bool = Field(False, const=False) # TODO: remove this if there nothing broken from the UI @@ -660,35 +690,7 @@ class SessionSearchFilterSchema(__MixedSearchFilter): source: Optional[Union[ErrorSource, str]] = Field(default=None) filters: List[IssueFilterSchema] = Field(default=[]) - @root_validator(pre=True) - def transform(cls, values): - if values.get("type") is None: - return values - values["type"] = { - "USEROS": FilterType.user_os.value, - "USERBROWSER": FilterType.user_browser.value, - "USERDEVICE": FilterType.user_device.value, - "USERCOUNTRY": FilterType.user_country.value, - "USERID": FilterType.user_id.value, - "USERANONYMOUSID": FilterType.user_anonymous_id.value, - "REFERRER": FilterType.referrer.value, - "REVID": FilterType.rev_id.value, - "USEROS_IOS": FilterType.user_os_ios.value, - "USERDEVICE_IOS": FilterType.user_device_ios.value, - "USERCOUNTRY_IOS": FilterType.user_country_ios.value, - "USERID_IOS": FilterType.user_id_ios.value, - "USERANONYMOUSID_IOS": FilterType.user_anonymous_id_ios.value, - "REVID_IOS": FilterType.rev_id_ios.value, - "DURATION": FilterType.duration.value, - "PLATFORM": FilterType.platform.value, - "METADATA": FilterType.metadata.value, - "ISSUE": FilterType.issue.value, - "EVENTS_COUNT": FilterType.events_count.value, - "UTM_SOURCE": FilterType.utm_source.value, - "UTM_MEDIUM": FilterType.utm_medium.value, - "UTM_CAMPAIGN": FilterType.utm_campaign.value - }.get(values["type"], values["type"]) - return values + transform = root_validator(pre=True, allow_reuse=True)(transform_old_FilterType) @root_validator def filter_validator(cls, values): @@ -1194,6 +1196,8 @@ class LiveSessionSearchFilterSchema(BaseModel): operator: Literal[SearchEventOperator._is, \ SearchEventOperator._contains] = Field(default=SearchEventOperator._contains) + transform = root_validator(pre=True, allow_reuse=True)(transform_old_FilterType) + @root_validator def validator(cls, values): if values.get("type") is not None and values["type"] == LiveFilterType.metadata: