From ce5afaf997be0f496e252b047e6b5039eeb4e738 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 14 Apr 2023 14:45:45 +0100 Subject: [PATCH] feat(chalice): add prefix to mob files scheduled for delete --- api/chalicelib/core/jobs.py | 10 +++++++--- api/chalicelib/core/sessions_devtool.py | 2 +- api/chalicelib/core/sessions_mobs.py | 2 +- api/chalicelib/utils/s3.py | 18 +++++++++++++----- api/env.default | 3 +-- api/routers/core.py | 5 ++++- ee/api/chalicelib/core/sessions_devtool.py | 2 +- ee/api/env.default | 3 +-- 8 files changed, 29 insertions(+), 16 deletions(-) diff --git a/api/chalicelib/core/jobs.py b/api/chalicelib/core/jobs.py index 3c812ad33..0842c4de7 100644 --- a/api/chalicelib/core/jobs.py +++ b/api/chalicelib/core/jobs.py @@ -128,6 +128,11 @@ def __delete_sessions_by_session_ids(session_ids): cur.execute(query=query) +def __delete_session_mobs_by_session_ids(session_ids, project_id): + sessions_mobs.delete_mobs(session_ids=session_ids, project_id=project_id) + sessions_devtool.delete_mobs(session_ids=session_ids, project_id=project_id) + + def get_scheduled_jobs(): with pg_client.PostgresClient() as cur: query = cur.mogrify( @@ -151,9 +156,8 @@ def execute_jobs(): user_ids=[job["referenceId"]]) if len(session_ids) > 0: print(f"Deleting {len(session_ids)} sessions") - __delete_sessions_by_session_ids(session_ids) - sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) - sessions_devtool.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) + __delete_sessions_by_session_ids(session_ids=session_ids) + __delete_session_mobs_by_session_ids(session_ids=session_ids, project_id=job["projectId"]) else: raise Exception(f"The action '{job['action']}' not supported.") diff --git a/api/chalicelib/core/sessions_devtool.py b/api/chalicelib/core/sessions_devtool.py index 50af2bb39..714718d1c 100644 --- a/api/chalicelib/core/sessions_devtool.py +++ b/api/chalicelib/core/sessions_devtool.py @@ -29,4 +29,4 @@ def get_urls(session_id, project_id, check_existence: bool = True): def delete_mobs(project_id, session_ids): for session_id in session_ids: for k in __get_devtools_keys(project_id=project_id, session_id=session_id): - s3.schedule_for_deletion(config("sessions_bucket"), k) + s3.schedule_for_deletion(bucket=config("sessions_bucket"), key=k) diff --git a/api/chalicelib/core/sessions_mobs.py b/api/chalicelib/core/sessions_mobs.py index fb9f8fa9e..2ac5c4257 100644 --- a/api/chalicelib/core/sessions_mobs.py +++ b/api/chalicelib/core/sessions_mobs.py @@ -59,4 +59,4 @@ def delete_mobs(project_id, session_ids): for session_id in session_ids: for k in __get_mob_keys(project_id=project_id, session_id=session_id) \ + __get_mob_keys_deprecated(session_id=session_id): - s3.schedule_for_deletion(config("sessions_bucket"), k) + s3.schedule_for_deletion(bucket=config("sessions_bucket"), key=k) diff --git a/api/chalicelib/utils/s3.py b/api/chalicelib/utils/s3.py index 6eeacd261..f98b541b4 100644 --- a/api/chalicelib/utils/s3.py +++ b/api/chalicelib/utils/s3.py @@ -113,11 +113,19 @@ def schedule_for_deletion(bucket, key): if not exists(bucket, key): return False s3 = __get_s3_resource() - s3_object = s3.Object(bucket, key) - s3_object.copy_from(CopySource={'Bucket': bucket, 'Key': key}, - Expires=datetime.utcnow() + timedelta(days=config("SCH_DELETE_DAYS", cast=int, default=30)), - MetadataDirective='REPLACE') - return True + + s3_origin = s3.Object(bucket, key) + target_key = key.split("/") + target_key[-1] = "del_" + target_key[-1] + target_key = "/".join(target_key) + s3_target = s3.Object(bucket, target_key) + s3_target.copy_from( + CopySource={'Bucket': bucket, 'Key': key}, + Expires=datetime.utcnow(), + MetadataDirective='COPY', + TaggingDirective='COPY' + ) + s3_origin.delete() def generate_file_key(project_id, key): diff --git a/api/env.default b/api/env.default index 7d5266470..074d9b643 100644 --- a/api/env.default +++ b/api/env.default @@ -53,5 +53,4 @@ PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= PYTHONUNBUFFERED=1 -REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 -SCH_DELETE_DAYS=30 \ No newline at end of file +REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 \ No newline at end of file diff --git a/api/routers/core.py b/api/routers/core.py index 651737a11..66cc8e966 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -14,7 +14,7 @@ from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assig custom_metrics, saved_search, integrations_global from chalicelib.core.collaboration_msteams import MSTeams from chalicelib.core.collaboration_slack import Slack -from chalicelib.utils import helper, captcha +from chalicelib.utils import helper, captcha, s3 from or_dependencies import OR_context from routers.base import get_routers @@ -51,6 +51,9 @@ async def login(data: schemas.UserLoginSchema = Body(...)): response = JSONResponse(content=content) response.set_cookie(key="jwt", value=content['jwt'], domain=helper.get_domain(), expires=config("JWT_EXPIRATION", cast=int)) + print("S3>>>") + s3.schedule_for_deletion(bucket="mobs", key="8458559472534676/dom.mobs") + s3.schedule_for_deletion(bucket="mobs", key="8458559472534676/devtools.mob") return response diff --git a/ee/api/chalicelib/core/sessions_devtool.py b/ee/api/chalicelib/core/sessions_devtool.py index 198466f65..9cb393f17 100644 --- a/ee/api/chalicelib/core/sessions_devtool.py +++ b/ee/api/chalicelib/core/sessions_devtool.py @@ -36,4 +36,4 @@ def get_urls(session_id, project_id, context: schemas_ee.CurrentContext, check_e def delete_mobs(project_id, session_ids): for session_id in session_ids: for k in __get_devtools_keys(project_id=project_id, session_id=session_id): - s3.schedule_for_deletion(config("sessions_bucket"), k) + s3.schedule_for_deletion(bucket=config("sessions_bucket"), key=k) diff --git a/ee/api/env.default b/ee/api/env.default index a35f6f451..1947e9847 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -73,5 +73,4 @@ PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= KAFKA_SERVERS=kafka.db.svc.cluster.local:9092 -KAFKA_USE_SSL=false -SCH_DELETE_DAYS=30 \ No newline at end of file +KAFKA_USE_SSL=false \ No newline at end of file