feat(chalice): add prefix to mob files scheduled for delete

api/chalicelib/core/jobs.py

@@ -128,6 +128,11 @@ def __delete_sessions_by_session_ids(session_ids):
         cur.execute(query=query)
 
 
+def __delete_session_mobs_by_session_ids(session_ids, project_id):
+    sessions_mobs.delete_mobs(session_ids=session_ids, project_id=project_id)
+    sessions_devtool.delete_mobs(session_ids=session_ids, project_id=project_id)
+
+
 def get_scheduled_jobs():
     with pg_client.PostgresClient() as cur:
         query = cur.mogrify(
@@ -151,9 +156,8 @@ def execute_jobs():
                                                             user_ids=[job["referenceId"]])
                 if len(session_ids) > 0:
                     print(f"Deleting {len(session_ids)} sessions")
-                    __delete_sessions_by_session_ids(session_ids)
-                    sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"])
-                    sessions_devtool.delete_mobs(session_ids=session_ids, project_id=job["projectId"])
+                    __delete_sessions_by_session_ids(session_ids=session_ids)
+                    __delete_session_mobs_by_session_ids(session_ids=session_ids, project_id=job["projectId"])
             else:
                 raise Exception(f"The action '{job['action']}' not supported.")
 

api/chalicelib/core/sessions_devtool.py

@@ -29,4 +29,4 @@ def get_urls(session_id, project_id, check_existence: bool = True):
 def delete_mobs(project_id, session_ids):
     for session_id in session_ids:
         for k in __get_devtools_keys(project_id=project_id, session_id=session_id):
-            s3.schedule_for_deletion(config("sessions_bucket"), k)
+            s3.schedule_for_deletion(bucket=config("sessions_bucket"), key=k)

api/chalicelib/core/sessions_mobs.py

@@ -59,4 +59,4 @@ def delete_mobs(project_id, session_ids):
     for session_id in session_ids:
         for k in __get_mob_keys(project_id=project_id, session_id=session_id) \
                  + __get_mob_keys_deprecated(session_id=session_id):
-            s3.schedule_for_deletion(config("sessions_bucket"), k)
+            s3.schedule_for_deletion(bucket=config("sessions_bucket"), key=k)

api/chalicelib/utils/s3.py

@@ -113,11 +113,19 @@ def schedule_for_deletion(bucket, key):
     if not exists(bucket, key):
         return False
     s3 = __get_s3_resource()
-    s3_object = s3.Object(bucket, key)
-    s3_object.copy_from(CopySource={'Bucket': bucket, 'Key': key},
-                        Expires=datetime.utcnow() + timedelta(days=config("SCH_DELETE_DAYS", cast=int, default=30)),
-                        MetadataDirective='REPLACE')
-    return True
+
+    s3_origin = s3.Object(bucket, key)
+    target_key = key.split("/")
+    target_key[-1] = "del_" + target_key[-1]
+    target_key = "/".join(target_key)
+    s3_target = s3.Object(bucket, target_key)
+    s3_target.copy_from(
+        CopySource={'Bucket': bucket, 'Key': key},
+        Expires=datetime.utcnow(),
+        MetadataDirective='COPY',
+        TaggingDirective='COPY'
+    )
+    s3_origin.delete()
 
 
 def generate_file_key(project_id, key):

api/env.default

@@ -53,5 +53,4 @@ PRESIGNED_URL_EXPIRATION=3600
 ASSIST_JWT_EXPIRATION=144000
 ASSIST_JWT_SECRET=
 PYTHONUNBUFFERED=1
-REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379
-SCH_DELETE_DAYS=30
+REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379

api/routers/core.py

@@ -14,7 +14,7 @@ from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assig
     custom_metrics, saved_search, integrations_global
 from chalicelib.core.collaboration_msteams import MSTeams
 from chalicelib.core.collaboration_slack import Slack
-from chalicelib.utils import helper, captcha
+from chalicelib.utils import helper, captcha, s3
 from or_dependencies import OR_context
 from routers.base import get_routers
 
@@ -51,6 +51,9 @@ async def login(data: schemas.UserLoginSchema = Body(...)):
     response = JSONResponse(content=content)
     response.set_cookie(key="jwt", value=content['jwt'], domain=helper.get_domain(),
                         expires=config("JWT_EXPIRATION", cast=int))
+    print("S3>>>")
+    s3.schedule_for_deletion(bucket="mobs", key="8458559472534676/dom.mobs")
+    s3.schedule_for_deletion(bucket="mobs", key="8458559472534676/devtools.mob")
     return response
 
 

ee/api/chalicelib/core/sessions_devtool.py

@@ -36,4 +36,4 @@ def get_urls(session_id, project_id, context: schemas_ee.CurrentContext, check_e
 def delete_mobs(project_id, session_ids):
     for session_id in session_ids:
         for k in __get_devtools_keys(project_id=project_id, session_id=session_id):
-            s3.schedule_for_deletion(config("sessions_bucket"), k)
+            s3.schedule_for_deletion(bucket=config("sessions_bucket"), key=k)

ee/api/env.default

@@ -73,5 +73,4 @@ PRESIGNED_URL_EXPIRATION=3600
 ASSIST_JWT_EXPIRATION=144000
 ASSIST_JWT_SECRET=
 KAFKA_SERVERS=kafka.db.svc.cluster.local:9092
-KAFKA_USE_SSL=false
-SCH_DELETE_DAYS=30
+KAFKA_USE_SSL=false