diff --git a/api/chalicelib/core/jobs.py b/api/chalicelib/core/jobs.py index bc5ce81ab..8ca556fdf 100644 --- a/api/chalicelib/core/jobs.py +++ b/api/chalicelib/core/jobs.py @@ -1,6 +1,6 @@ from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC -from chalicelib.core import sessions, sessions_mobs, sessions_devtool +from chalicelib.core import sessions_mobs, sessions_devtool class Actions: @@ -103,6 +103,32 @@ def format_datetime(r): r["start_at"] = TimeUTC.datetime_to_timestamp(r["start_at"]) +def __get_session_ids_by_user_ids(project_id, user_ids): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """SELECT session_id + FROM public.sessions + WHERE project_id = %(project_id)s + AND user_id IN %(userId)s;""", + {"project_id": project_id, "userId": tuple(user_ids)}) + cur.execute(query=query) + ids = cur.fetchall() + return [s["session_id"] for s in ids] + + +def __delete_sessions_by_session_ids(session_ids): + with pg_client.PostgresClient(unlimited_query=True) as cur: + query = cur.mogrify( + """DELETE FROM public.sessions + WHERE session_id IN %(session_ids)s + LIMIT 1000;""", + {"session_ids": tuple(session_ids)} + ) + cur.execute(query=query) + + return True + + def get_scheduled_jobs(): with pg_client.PostgresClient() as cur: query = cur.mogrify( @@ -122,11 +148,11 @@ def execute_jobs(): print(f"Executing jobId:{job['jobId']}") try: if job["action"] == Actions.DELETE_USER_DATA: - session_ids = sessions.get_session_ids_by_user_ids(project_id=job["projectId"], - user_ids=[job["referenceId"]]) + session_ids = __get_session_ids_by_user_ids(project_id=job["projectId"], + user_ids=[job["referenceId"]]) if len(session_ids) > 0: print(f"Deleting {len(session_ids)} sessions") - sessions.delete_sessions_by_session_ids(session_ids) + __delete_sessions_by_session_ids(session_ids) sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) sessions_devtool.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) else: diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 7de8253da..e7283755e 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1065,31 +1065,6 @@ def get_session_user(project_id, user_id): return helper.dict_to_camel_case(data) -def get_session_ids_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient() as cur: - query = cur.mogrify( - """SELECT session_id - FROM public.sessions - WHERE project_id = %(project_id)s - AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)}) - cur.execute(query=query) - ids = cur.fetchall() - return [s["session_id"] for s in ids] - - -def delete_sessions_by_session_ids(session_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """DELETE FROM public.sessions - WHERE session_id IN %(session_ids)s;""", - {"session_ids": tuple(session_ids)} - ) - cur.execute(query=query) - - return True - - def count_all(): with pg_client.PostgresClient(unlimited_query=True) as cur: cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions") diff --git a/api/chalicelib/utils/s3.py b/api/chalicelib/utils/s3.py index cdd22aa4e..6eeacd261 100644 --- a/api/chalicelib/utils/s3.py +++ b/api/chalicelib/utils/s3.py @@ -115,7 +115,7 @@ def schedule_for_deletion(bucket, key): s3 = __get_s3_resource() s3_object = s3.Object(bucket, key) s3_object.copy_from(CopySource={'Bucket': bucket, 'Key': key}, - Expires=datetime.utcnow() + timedelta(days=config("SCH_DELETE_DAYS", cast=int, default=7)), + Expires=datetime.utcnow() + timedelta(days=config("SCH_DELETE_DAYS", cast=int, default=30)), MetadataDirective='REPLACE') return True diff --git a/api/env.default b/api/env.default index e0560619f..7d5266470 100644 --- a/api/env.default +++ b/api/env.default @@ -54,4 +54,4 @@ ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= PYTHONUNBUFFERED=1 REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 -SCH_DELETE_DAYS=7 \ No newline at end of file +SCH_DELETE_DAYS=30 \ No newline at end of file diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index 1c4d20883..b7539c3c2 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -1396,31 +1396,6 @@ def get_session_user(project_id, user_id): return helper.dict_to_camel_case(data) -def get_session_ids_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient() as cur: - query = cur.mogrify( - """SELECT session_id - FROM public.sessions - WHERE project_id = %(project_id)s - AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)}) - cur.execute(query=query) - ids = cur.fetchall() - return [s["session_id"] for s in ids] - - -def delete_sessions_by_session_ids(session_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """DELETE FROM public.sessions - WHERE session_id IN %(session_ids)s;""", - {"session_ids": tuple(session_ids)} - ) - cur.execute(query=query) - - return True - - def count_all(): with ch_client.ClickHouseClient() as cur: row = cur.execute(query=f"SELECT COUNT(session_id) AS count FROM {exp_ch_helper.get_main_sessions_table()}") diff --git a/ee/api/env.default b/ee/api/env.default index 603c291b0..a35f6f451 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -74,4 +74,4 @@ ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= KAFKA_SERVERS=kafka.db.svc.cluster.local:9092 KAFKA_USE_SSL=false -SCH_DELETE_DAYS=7 \ No newline at end of file +SCH_DELETE_DAYS=30 \ No newline at end of file