From 9542dcad93cac8fe33a16c6b57b61d7d9eb23381 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 17 Nov 2022 12:43:00 +0100 Subject: [PATCH 1/4] feat(chalice): fixed sourcemaps process non js --- api/chalicelib/core/sourcemaps.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/chalicelib/core/sourcemaps.py b/api/chalicelib/core/sourcemaps.py index 921649d97..597b260d3 100644 --- a/api/chalicelib/core/sourcemaps.py +++ b/api/chalicelib/core/sourcemaps.py @@ -100,7 +100,6 @@ def get_traces_group(project_id, payload): and not (file_url[:params_idx] if params_idx > -1 else file_url).endswith(".js"): print(f"{u['absPath']} sourcemap is not a JS file") payloads[key] = None - continue if key not in payloads: file_exists_in_bucket = len(file_url) > 0 and s3.exists(config('sourcemaps_bucket'), key) From 78a1f27984d53d3906e00be97b09d4d2657b9b17 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 17 Nov 2022 17:17:02 +0100 Subject: [PATCH 2/4] feat(chalice): changed EFS patterns --- api/env.default | 2 +- ee/api/env.default | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/env.default b/api/env.default index 55aefdc35..3ee65e89c 100644 --- a/api/env.default +++ b/api/env.default @@ -46,7 +46,7 @@ EFS_SESSION_MOB_PATTERN=%(sessionId)s EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)sdevtools SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe -DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mobs +DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= diff --git a/ee/api/env.default b/ee/api/env.default index 7fd61f4a0..f5574a8a1 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -65,7 +65,7 @@ EFS_SESSION_MOB_PATTERN=%(sessionId)s EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)sdevtools SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe -DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mobs +DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= \ No newline at end of file From 42d97bef131ecfeceec7c2c6b704cc3d295ad038 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 18 Nov 2022 13:04:32 +0100 Subject: [PATCH 3/4] feat(chalice): vault support old format and new format feat(chalice): vault support devtools --- api/chalicelib/core/sessions_mobs.py | 25 +++++------ ee/api/chalicelib/core/sessions_favorite.py | 49 ++++++++++----------- ee/api/chalicelib/utils/s3_extra.py | 10 +++-- 3 files changed, 41 insertions(+), 43 deletions(-) diff --git a/api/chalicelib/core/sessions_mobs.py b/api/chalicelib/core/sessions_mobs.py index 3d966a47c..e2b22dc1b 100644 --- a/api/chalicelib/core/sessions_mobs.py +++ b/api/chalicelib/core/sessions_mobs.py @@ -15,6 +15,10 @@ def __get_mob_keys(project_id, session_id): ] +def __get_mob_keys_deprecated(session_id): + return [str(session_id), str(session_id) + "e"] + + def get_urls(project_id, session_id): results = [] for k in __get_mob_keys(project_id=project_id, session_id=session_id): @@ -27,23 +31,14 @@ def get_urls(project_id, session_id): def get_urls_depercated(session_id): - return [ - client.generate_presigned_url( + results = [] + for k in __get_mob_keys_deprecated(session_id=session_id): + results.append(client.generate_presigned_url( 'get_object', - Params={ - 'Bucket': config("sessions_bucket"), - 'Key': str(session_id) - }, + Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=100000 - ), - client.generate_presigned_url( - 'get_object', - Params={ - 'Bucket': config("sessions_bucket"), - 'Key': str(session_id) + "e" - }, - ExpiresIn=100000 - )] + )) + return results def get_ios(session_id): diff --git a/ee/api/chalicelib/core/sessions_favorite.py b/ee/api/chalicelib/core/sessions_favorite.py index 3d6496424..7af995bad 100644 --- a/ee/api/chalicelib/core/sessions_favorite.py +++ b/ee/api/chalicelib/core/sessions_favorite.py @@ -1,7 +1,7 @@ from decouple import config import schemas_ee -from chalicelib.core import sessions, sessions_favorite_exp +from chalicelib.core import sessions, sessions_favorite_exp, sessions_mobs, sessions_devtool from chalicelib.utils import pg_client, s3_extra @@ -34,32 +34,31 @@ def remove_favorite_session(context: schemas_ee.CurrentContext, project_id, sess def favorite_session(context: schemas_ee.CurrentContext, project_id, session_id): + keys = sessions_mobs.__get_mob_keys(project_id=project_id, session_id=session_id) + keys += sessions_mobs.__get_mob_keys_deprecated(session_id=session_id) # To support old sessions + keys += sessions_devtool.__get_devtools_keys(project_id=project_id, session_id=session_id) + if favorite_session_exists(user_id=context.user_id, session_id=session_id): - key = str(session_id) - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_D_VALUE', default='default')) - except Exception as e: - print(f"!!!Error while tagging: {key} to default") - print(str(e)) - key = str(session_id) + "e" - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_D_VALUE', default='default')) - except Exception as e: - print(f"!!!Error while tagging: {key} to default") - print(str(e)) + tag = config('RETENTION_D_VALUE', default='default') + + for k in keys: + try: + s3_extra.tag_session(file_key=k, tag_value=tag) + except Exception as e: + print(f"!!!Error while tagging: {k} to {tag} for removal") + print(str(e)) + return remove_favorite_session(context=context, project_id=project_id, session_id=session_id) - key = str(session_id) - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_L_VALUE', default='vault')) - except Exception as e: - print(f"!!!Error while tagging: {key} to vault") - print(str(e)) - key = str(session_id) + "e" - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_L_VALUE', default='vault')) - except Exception as e: - print(f"!!!Error while tagging: {key} to vault") - print(str(e)) + + tag = config('RETENTION_L_VALUE', default='vault') + + for k in keys: + try: + s3_extra.tag_session(file_key=k, tag_value=tag) + except Exception as e: + print(f"!!!Error while tagging: {k} to {tag} for vault") + print(str(e)) + return add_favorite_session(context=context, project_id=project_id, session_id=session_id) diff --git a/ee/api/chalicelib/utils/s3_extra.py b/ee/api/chalicelib/utils/s3_extra.py index f2a538dcc..4ebf60fec 100644 --- a/ee/api/chalicelib/utils/s3_extra.py +++ b/ee/api/chalicelib/utils/s3_extra.py @@ -3,10 +3,14 @@ from decouple import config from chalicelib.utils.s3 import client -def tag_file(session_id, tag_key='retention', tag_value='vault'): +def tag_session(file_key, tag_key='retention', tag_value='vault'): + return tag_file(file_key=file_key, bucket=config("sessions_bucket"), tag_key=tag_key, tag_value=tag_value) + + +def tag_file(file_key, bucket, tag_key, tag_value): return client.put_object_tagging( - Bucket=config("sessions_bucket"), - Key=session_id, + Bucket=bucket, + Key=file_key, Tagging={ 'TagSet': [ { From b7e6a237e4f697875b96a0b9b9e9f6562d3de4b5 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 18 Nov 2022 13:14:44 +0100 Subject: [PATCH 4/4] feat(chalice): refactored s3 client --- api/chalicelib/core/sessions_devtool.py | 4 ++-- api/chalicelib/core/sessions_mobs.py | 7 +++---- ee/api/chalicelib/core/sessions_devtool.py | 4 ++-- ee/api/chalicelib/utils/s3_extra.py | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/api/chalicelib/core/sessions_devtool.py b/api/chalicelib/core/sessions_devtool.py index eef7b8e6b..2afc3c366 100644 --- a/api/chalicelib/core/sessions_devtool.py +++ b/api/chalicelib/core/sessions_devtool.py @@ -1,6 +1,6 @@ from decouple import config -from chalicelib.utils.s3 import client +from chalicelib.utils import s3 def __get_devtools_keys(project_id, session_id): @@ -16,7 +16,7 @@ def __get_devtools_keys(project_id, session_id): def get_urls(session_id, project_id): results = [] for k in __get_devtools_keys(project_id=project_id, session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) diff --git a/api/chalicelib/core/sessions_mobs.py b/api/chalicelib/core/sessions_mobs.py index e2b22dc1b..9a9237be8 100644 --- a/api/chalicelib/core/sessions_mobs.py +++ b/api/chalicelib/core/sessions_mobs.py @@ -1,7 +1,6 @@ from decouple import config from chalicelib.utils import s3 -from chalicelib.utils.s3 import client def __get_mob_keys(project_id, session_id): @@ -22,7 +21,7 @@ def __get_mob_keys_deprecated(session_id): def get_urls(project_id, session_id): results = [] for k in __get_mob_keys(project_id=project_id, session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) @@ -33,7 +32,7 @@ def get_urls(project_id, session_id): def get_urls_depercated(session_id): results = [] for k in __get_mob_keys_deprecated(session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=100000 @@ -42,7 +41,7 @@ def get_urls_depercated(session_id): def get_ios(session_id): - return client.generate_presigned_url( + return s3.client.generate_presigned_url( 'get_object', Params={ 'Bucket': config("ios_bucket"), diff --git a/ee/api/chalicelib/core/sessions_devtool.py b/ee/api/chalicelib/core/sessions_devtool.py index ed6ecf694..9435c2e24 100644 --- a/ee/api/chalicelib/core/sessions_devtool.py +++ b/ee/api/chalicelib/core/sessions_devtool.py @@ -3,7 +3,7 @@ from fastapi.security import SecurityScopes import schemas_ee from chalicelib.core import permissions -from chalicelib.utils.s3 import client +from chalicelib.utils import s3 SCOPES = SecurityScopes([schemas_ee.Permissions.dev_tools]) @@ -23,7 +23,7 @@ def get_urls(session_id, project_id, context: schemas_ee.CurrentContext): return [] results = [] for k in __get_devtools_keys(project_id=project_id, session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) diff --git a/ee/api/chalicelib/utils/s3_extra.py b/ee/api/chalicelib/utils/s3_extra.py index 4ebf60fec..0e594c890 100644 --- a/ee/api/chalicelib/utils/s3_extra.py +++ b/ee/api/chalicelib/utils/s3_extra.py @@ -1,6 +1,6 @@ from decouple import config -from chalicelib.utils.s3 import client +from chalicelib.utils import s3 def tag_session(file_key, tag_key='retention', tag_value='vault'): @@ -8,7 +8,7 @@ def tag_session(file_key, tag_key='retention', tag_value='vault'): def tag_file(file_key, bucket, tag_key, tag_value): - return client.put_object_tagging( + return s3.client.put_object_tagging( Bucket=bucket, Key=file_key, Tagging={