From 42d97bef131ecfeceec7c2c6b704cc3d295ad038 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 18 Nov 2022 13:04:32 +0100 Subject: [PATCH 1/4] feat(chalice): vault support old format and new format feat(chalice): vault support devtools --- api/chalicelib/core/sessions_mobs.py | 25 +++++------ ee/api/chalicelib/core/sessions_favorite.py | 49 ++++++++++----------- ee/api/chalicelib/utils/s3_extra.py | 10 +++-- 3 files changed, 41 insertions(+), 43 deletions(-) diff --git a/api/chalicelib/core/sessions_mobs.py b/api/chalicelib/core/sessions_mobs.py index 3d966a47c..e2b22dc1b 100644 --- a/api/chalicelib/core/sessions_mobs.py +++ b/api/chalicelib/core/sessions_mobs.py @@ -15,6 +15,10 @@ def __get_mob_keys(project_id, session_id): ] +def __get_mob_keys_deprecated(session_id): + return [str(session_id), str(session_id) + "e"] + + def get_urls(project_id, session_id): results = [] for k in __get_mob_keys(project_id=project_id, session_id=session_id): @@ -27,23 +31,14 @@ def get_urls(project_id, session_id): def get_urls_depercated(session_id): - return [ - client.generate_presigned_url( + results = [] + for k in __get_mob_keys_deprecated(session_id=session_id): + results.append(client.generate_presigned_url( 'get_object', - Params={ - 'Bucket': config("sessions_bucket"), - 'Key': str(session_id) - }, + Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=100000 - ), - client.generate_presigned_url( - 'get_object', - Params={ - 'Bucket': config("sessions_bucket"), - 'Key': str(session_id) + "e" - }, - ExpiresIn=100000 - )] + )) + return results def get_ios(session_id): diff --git a/ee/api/chalicelib/core/sessions_favorite.py b/ee/api/chalicelib/core/sessions_favorite.py index 3d6496424..7af995bad 100644 --- a/ee/api/chalicelib/core/sessions_favorite.py +++ b/ee/api/chalicelib/core/sessions_favorite.py @@ -1,7 +1,7 @@ from decouple import config import schemas_ee -from chalicelib.core import sessions, sessions_favorite_exp +from chalicelib.core import sessions, sessions_favorite_exp, sessions_mobs, sessions_devtool from chalicelib.utils import pg_client, s3_extra @@ -34,32 +34,31 @@ def remove_favorite_session(context: schemas_ee.CurrentContext, project_id, sess def favorite_session(context: schemas_ee.CurrentContext, project_id, session_id): + keys = sessions_mobs.__get_mob_keys(project_id=project_id, session_id=session_id) + keys += sessions_mobs.__get_mob_keys_deprecated(session_id=session_id) # To support old sessions + keys += sessions_devtool.__get_devtools_keys(project_id=project_id, session_id=session_id) + if favorite_session_exists(user_id=context.user_id, session_id=session_id): - key = str(session_id) - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_D_VALUE', default='default')) - except Exception as e: - print(f"!!!Error while tagging: {key} to default") - print(str(e)) - key = str(session_id) + "e" - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_D_VALUE', default='default')) - except Exception as e: - print(f"!!!Error while tagging: {key} to default") - print(str(e)) + tag = config('RETENTION_D_VALUE', default='default') + + for k in keys: + try: + s3_extra.tag_session(file_key=k, tag_value=tag) + except Exception as e: + print(f"!!!Error while tagging: {k} to {tag} for removal") + print(str(e)) + return remove_favorite_session(context=context, project_id=project_id, session_id=session_id) - key = str(session_id) - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_L_VALUE', default='vault')) - except Exception as e: - print(f"!!!Error while tagging: {key} to vault") - print(str(e)) - key = str(session_id) + "e" - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_L_VALUE', default='vault')) - except Exception as e: - print(f"!!!Error while tagging: {key} to vault") - print(str(e)) + + tag = config('RETENTION_L_VALUE', default='vault') + + for k in keys: + try: + s3_extra.tag_session(file_key=k, tag_value=tag) + except Exception as e: + print(f"!!!Error while tagging: {k} to {tag} for vault") + print(str(e)) + return add_favorite_session(context=context, project_id=project_id, session_id=session_id) diff --git a/ee/api/chalicelib/utils/s3_extra.py b/ee/api/chalicelib/utils/s3_extra.py index f2a538dcc..4ebf60fec 100644 --- a/ee/api/chalicelib/utils/s3_extra.py +++ b/ee/api/chalicelib/utils/s3_extra.py @@ -3,10 +3,14 @@ from decouple import config from chalicelib.utils.s3 import client -def tag_file(session_id, tag_key='retention', tag_value='vault'): +def tag_session(file_key, tag_key='retention', tag_value='vault'): + return tag_file(file_key=file_key, bucket=config("sessions_bucket"), tag_key=tag_key, tag_value=tag_value) + + +def tag_file(file_key, bucket, tag_key, tag_value): return client.put_object_tagging( - Bucket=config("sessions_bucket"), - Key=session_id, + Bucket=bucket, + Key=file_key, Tagging={ 'TagSet': [ { From b7e6a237e4f697875b96a0b9b9e9f6562d3de4b5 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 18 Nov 2022 13:14:44 +0100 Subject: [PATCH 2/4] feat(chalice): refactored s3 client --- api/chalicelib/core/sessions_devtool.py | 4 ++-- api/chalicelib/core/sessions_mobs.py | 7 +++---- ee/api/chalicelib/core/sessions_devtool.py | 4 ++-- ee/api/chalicelib/utils/s3_extra.py | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/api/chalicelib/core/sessions_devtool.py b/api/chalicelib/core/sessions_devtool.py index eef7b8e6b..2afc3c366 100644 --- a/api/chalicelib/core/sessions_devtool.py +++ b/api/chalicelib/core/sessions_devtool.py @@ -1,6 +1,6 @@ from decouple import config -from chalicelib.utils.s3 import client +from chalicelib.utils import s3 def __get_devtools_keys(project_id, session_id): @@ -16,7 +16,7 @@ def __get_devtools_keys(project_id, session_id): def get_urls(session_id, project_id): results = [] for k in __get_devtools_keys(project_id=project_id, session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) diff --git a/api/chalicelib/core/sessions_mobs.py b/api/chalicelib/core/sessions_mobs.py index e2b22dc1b..9a9237be8 100644 --- a/api/chalicelib/core/sessions_mobs.py +++ b/api/chalicelib/core/sessions_mobs.py @@ -1,7 +1,6 @@ from decouple import config from chalicelib.utils import s3 -from chalicelib.utils.s3 import client def __get_mob_keys(project_id, session_id): @@ -22,7 +21,7 @@ def __get_mob_keys_deprecated(session_id): def get_urls(project_id, session_id): results = [] for k in __get_mob_keys(project_id=project_id, session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) @@ -33,7 +32,7 @@ def get_urls(project_id, session_id): def get_urls_depercated(session_id): results = [] for k in __get_mob_keys_deprecated(session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=100000 @@ -42,7 +41,7 @@ def get_urls_depercated(session_id): def get_ios(session_id): - return client.generate_presigned_url( + return s3.client.generate_presigned_url( 'get_object', Params={ 'Bucket': config("ios_bucket"), diff --git a/ee/api/chalicelib/core/sessions_devtool.py b/ee/api/chalicelib/core/sessions_devtool.py index ed6ecf694..9435c2e24 100644 --- a/ee/api/chalicelib/core/sessions_devtool.py +++ b/ee/api/chalicelib/core/sessions_devtool.py @@ -3,7 +3,7 @@ from fastapi.security import SecurityScopes import schemas_ee from chalicelib.core import permissions -from chalicelib.utils.s3 import client +from chalicelib.utils import s3 SCOPES = SecurityScopes([schemas_ee.Permissions.dev_tools]) @@ -23,7 +23,7 @@ def get_urls(session_id, project_id, context: schemas_ee.CurrentContext): return [] results = [] for k in __get_devtools_keys(project_id=project_id, session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) diff --git a/ee/api/chalicelib/utils/s3_extra.py b/ee/api/chalicelib/utils/s3_extra.py index 4ebf60fec..0e594c890 100644 --- a/ee/api/chalicelib/utils/s3_extra.py +++ b/ee/api/chalicelib/utils/s3_extra.py @@ -1,6 +1,6 @@ from decouple import config -from chalicelib.utils.s3 import client +from chalicelib.utils import s3 def tag_session(file_key, tag_key='retention', tag_value='vault'): @@ -8,7 +8,7 @@ def tag_session(file_key, tag_key='retention', tag_value='vault'): def tag_file(file_key, bucket, tag_key, tag_value): - return client.put_object_tagging( + return s3.client.put_object_tagging( Bucket=bucket, Key=file_key, Tagging={ From a2b2cd6f8694a44ff224accad2658757fcc24105 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 18 Nov 2022 15:49:14 +0100 Subject: [PATCH 3/4] feat(alerts): format values for notification --- api/chalicelib/core/alerts_processor.py | 15 +++++++++++++-- ee/api/chalicelib/core/alerts_processor.py | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index 2ed9105b2..76ae5c615 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -199,7 +199,8 @@ def process(): logging.info(f"Valid alert, notifying users, alertId:{alert['alertId']} name: {alert['name']}") notifications.append(generate_notification(alert, result)) except Exception as e: - logging.error(f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}") + logging.error( + f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}") logging.error(query) logging.error(e) cur = cur.recreate(rollback=True) @@ -212,12 +213,22 @@ def process(): alerts.process_notifications(notifications) +def __format_value(x): + if x % 1 == 0: + x = int(x) + else: + x = round(x, 2) + return f"{x:,}" + + def generate_notification(alert, result): + left = __format_value(result['value']) + right = __format_value(alert['query']['right']) return { "alertId": alert["alertId"], "tenantId": alert["tenantId"], "title": alert["name"], - "description": f"has been triggered, {alert['query']['left']} = {round(result['value'], 2)} ({alert['query']['operator']} {alert['query']['right']}).", + "description": f"has been triggered, {alert['query']['left']} = {left} ({alert['query']['operator']} {right}).", "buttonText": "Check metrics for more details", "buttonUrl": f"/{alert['projectId']}/metrics", "imageUrl": None, diff --git a/ee/api/chalicelib/core/alerts_processor.py b/ee/api/chalicelib/core/alerts_processor.py index 087f23a05..326d17ffc 100644 --- a/ee/api/chalicelib/core/alerts_processor.py +++ b/ee/api/chalicelib/core/alerts_processor.py @@ -204,7 +204,8 @@ def process(): logging.info(f"Valid alert, notifying users, alertId:{alert['alertId']} name: {alert['name']}") notifications.append(generate_notification(alert, result)) except Exception as e: - logging.error(f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}") + logging.error( + f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}") logging.error(query) logging.error(e) cur = cur.recreate(rollback=True) @@ -217,12 +218,22 @@ def process(): alerts.process_notifications(notifications) +def __format_value(x): + if x % 1 == 0: + x = int(x) + else: + x = round(x, 2) + return f"{x:,}" + + def generate_notification(alert, result): + left = __format_value(result['value']) + right = __format_value(alert['query']['right']) return { "alertId": alert["alertId"], "tenantId": alert["tenantId"], "title": alert["name"], - "description": f"has been triggered, {alert['query']['left']} = {round(result['value'], 2)} ({alert['query']['operator']} {alert['query']['right']}).", + "description": f"has been triggered, {alert['query']['left']} = {left} ({alert['query']['operator']} {right}).", "buttonText": "Check metrics for more details", "buttonUrl": f"/{alert['projectId']}/metrics", "imageUrl": None, From 287ba8163c018133140eeb2236a1e882894e2456 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 18 Nov 2022 16:23:25 +0100 Subject: [PATCH 4/4] feat(chalice): ignore more html-sourcemaps --- api/chalicelib/core/sourcemaps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/core/sourcemaps.py b/api/chalicelib/core/sourcemaps.py index 597b260d3..89df77926 100644 --- a/api/chalicelib/core/sourcemaps.py +++ b/api/chalicelib/core/sourcemaps.py @@ -77,7 +77,7 @@ def format_payload(p, truncate_to_first=False): def url_exists(url): try: r = requests.head(url, allow_redirects=False) - return r.status_code == 200 and r.headers.get("Content-Type") != "text/html" + return r.status_code == 200 and "text/html" not in r.headers.get("Content-Type", "") except Exception as e: print(f"!! Issue checking if URL exists: {url}") print(e)