From 69f2523a611b4d6dcffdd48336dae44b0c0af4ca Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Thu, 13 May 2021 14:20:08 +0200 Subject: [PATCH 1/9] fix (frontend): messages parce loop recovered --- .../player/MessageDistributor/MessageDistributor.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frontend/app/player/MessageDistributor/MessageDistributor.js b/frontend/app/player/MessageDistributor/MessageDistributor.js index ebdd423ce..fa3641b7a 100644 --- a/frontend/app/player/MessageDistributor/MessageDistributor.js +++ b/frontend/app/player/MessageDistributor/MessageDistributor.js @@ -164,6 +164,16 @@ export default class MessageDistributor extends StatedScreen { let mCount = 0; const msgs = []; + while (mGen.hasNext()) { + mCount++; + const next = mGen.next(); + if (next != null) { + this.#lastMessageTime = next[0].time; + this.#distributeMessage(next[0], next[1]); + msgs.push(next[0]); + } + } + // Hack for upet (TODO: fix ordering in one mutation (removes first)) const headChildrenIds = msgs.filter(m => m.parentID === 1).map(m => m.id); //const createNodeTypes = ["create_text_node", "create_element_node"]; From eacb318927d9417fa7812408fa81f8b8eeb17000 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Thu, 13 May 2021 14:21:03 +0200 Subject: [PATCH 2/9] fix (helm): minio edp var for storage --- backend/services/storage/main.go | 6 +++--- scripts/helm/app/storage.yaml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/services/storage/main.go b/backend/services/storage/main.go index 83337d8a6..2d008d868 100644 --- a/backend/services/storage/main.go +++ b/backend/services/storage/main.go @@ -23,7 +23,7 @@ func main() { storageWeb := storage.NewS3(env.String("AWS_REGION_WEB"), env.String("S3_BUCKET_WEB")) - storageIos := storage.NewS3(env.String("AWS_REGION_IOS"), env.String("S3_BUCKET_IOS")) + //storageIos := storage.NewS3(env.String("AWS_REGION_IOS"), env.String("S3_BUCKET_IOS")) FS_DIR := env.String("FS_DIR") + "/" var uploadKey func(string, int, *storage.S3) @@ -54,8 +54,8 @@ func main() { switch msg.(type) { case *messages.SessionEnd: uploadKey(strconv.FormatUint(sessionID, 10), 5, storageWeb) - case *messages.IOSSessionEnd: - uploadKey(strconv.FormatUint(sessionID, 10), 5, storageIos) + //case *messages.IOSSessionEnd: + // uploadKey(strconv.FormatUint(sessionID, 10), 5, storageIos) } }, ) diff --git a/scripts/helm/app/storage.yaml b/scripts/helm/app/storage.yaml index 40225f1c3..18890847a 100644 --- a/scripts/helm/app/storage.yaml +++ b/scripts/helm/app/storage.yaml @@ -31,6 +31,7 @@ pvc: storageSize: 5Gi env: + AWS_ENDPOINT: http://minio.db.svc.cluster.local:9000 AWS_ACCESS_KEY_ID: "minios3AccessKeyS3cr3t" AWS_SECRET_ACCESS_KEY: "m1n10s3CretK3yPassw0rd" AWS_REGION_WEB: eu-central-1 From 2e09d56eeaea92dae05adc4ce82d9fdac56067cd Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Thu, 13 May 2021 14:28:18 +0200 Subject: [PATCH 3/9] fix (helm): minio edp for the assets service --- scripts/helm/app/assets.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/helm/app/assets.yaml b/scripts/helm/app/assets.yaml index a3734d802..390fe4e07 100644 --- a/scripts/helm/app/assets.yaml +++ b/scripts/helm/app/assets.yaml @@ -24,6 +24,7 @@ resources: env: ASSETS_ORIGIN: /asayer-sessions-assets # TODO: full path (with the minio prefix) S3_BUCKET_ASSETS: asayer-sessions-assets + AWS_ENDPOINT: http://minio.db.svc.cluster.local:9000 AWS_ACCESS_KEY_ID: "minios3AccessKeyS3cr3t" AWS_SECRET_ACCESS_KEY: "m1n10s3CretK3yPassw0rd" AWS_REGION: us-east-1 From 28d9820416e8f65264d0e2b9e0551a9406645ecd Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Thu, 13 May 2021 18:01:36 +0530 Subject: [PATCH 4/9] fix(chalice): proper s3 domain name to sign url Signed-off-by: Rajesh Rajendran --- scripts/helm/roles/openreplay/templates/chalice.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/helm/roles/openreplay/templates/chalice.yaml b/scripts/helm/roles/openreplay/templates/chalice.yaml index 70a03dd89..90b6de579 100644 --- a/scripts/helm/roles/openreplay/templates/chalice.yaml +++ b/scripts/helm/roles/openreplay/templates/chalice.yaml @@ -12,3 +12,4 @@ env: S3_SECRET: "{{ minio_secret_key }}" sourcemaps_bucket_key: "{{ minio_access_key }}" sourcemaps_bucket_secret: "{{ minio_secret_key }}" + S3_HOST: "https://{{ domain_name }}" From 91cbd593b83b29a9c9c3a13f557f2f8ab4ea1387 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 14 May 2021 12:32:59 +0530 Subject: [PATCH 5/9] chore(deployment): making s3_host as mandatory variable Signed-off-by: Rajesh Rajendran --- scripts/helm/app/chalice.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/helm/app/chalice.yaml b/scripts/helm/app/chalice.yaml index e43cdbeec..30b02f747 100644 --- a/scripts/helm/app/chalice.yaml +++ b/scripts/helm/app/chalice.yaml @@ -59,7 +59,9 @@ env: jwt_secret: '' jwt_algorithm: 'HS512' jwt_exp_delta_seconds: '2592000' - S3_HOST: 's3://minio.db.svc.cluster.local:9000' + # Override with your https://domain_name + # eg: https://openreplay.mycompany.com + S3_HOST: '' S3_KEY: minios3AccessKeyS3cr3t S3_SECRET: m1n10s3CretK3yPassw0rd # Enable logging for python app From 409d071aac430c6a5ef995257ba8e47c9595b26d Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 14 May 2021 22:14:36 +0530 Subject: [PATCH 6/9] chore(build): script to build and push docker images. --- api/build.sh | 3 +++ backend/build.sh | 6 ++++++ scripts/helm/build_deploy.sh | 16 ++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 scripts/helm/build_deploy.sh diff --git a/api/build.sh b/api/build.sh index 390fbaca2..af75324b0 100644 --- a/api/build.sh +++ b/api/build.sh @@ -25,6 +25,9 @@ function build_api(){ envarg="default-ee" } docker build -f ./Dockerfile --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/chalice:${git_sha1} . + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/chalice:${git_sha1} + } } check_prereq diff --git a/backend/build.sh b/backend/build.sh index 6d7e9467e..c760c1b9b 100644 --- a/backend/build.sh +++ b/backend/build.sh @@ -27,11 +27,17 @@ function build_api(){ [[ $2 != "" ]] && { image="$2" docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image . + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1} + } return } for image in $(ls services); do docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image . + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1} + } echo "::set-output name=image::${DOCKER_REPO:-'local'}/$image:${git_sha1}" done } diff --git a/scripts/helm/build_deploy.sh b/scripts/helm/build_deploy.sh new file mode 100644 index 000000000..7e9781231 --- /dev/null +++ b/scripts/helm/build_deploy.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +# This script will build and push docker image to registry + +echo $DOCKER_REPO +[[ -z DOCKER_REPO ]] && { + echo Set DOCKER_REPO="your docker registry" + exit 1 +} || { + docker login $DOCKER_REPO + cd ../../api + PUSH_IMAGE=1 bash build.sh + cd ../backend + PUSH_IMAGE=1 bash build.sh +} From 71135413779417968a426619d963d9b14b104f6c Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 14 May 2021 22:29:36 +0530 Subject: [PATCH 7/9] chore(script): update usage for build deploy. Signed-off-by: Rajesh Rajendran --- scripts/helm/build_deploy.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/helm/build_deploy.sh b/scripts/helm/build_deploy.sh index 7e9781231..aec8ce3c9 100644 --- a/scripts/helm/build_deploy.sh +++ b/scripts/helm/build_deploy.sh @@ -3,6 +3,8 @@ set -e # This script will build and push docker image to registry +# Usage: IMAGE_TAG=latest DOCKER_REPO=rg.fr-par.scw.cloud/foss bash build_deploy.sh + echo $DOCKER_REPO [[ -z DOCKER_REPO ]] && { echo Set DOCKER_REPO="your docker registry" From 84dd5cd24d41d74e2710f01b0c450b53a423bf8c Mon Sep 17 00:00:00 2001 From: KRAIEM Taha Yassine Date: Sun, 16 May 2021 00:52:58 +0200 Subject: [PATCH 8/9] Changes: - fixed get slack channels - fixed metrics filter - set env-vars --- api/.chalice/config.json | 12 ++++++------ api/chalicelib/core/dashboard.py | 29 +++++++++++++++++----------- api/chalicelib/core/sessions_mobs.py | 10 +++++----- ee/api/.chalice/config.json | 16 +++++++-------- ee/api/chalicelib/utils/s3.py | 16 ++++++--------- scripts/helm/app/chalice.yaml | 10 +++++----- 6 files changed, 48 insertions(+), 45 deletions(-) diff --git a/api/.chalice/config.json b/api/.chalice/config.json index a5d4cad48..8f2874beb 100644 --- a/api/.chalice/config.json +++ b/api/.chalice/config.json @@ -6,7 +6,7 @@ }, "stages": { "default-foss": { - "api_gateway_stage": "default-foss", + "api_gateway_stage": "default-fos", "manage_iam_role": false, "iam_role_arn": "", "autogen_policy": true, @@ -28,14 +28,14 @@ "assign_link": "http://127.0.0.1:8000/async/email_assignment", "captcha_server": "", "captcha_key": "", - "sessions_bucket": "", - "sessions_region": "", + "sessions_bucket": "asayer-mobs", + "sessions_region": "us-east-1", "put_S3_TTL": "20", - "sourcemaps_bucket": "", + "sourcemaps_bucket": "asayer-sourcemaps", "sourcemaps_bucket_key": "", "sourcemaps_bucket_secret": "", - "sourcemaps_bucket_region": "", - "js_cache_bucket": "", + "sourcemaps_bucket_region": "us-east-1", + "js_cache_bucket": "asayer-sessions-assets", "async_Token": "", "EMAIL_HOST": "", "EMAIL_PORT": "587", diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index e739ceb04..a778dcdfc 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -82,7 +82,6 @@ def __get_meta_constraint(project_id, data): constraints = [] meta_keys = metadata.get(project_id=project_id) meta_keys = {m["key"]: m["index"] for m in meta_keys} - print(meta_keys) for i, f in enumerate(data.get("filters", [])): if f["key"] in meta_keys.keys(): @@ -93,20 +92,27 @@ def __get_meta_constraint(project_id, data): constraints.append(f"{key} = %({f['key']}_{i})s") else: filter_type = f["key"].upper() - if filter_type == sessions_metas.meta_type.USERBROWSER: + filter_type = [filter_type, "USER" + filter_type, filter_type[4:]] + if any(item in [sessions_metas.meta_type.USERBROWSER] \ + for item in filter_type): constraints.append(f"sessions.user_browser = %({f['key']}_{i})s") - elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: + elif any(item in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS] \ + for item in filter_type): constraints.append(f"sessions.user_os = %({f['key']}_{i})s") - elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: + elif any(item in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS] \ + for item in filter_type): constraints.append(f"sessions.user_device = %({f['key']}_{i})s") - elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: + elif any(item in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS] \ + for item in filter_type): constraints.append(f"sessions.user_country = %({f['key']}_{i})s") - elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif any(item in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS] \ + for item in filter_type): constraints.append(f"sessions.user_id = %({f['key']}_{i})s") - elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, - sessions_metas.meta_type.USERANONYMOUSID_IOS]: + elif any(item in [sessions_metas.meta_type.USERANONYMOUSID, sessions_metas.meta_type.USERANONYMOUSID_IOS] \ + for item in filter_type): constraints.append(f"sessions.user_anonymous_id = %({f['key']}_{i})s") - elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: + elif any(item in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS] \ + for item in filter_type): constraints.append(f"sessions.rev_id = %({f['key']}_{i})s") return constraints @@ -140,6 +146,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1) ORDER BY generated_timestamp;""" params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} + print(cur.mogrify(pg_query, params)) cur.execute(cur.mogrify(pg_query, params)) rows = cur.fetchall() results = { @@ -634,8 +641,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, WHERE {" AND ".join(pg_sub_query)} LIMIT 10;""" print(cur.mogrify(pg_query, {"project_id": project_id, - "value": helper.string_to_sql_like(text), - "platform_0": platform})) + "value": helper.string_to_sql_like(text), + "platform_0": platform})) cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text), "platform_0": platform})) diff --git a/api/chalicelib/core/sessions_mobs.py b/api/chalicelib/core/sessions_mobs.py index d3da91f52..75ac59307 100644 --- a/api/chalicelib/core/sessions_mobs.py +++ b/api/chalicelib/core/sessions_mobs.py @@ -4,11 +4,11 @@ import boto3 def get_web(sessionId): - return boto3.client('s3', - endpoint_url=environ["S3_HOST"], - aws_access_key_id=environ["S3_KEY"], - aws_secret_access_key=environ["S3_SECRET"], - region_name=environ["sessions_region"]).generate_presigned_url( + return boto3.client('s3', + endpoint_url=environ["S3_HOST"], + aws_access_key_id=environ["S3_KEY"], + aws_secret_access_key=environ["S3_SECRET"], + region_name=environ["sessions_region"]).generate_presigned_url( 'get_object', Params={ 'Bucket': environ["sessions_bucket"], diff --git a/ee/api/.chalice/config.json b/ee/api/.chalice/config.json index 4136e6ee0..605e5b7c1 100644 --- a/ee/api/.chalice/config.json +++ b/ee/api/.chalice/config.json @@ -31,14 +31,14 @@ "assign_link": "http://127.0.0.1:8000/async/email_assignment", "captcha_server": "", "captcha_key": "", - "sessions_bucket": "asayer-mobs-staging", - "sessions_region": "eu-central-1", + "sessions_bucket": "asayer-mobs", + "sessions_region": "us-east-1", "put_S3_TTL": "20", - "sourcemaps_bucket": "", + "sourcemaps_bucket": "asayer-sourcemaps", "sourcemaps_bucket_key": "", "sourcemaps_bucket_secret": "", - "sourcemaps_bucket_region": "", - "js_cache_bucket": "", + "sourcemaps_bucket_region": "us-east-1", + "js_cache_bucket": "asayer-sessions-assets", "async_Token": "", "EMAIL_HOST": "", "EMAIL_PORT": "587", @@ -51,9 +51,9 @@ "EMAIL_FROM": "OpenReplay", "SITE_URL": "", "announcement_url": "", - "jwt_secret": "", - "jwt_algorithm": "", - "jwt_exp_delta_seconds": "", + "jwt_secret": "SET A RANDOM STRING HERE", + "jwt_algorithm": "HS512", + "jwt_exp_delta_seconds": "2592000", "S3_HOST": "", "S3_KEY": "", "S3_SECRET": "" diff --git a/ee/api/chalicelib/utils/s3.py b/ee/api/chalicelib/utils/s3.py index 3ee2ba621..29a8d28bc 100644 --- a/ee/api/chalicelib/utils/s3.py +++ b/ee/api/chalicelib/utils/s3.py @@ -1,19 +1,15 @@ from botocore.exceptions import ClientError from chalicelib.utils.helper import environ -from chalicelib.utils import helper import boto3 -if helper.is_free_open_source_edition() or helper.is_enterprise_edition(): - from botocore.client import Config +from botocore.client import Config - client = boto3.client('s3', endpoint_url=environ["S3_HOST"], - aws_access_key_id=environ["S3_KEY"], - aws_secret_access_key=environ["S3_SECRET"], - config=Config(signature_version='s3v4'), - region_name='us-east-1') -else: - client = boto3.client('s3') +client = boto3.client('s3', endpoint_url=environ["S3_HOST"], + aws_access_key_id=environ["S3_KEY"], + aws_secret_access_key=environ["S3_SECRET"], + config=Config(signature_version='s3v4'), + region_name='us-east-1') def exists(bucket, key): diff --git a/scripts/helm/app/chalice.yaml b/scripts/helm/app/chalice.yaml index 30b02f747..eccd3b785 100644 --- a/scripts/helm/app/chalice.yaml +++ b/scripts/helm/app/chalice.yaml @@ -37,12 +37,12 @@ env: captcha_server: '' captcha_key: '' sessions_bucket: asayer-mobs - sessions_region: eu-central-1 + sessions_region: us-east-1 put_S3_TTL: '20' sourcemaps_bucket: asayer-sourcemaps sourcemaps_bucket_key: minios3AccessKeyS3cr3t - sourcemaps_bucket_secret: m1n10s3CretK3yPassw0rd - sourcemaps_bucket_region: eu-central-1 + sourcemaps_bucket_secret: m1n10s3CretK3yPassw0rd + sourcemaps_bucket_region: us-east-1 js_cache_bucket: asayer-sessions-assets async_Token: '' EMAIL_HOST: '' @@ -56,8 +56,8 @@ env: EMAIL_FROM: OpenReplay SITE_URL: '' announcement_url: '' - jwt_secret: '' - jwt_algorithm: 'HS512' + jwt_secret: SET A RANDOM STRING HERE + jwt_algorithm: HS512 jwt_exp_delta_seconds: '2592000' # Override with your https://domain_name # eg: https://openreplay.mycompany.com From bda85b50323bf8b6d967f6ad484b4e3bb32ddbd4 Mon Sep 17 00:00:00 2001 From: KRAIEM Taha Yassine Date: Sun, 16 May 2021 00:55:40 +0200 Subject: [PATCH 9/9] Changes: - added `asayer-sourcemaps` to nginx --- .../helm/nginx-ingress/nginx-ingress/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml b/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml index 616c19ccc..5cb1ef188 100644 --- a/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml +++ b/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml @@ -8,7 +8,7 @@ data: location /healthz { return 200 'OK'; } - location ~ ^/(asayer-mobs|asayer-sessions-assets|frontend|static)/ { + location ~ ^/(asayer-mobs|asayer-sessions-assets|frontend|static|asayer-sourcemaps)/ { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme;