From 1834806f4a8e8f24911e8677fe4c02345b121b73 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 19 Jan 2022 22:47:06 +0530 Subject: [PATCH 01/33] chore(helm): using minio keys to initialize minio rather than s3. Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/openreplay/templates/job.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/openreplay/templates/job.yaml b/scripts/helmcharts/openreplay/templates/job.yaml index 74e2cadcc..4e0cdebf4 100644 --- a/scripts/helmcharts/openreplay/templates/job.yaml +++ b/scripts/helmcharts/openreplay/templates/job.yaml @@ -81,9 +81,9 @@ spec: - name: CHART_APP_VERSION value: "{{ .Chart.AppVersion }}" - name: MINIO_ACCESS_KEY - value: "{{ .Values.global.s3.accessKey }}" + value: "{{ .Values.minio.global.minio.accessKey }}" - name: MINIO_SECRET_KEY - value: "{{ .Values.global.s3.secretKey }}" + value: "{{ .Values.minio.global.minio.secretKey }}" command: - /bin/bash - /opt/migrations/dbops.sh From 1fb6e470aab6bd8816595636f96166d8696cfd72 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 19 Jan 2022 22:51:17 +0530 Subject: [PATCH 02/33] docs(helm): update vars.yaml examples Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/vars.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 50bf9edd2..e8ef20bf7 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -87,14 +87,14 @@ chalice: # limits: # cpu: 1 -# nginx-ingress: +## Changes to nginx # -### If you want to redirect nginx http to https +# nginx-ingress: # customServerConfigs: | +# # Redirecting http to https # return 301 https://$host$request_uri; # ### Change the ssl certificates -# ### Public certificate ( content from site.crt, mind the indentation ) # ssl_certificate: |- # -----BEGIN CERTIFICATE----- @@ -109,10 +109,5 @@ chalice: # TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T # e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP # 3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 -# WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa -# 8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb -# esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn -# PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6 -# PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5 # 7qINKoGovqXB1oAdopIl1z64e7MWVE4= # -----END PRIVATE KEY----- From e01c3ccf04f8a161784872b8e9a5d13dbf623b81 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 11:00:24 +0100 Subject: [PATCH 03/33] Update LICENSE.md --- ee/LICENSE.md | 36 ++---------------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/ee/LICENSE.md b/ee/LICENSE.md index d99b63d76..5f6043f8f 100644 --- a/ee/LICENSE.md +++ b/ee/LICENSE.md @@ -1,36 +1,4 @@ The OpenReplay Enterprise license (the “Enterprise License”) -Copyright (c) 2021 Asayer SAS. +Copyright (c) 2022 Asayer SAS. -With regard to the OpenReplay Software: - -This software and associated documentation files (the "Software") may only be -used in production, if you (and any entity that you represent) have agreed to, -and are in compliance with, the OpenReplay Subscription Terms of Service, available -at https://openreplay.com/terms.html (the “Enterprise Edition”), or other -agreement governing the use of the Software, as agreed by you and OpenReplay, -and otherwise have a valid OpenReplay Enterprise license for the -correct usage. Subject to the foregoing sentence, you are free to -modify this Software and publish patches to the Software. You agree that OpenReplay -and/or its licensors (as applicable) retain all right, title and interest in and -to all such modifications and/or patches, and all such modifications and/or -patches may only be used, copied, modified, displayed, distributed, or otherwise -exploited with a valid OpenReplay Enterprise license for the correct -number of user seats and profiles. Notwithstanding the foregoing, you may copy and modify -the Software for development and testing purposes, without requiring a -subscription. You agree that OpenReplay and/or its licensors (as applicable) retain -all right, title and interest in and to all such modifications. You are not -granted any other rights beyond what is expressly stated herein. Subject to the -foregoing, it is forbidden to copy, merge, publish, distribute, sublicense, -and/or sell the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -For all third party components incorporated into the OpenReplay Software, those -components are licensed under the original license provided by the owner of the -applicable component. +To license the Enterprise Edition of OpenReplay, and take advantage of its additional features, functionality and support, you must agree to the terms of the OpenReplay Enterprise License Agreement. Please contact OpenReplay at [sales@openreplay.com](mailto:sales@openreplay.com). From b422eb52107ca60b0e5435b19e333f2f1c06e07e Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 12:04:54 +0100 Subject: [PATCH 04/33] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 940c97860..406fe9608 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 Asayer SAS. +Copyright (c) 2022 Asayer SAS. Portions of this software are licensed as follows: From 4951b3105bb0847ab035880d37bafdb997eb8e73 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 14:25:11 +0100 Subject: [PATCH 05/33] Update third-party.md --- third-party.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/third-party.md b/third-party.md index a26f646c3..98aefe5b0 100644 --- a/third-party.md +++ b/third-party.md @@ -1,4 +1,4 @@ -## Licenses (as of January 16, 2022) +## Licenses (as of January 21, 2022) Below is the list of dependencies used in OpenReplay software. Licenses may change between versions, so please keep this up to date with every new library you use. @@ -84,10 +84,9 @@ Below is the list of dependencies used in OpenReplay software. Licenses may chan | redux-immutable | BSD3 | JavaScript | | redux-thunk | MIT | JavaScript | | semantic-ui-react | MIT | JavaScript | -| socket.io-client | MIT | JavaScript | +| socketio | MIT | JavaScript | | source-map | BSD3 | JavaScript | | aws-sdk | Apache2 | JavaScript | | serverless | MIT | JavaScript | | lib/pq | MIT | Go | | peerjs | MIT | JavaScript | -| antonmedv/finder | MIT | JavaScript | From 293619e61e38eb50f6ad2c0890f7445c7a65fb1e Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 14:25:18 +0100 Subject: [PATCH 06/33] feat(api): EE-SSO extra endpoint for response processing using tenantKey --- ee/api/routers/saml.py | 85 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/ee/api/routers/saml.py b/ee/api/routers/saml.py index 50723a1db..73b0e7393 100644 --- a/ee/api/routers/saml.py +++ b/ee/api/routers/saml.py @@ -43,6 +43,8 @@ async def process_sso_assertion(request: Request): user_data = auth.get_attributes() elif auth.get_settings().is_debug_active(): error_reason = auth.get_last_error_reason() + print("SAML2 error:") + print(error_reason) return {"errors": [error_reason]} email = auth.get_nameid() @@ -108,6 +110,88 @@ async def process_sso_assertion(request: Request): headers={'Location': SAML2_helper.get_landing_URL(jwt)}) +@public_app.post('/sso/saml2/acs/{tenantKey}', tags=["saml2"]) +async def process_sso_assertion_tk(tenantKey: str, request: Request): + req = await prepare_request(request=request) + session = req["cookie"]["session"] + auth = init_saml_auth(req) + + request_id = None + if 'AuthNRequestID' in session: + request_id = session['AuthNRequestID'] + + auth.process_response(request_id=request_id) + errors = auth.get_errors() + user_data = {} + if len(errors) == 0: + if 'AuthNRequestID' in session: + del session['AuthNRequestID'] + user_data = auth.get_attributes() + elif auth.get_settings().is_debug_active(): + error_reason = auth.get_last_error_reason() + print("SAML2 error:") + print(error_reason) + return {"errors": [error_reason]} + + email = auth.get_nameid() + print("received nameId:") + print(email) + existing = users.get_by_email_only(auth.get_nameid()) + + internal_id = next(iter(user_data.get("internalId", [])), None) + + t = tenants.get_by_tenant_key(tenantKey) + if t is None: + print("invalid tenantKey, please copy the correct value from Preferences > Account") + return {"errors": ["invalid tenantKey, please copy the correct value from Preferences > Account"]} + print(user_data) + role_name = user_data.get("role", []) + if len(role_name) == 0: + print("No role specified, setting role to member") + role_name = ["member"] + role_name = role_name[0] + role = roles.get_role_by_name(tenant_id=t['tenantId'], name=role_name) + if role is None: + return {"errors": [f"role {role_name} not found, please create it in openreplay first"]} + + admin_privileges = user_data.get("adminPrivileges", []) + admin_privileges = not (len(admin_privileges) == 0 + or admin_privileges[0] is None + or admin_privileges[0].lower() == "false") + + if existing is None: + deleted = users.get_deleted_user_by_email(auth.get_nameid()) + if deleted is not None: + print("== restore deleted user ==") + users.restore_sso_user(user_id=deleted["userId"], tenant_id=t['tenantId'], email=email, + admin=admin_privileges, origin=SAML2_helper.get_saml2_provider(), + name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])), + internal_id=internal_id, role_id=role["roleId"]) + else: + print("== new user ==") + users.create_sso_user(tenant_id=t['tenantId'], email=email, admin=admin_privileges, + origin=SAML2_helper.get_saml2_provider(), + name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])), + internal_id=internal_id, role_id=role["roleId"]) + else: + if t['tenantId'] != existing["tenantId"]: + print("user exists for a different tenant") + return {"errors": ["user exists for a different tenant"]} + if existing.get("origin") is None: + print(f"== migrating user to {SAML2_helper.get_saml2_provider()} ==") + users.update(tenant_id=t['tenantId'], user_id=existing["id"], + changes={"origin": SAML2_helper.get_saml2_provider(), "internal_id": internal_id}) + expiration = auth.get_session_expiration() + expiration = expiration if expiration is not None and expiration > 10 * 60 \ + else int(config("sso_exp_delta_seconds", cast=int, default=24 * 60 * 60)) + jwt = users.authenticate_sso(email=email, internal_id=internal_id, exp=expiration) + if jwt is None: + return {"errors": ["null JWT"]} + return Response( + status_code=status.HTTP_302_FOUND, + headers={'Location': SAML2_helper.get_landing_URL(jwt)}) + + @public_app.get('/sso/saml2/sls', tags=["saml2"]) async def process_sls_assertion(request: Request): req = await prepare_request(request=request) @@ -143,6 +227,7 @@ async def process_sls_assertion(request: Request): return RedirectResponse(url=config("SITE_URL")) +@public_app.get('/sso/saml2/metadata/', tags=["saml2"]) @public_app.get('/sso/saml2/metadata', tags=["saml2"]) async def saml2_metadata(request: Request): req = await prepare_request(request=request) From 2482efb66d5ddc76e369cec9bdde7551316ef84b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 14:59:39 +0100 Subject: [PATCH 07/33] feat(api): EE-SSO remove /tenantKey from ACS endpoint --- ee/api/chalicelib/utils/SAML2_helper.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ee/api/chalicelib/utils/SAML2_helper.py b/ee/api/chalicelib/utils/SAML2_helper.py index a2a4e1e6e..6a1ee4eca 100644 --- a/ee/api/chalicelib/utils/SAML2_helper.py +++ b/ee/api/chalicelib/utils/SAML2_helper.py @@ -86,12 +86,22 @@ async def prepare_request(request: Request): session = {} # If server is behind proxys or balancers use the HTTP_X_FORWARDED fields headers = request.headers - url_data = urlparse('%s://%s' % (headers.get('x-forwarded-proto', 'http'), headers['host'])) + proto = headers.get('x-forwarded-proto', 'http') + url_data = urlparse('%s://%s' % (proto, headers['host'])) + path = request.url.path + # remove / from the /acs/ + if path.endswith("/"): + path = path[:-1] + # remove /{tenantKey} from /acs/{tenantKey} + if not path.endswith("/acs"): + parts = path.split("/") + if len(parts) > 2 and parts[-2] == "acs": + path = "/".join(parts[:-1]) return { - 'https': 'on' if request.headers.get('x-forwarded-proto', 'http') == 'https' else 'off', + 'https': 'on' if proto == 'https' else 'off', 'http_host': request.headers['host'], 'server_port': url_data.port, - 'script_name': "/api" + request.url.path, + 'script_name': "/api" + path, 'get_data': request.args.copy(), # Uncomment if using ADFS as IdP, https://github.com/onelogin/python-saml/pull/144 # 'lowercase_urlencoding': True, From 1ab8c1068784059658b8c39096695ddf2888902d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 15:26:44 +0100 Subject: [PATCH 08/33] feat(api): EE-SSO log request --- ee/api/routers/saml.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ee/api/routers/saml.py b/ee/api/routers/saml.py index 73b0e7393..117960b50 100644 --- a/ee/api/routers/saml.py +++ b/ee/api/routers/saml.py @@ -113,6 +113,9 @@ async def process_sso_assertion(request: Request): @public_app.post('/sso/saml2/acs/{tenantKey}', tags=["saml2"]) async def process_sso_assertion_tk(tenantKey: str, request: Request): req = await prepare_request(request=request) + print("------------") + print(req) + print("------------") session = req["cookie"]["session"] auth = init_saml_auth(req) From 320efff88ecfb14e154e04b4f0185bf5ed94575b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 15:46:46 +0100 Subject: [PATCH 09/33] feat(api): EE allow live editing --- api/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/entrypoint.sh b/api/entrypoint.sh index 60fefb5c0..a092737be 100755 --- a/api/entrypoint.sh +++ b/api/entrypoint.sh @@ -1,2 +1,2 @@ #!/bin/bash -uvicorn app:app --host 0.0.0.0 +uvicorn app:app --host 0.0.0.0 --reload From 8639f893792a37bae8819a0fc721b3f7a17d9c67 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 15:50:15 +0100 Subject: [PATCH 10/33] feat(api): EE allow live editing --- ee/api/.gitignore | 2 +- ee/api/entrypoint.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100755 ee/api/entrypoint.sh diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 8afea0ab6..f1ff9550b 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -253,7 +253,7 @@ Pipfile /db_changes.sql /Dockerfile.bundle /entrypoint.bundle.sh -/entrypoint.sh +#/entrypoint.sh /chalicelib/core/heatmaps.py /routers/subs/insights.py /schemas.py diff --git a/ee/api/entrypoint.sh b/ee/api/entrypoint.sh new file mode 100755 index 000000000..a092737be --- /dev/null +++ b/ee/api/entrypoint.sh @@ -0,0 +1,2 @@ +#!/bin/bash +uvicorn app:app --host 0.0.0.0 --reload From c181d2198a4abd847210a05fbff2c436d09b3683 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 21 Jan 2022 22:03:49 +0530 Subject: [PATCH 11/33] chore(nginx): Adding protocol scheme forwarding Signed-off-by: Rajesh Rajendran --- .../charts/nginx-ingress/templates/configMap.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml index f635b270c..77100c35c 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml @@ -62,7 +62,7 @@ data: proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $origin_proto; proxy_pass http://chalice-openreplay.app.svc.cluster.local:8000; } location /assist/ { @@ -134,6 +134,10 @@ data: default upgrade; '' close; } + map $http_x_forwarded_proto $origin_proto { + default $http_x_forwarded_proto; + '' $scheme; + } server { listen 80 default_server; listen [::]:80 default_server; From f519700feabf8d115832c91e0030dc1b3d7f1e9d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 18:43:12 +0100 Subject: [PATCH 12/33] feat(api): EE-SSO include tenantKey in ACS URL feat(helm): idp_sp_tk for chalice env vars --- ee/api/chalicelib/utils/SAML2_helper.py | 29 ++++++++++--------- ee/api/routers/saml.py | 9 +++--- scripts/helm/app/chalice.yaml | 1 + .../openreplay/charts/chalice/values.yaml | 1 + scripts/helmcharts/vars.yaml | 1 + 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/ee/api/chalicelib/utils/SAML2_helper.py b/ee/api/chalicelib/utils/SAML2_helper.py index 6a1ee4eca..86a6c4683 100644 --- a/ee/api/chalicelib/utils/SAML2_helper.py +++ b/ee/api/chalicelib/utils/SAML2_helper.py @@ -12,11 +12,11 @@ SAML2 = { "sp": { "entityId": config("SITE_URL") + "/api/sso/saml2/metadata/", "assertionConsumerService": { - "url": config("SITE_URL") + "/api/sso/saml2/acs", + "url": config("SITE_URL") + "/api/sso/saml2/acs/", "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" }, "singleLogoutService": { - "url": config("SITE_URL") + "/api/sso/saml2/sls", + "url": config("SITE_URL") + "/api/sso/saml2/sls/", "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" }, "NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress", @@ -25,6 +25,12 @@ SAML2 = { }, "idp": None } + +# in case tenantKey is included in the URL +sp_acs = config("idp_sp_tk", default="") +if sp_acs is not None and len(sp_acs) > 0: + SAML2["sp"]["assertionConsumerService"]["url"] += sp_acs + "/" + idp = None # SAML2 config handler if config("SAML2_MD_URL", default=None) is not None and len(config("SAML2_MD_URL")) > 0: @@ -60,12 +66,9 @@ else: def init_saml_auth(req): # auth = OneLogin_Saml2_Auth(req, custom_base_path=environ['SAML_PATH']) - if idp is None: raise Exception("No SAML2 config provided") - auth = OneLogin_Saml2_Auth(req, old_settings=SAML2) - - return auth + return OneLogin_Saml2_Auth(req, old_settings=SAML2) async def prepare_request(request: Request): @@ -87,16 +90,14 @@ async def prepare_request(request: Request): # If server is behind proxys or balancers use the HTTP_X_FORWARDED fields headers = request.headers proto = headers.get('x-forwarded-proto', 'http') + if headers.get('x-forwarded-proto') is not None: + print(f"x-forwarded-proto: {proto}") url_data = urlparse('%s://%s' % (proto, headers['host'])) path = request.url.path - # remove / from the /acs/ - if path.endswith("/"): - path = path[:-1] - # remove /{tenantKey} from /acs/{tenantKey} - if not path.endswith("/acs"): - parts = path.split("/") - if len(parts) > 2 and parts[-2] == "acs": - path = "/".join(parts[:-1]) + # add / to /acs + if not path.endswith("/"): + path = path + '/' + return { 'https': 'on' if proto == 'https' else 'off', 'http_host': request.headers['host'], diff --git a/ee/api/routers/saml.py b/ee/api/routers/saml.py index 117960b50..ee0f0333b 100644 --- a/ee/api/routers/saml.py +++ b/ee/api/routers/saml.py @@ -16,6 +16,7 @@ from starlette import status @public_app.get("/sso/saml2", tags=["saml2"]) +@public_app.get("/sso/saml2/", tags=["saml2"]) async def start_sso(request: Request): request.path = '' req = await prepare_request(request=request) @@ -25,6 +26,7 @@ async def start_sso(request: Request): @public_app.post('/sso/saml2/acs', tags=["saml2"]) +@public_app.post('/sso/saml2/acs/', tags=["saml2"]) async def process_sso_assertion(request: Request): req = await prepare_request(request=request) session = req["cookie"]["session"] @@ -111,11 +113,9 @@ async def process_sso_assertion(request: Request): @public_app.post('/sso/saml2/acs/{tenantKey}', tags=["saml2"]) +@public_app.post('/sso/saml2/acs/{tenantKey}/', tags=["saml2"]) async def process_sso_assertion_tk(tenantKey: str, request: Request): req = await prepare_request(request=request) - print("------------") - print(req) - print("------------") session = req["cookie"]["session"] auth = init_saml_auth(req) @@ -196,6 +196,7 @@ async def process_sso_assertion_tk(tenantKey: str, request: Request): @public_app.get('/sso/saml2/sls', tags=["saml2"]) +@public_app.get('/sso/saml2/sls/', tags=["saml2"]) async def process_sls_assertion(request: Request): req = await prepare_request(request=request) session = req["cookie"]["session"] @@ -230,8 +231,8 @@ async def process_sls_assertion(request: Request): return RedirectResponse(url=config("SITE_URL")) -@public_app.get('/sso/saml2/metadata/', tags=["saml2"]) @public_app.get('/sso/saml2/metadata', tags=["saml2"]) +@public_app.get('/sso/saml2/metadata/', tags=["saml2"]) async def saml2_metadata(request: Request): req = await prepare_request(request=request) auth = init_saml_auth(req) diff --git a/scripts/helm/app/chalice.yaml b/scripts/helm/app/chalice.yaml index fcbea8ed6..09d015401 100644 --- a/scripts/helm/app/chalice.yaml +++ b/scripts/helm/app/chalice.yaml @@ -64,5 +64,6 @@ env: idp_x509cert: '' idp_sls_url: '' idp_name: '' + idp_sp_tk: '' assist_secret: '' iceServers: '' diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index e5c579739..8e0080746 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -94,6 +94,7 @@ env: idp_x509cert: '' idp_sls_url: '' idp_name: '' + idp_sp_tk: '' assist_secret: '' iceServers: '' diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index e8ef20bf7..12b4971f8 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -73,6 +73,7 @@ chalice: # idp_x509cert: '' # idp_sls_url: '' # idp_name: '' + # idp_sp_tk: '' # If you want to override something From 414a6da84d22eec9addca66351f43536449eabf0 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 23:04:43 +0100 Subject: [PATCH 13/33] feat(api): EE-SSO include idp_tenantKey in ACS URL/path feat(helm): idp_tenantKey for chalice env vars --- ee/api/chalicelib/utils/SAML2_helper.py | 2 +- scripts/helm/app/chalice.yaml | 2 +- scripts/helmcharts/openreplay/charts/chalice/values.yaml | 2 +- scripts/helmcharts/vars.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ee/api/chalicelib/utils/SAML2_helper.py b/ee/api/chalicelib/utils/SAML2_helper.py index 86a6c4683..c00081d2c 100644 --- a/ee/api/chalicelib/utils/SAML2_helper.py +++ b/ee/api/chalicelib/utils/SAML2_helper.py @@ -27,7 +27,7 @@ SAML2 = { } # in case tenantKey is included in the URL -sp_acs = config("idp_sp_tk", default="") +sp_acs = config("idp_tenantKey", default="") if sp_acs is not None and len(sp_acs) > 0: SAML2["sp"]["assertionConsumerService"]["url"] += sp_acs + "/" diff --git a/scripts/helm/app/chalice.yaml b/scripts/helm/app/chalice.yaml index 09d015401..2d6b53ead 100644 --- a/scripts/helm/app/chalice.yaml +++ b/scripts/helm/app/chalice.yaml @@ -64,6 +64,6 @@ env: idp_x509cert: '' idp_sls_url: '' idp_name: '' - idp_sp_tk: '' + idp_tenantKey: '' assist_secret: '' iceServers: '' diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index 8e0080746..5e76420e8 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -94,7 +94,7 @@ env: idp_x509cert: '' idp_sls_url: '' idp_name: '' - idp_sp_tk: '' + idp_tenantKey: '' assist_secret: '' iceServers: '' diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 12b4971f8..46d52911c 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -73,7 +73,7 @@ chalice: # idp_x509cert: '' # idp_sls_url: '' # idp_name: '' - # idp_sp_tk: '' + # idp_tenantKey: '' # If you want to override something From 9025433cf5143b50570e24457efb7e32c0da5796 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 11:24:23 +0100 Subject: [PATCH 14/33] feat(api): alerts/triggers endpoint --- api/chalicelib/core/alerts.py | 13 ++++++++++++- api/chalicelib/core/custom_metrics.py | 13 +++++++------ api/routers/core.py | 11 ++++++----- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/api/chalicelib/core/alerts.py b/api/chalicelib/core/alerts.py index c701f0ce0..6fe799c19 100644 --- a/api/chalicelib/core/alerts.py +++ b/api/chalicelib/core/alerts.py @@ -1,11 +1,12 @@ import json +import logging import time import schemas from chalicelib.core import notifications, slack, webhook from chalicelib.utils import pg_client, helper, email_helper from chalicelib.utils.TimeUTC import TimeUTC -import logging + def get(id): with pg_client.PostgresClient() as cur: @@ -157,3 +158,13 @@ def delete(project_id, alert_id): {"alert_id": alert_id, "project_id": project_id}) ) return {"data": {"state": "success"}} + + +def get_predefined_values(): + values = [e.value for e in schemas.AlertColumn] + values = [{"name": v, "value": v, + "unit": "count" if v.endswith(".count") else "ms", + "predefined": True, + "metricId": None, + "seriesId": None} for v in values] + return values diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 62bddfbb3..787b19837 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -205,17 +205,18 @@ def get_series_for_alert(project_id, user_id): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify( - """SELECT metric_id, - series_id, - metrics.name AS metric_name, - metric_series.name AS series_name, - index AS series_index + """SELECT series_id AS value, + metrics.name || '.' || (COALESCE(metric_series.name, 'series ' || index)) || '.count' AS name, + 'count' AS unit, + FALSE AS predefined, + metric_id, + series_id FROM metric_series INNER JOIN metrics USING (metric_id) WHERE metrics.deleted_at ISNULL AND metrics.project_id = %(project_id)s AND (user_id = %(user_id)s OR is_public) - ORDER BY metric_name, series_index, series_name;""", + ORDER BY name;""", {"project_id": project_id, "user_id": user_id} ) ) diff --git a/api/routers/core.py b/api/routers/core.py index 9c5bec86e..129ae9cc9 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -621,6 +621,12 @@ def get_all_alerts(projectId: int, context: schemas.CurrentContext = Depends(OR_ return {"data": alerts.get_all(projectId)} +@app.get('/{projectId}/alerts/triggers', tags=["alerts", "customMetrics"]) +def get_alerts_triggers(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": alerts.get_predefined_values() \ + + custom_metrics.get_series_for_alert(project_id=projectId, user_id=context.user_id)} + + @app.get('/{projectId}/alerts/{alertId}', tags=["alerts"]) def get_alert(projectId: int, alertId: int, context: schemas.CurrentContext = Depends(OR_context)): return {"data": alerts.get(alertId)} @@ -1072,11 +1078,6 @@ def get_custom_metric_chart(projectId: int, data: schemas.CustomMetricChartPaylo data=data)} -@app.get('/{projectId}/custom_metrics/series', tags=["customMetrics"]) -def get_series_for_alert(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.get_series_for_alert(project_id=projectId, user_id=context.user_id)} - - @app.post('/{projectId}/custom_metrics', tags=["customMetrics"]) @app.put('/{projectId}/custom_metrics', tags=["customMetrics"]) def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), From 258ea6d9238c72b2008ff81f78f2ecd301a90afe Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 12:35:12 +0100 Subject: [PATCH 15/33] feat(api): sessions-search support Old&Flat search payload --- api/routers/core.py | 2 +- api/schemas.py | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/api/routers/core.py b/api/routers/core.py index 129ae9cc9..bbeb30bcd 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -108,7 +108,7 @@ def events_search(projectId: int, q: str, type: Union[schemas.FilterType, schema @app.post('/{projectId}/sessions/search2', tags=["sessions"]) -def sessions_search2(projectId: int, data: schemas.SessionsSearchPayloadSchema = Body(...), +def sessions_search2(projectId: int, data: schemas.FlatSessionsSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): data = sessions.search2_pg(data, projectId, user_id=context.user_id) return {'data': data} diff --git a/api/schemas.py b/api/schemas.py index 0dfd949ac..949ab0dff 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -458,7 +458,15 @@ class IssueType(str, Enum): js_exception = 'js_exception' -class _SessionSearchEventRaw(BaseModel): +class __MixedSearchFilter(BaseModel): + is_event: bool = Field(...) + + class Config: + alias_generator = attribute_to_camel_case + + +class _SessionSearchEventRaw(__MixedSearchFilter): + is_event: bool = Field(True, const=True) custom: Optional[List[Union[int, str]]] = Field(None, min_items=1) customOperator: Optional[MathOperator] = Field(None) key: Optional[str] = Field(None) @@ -491,7 +499,8 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw): value: Union[List[_SessionSearchEventRaw], str, List[str]] = Field(...) -class _SessionSearchFilterSchema(BaseModel): +class _SessionSearchFilterSchema(__MixedSearchFilter): + is_event: bool = Field(False, const=False) custom: Optional[List[str]] = Field(None) key: Optional[str] = Field(None) value: Union[Optional[Union[IssueType, PlatformType, int, str]], @@ -536,6 +545,31 @@ class SessionsSearchPayloadSchema(BaseModel): alias_generator = attribute_to_camel_case +class FlatSessionsSearchPayloadSchema(SessionsSearchPayloadSchema): + events: Optional[List[_SessionSearchEventSchema]] = Field([]) + filters: List[Union[_SessionSearchFilterSchema, _SessionSearchEventSchema]] = Field([]) + + @root_validator(pre=True) + def flat_to_original(cls, values): + # in case the old search body was passed + if len(values.get("events", [])) > 0: + for v in values["events"]: + v["isEvent"] = True + for v in values.get("filters", []): + v["isEvent"] = False + else: + n_filters = [] + n_events = [] + for v in values.get("filters", []): + if v["isEvent"]: + n_events.append(v) + else: + n_filters.append(v) + values["events"] = n_events + values["filters"] = n_filters + return values + + class SessionsSearchCountSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(default=None) order: Optional[str] = Field(default=None) From 555de84ccf0078c0926eafb566ce264040ee6f52 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 15:15:20 +0100 Subject: [PATCH 16/33] feat(api): flat custom_metrics --- api/chalicelib/core/custom_metrics.py | 20 ++++++++++++++------ api/chalicelib/utils/helper.py | 11 +++++++++++ api/schemas.py | 3 ++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 787b19837..10e86024c 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -26,7 +26,9 @@ def try_live(project_id, data: schemas.TryCustomMetricsSchema): def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema): - metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id) + metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) + if metric is None: + return None metric: schemas.TryCustomMetricsSchema = schemas.TryCustomMetricsSchema.parse_obj({**data.dict(), **metric}) return try_live(project_id=project_id, data=metric) @@ -113,10 +115,10 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche u AS (UPDATE metric_series SET name=series.name, filter=series.filter, - index=series.filter.index + index=series.index FROM (VALUES {",".join([f"(%(u_series_id_{s['i']})s,%(u_index_{s['i']})s,%(u_name_{s['i']})s,%(u_filter_{s['i']})s::jsonb)" - for s in n_series])}) AS series(series_id, index, name, filter) - WHERE metric_id =%(metric_id)s AND series_id=series.series_id + for s in u_series])}) AS series(series_id, index, name, filter) + WHERE metric_series.metric_id =%(metric_id)s AND metric_series.series_id=series.series_id RETURNING 1)""") if len(d_series_ids) > 0: sub_queries.append("""\ @@ -133,7 +135,6 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche cur.execute( query ) - r = cur.fetchone() return get(metric_id=metric_id, project_id=project_id, user_id=user_id) @@ -158,6 +159,8 @@ def get_all(project_id, user_id): rows = cur.fetchall() for r in rows: r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + for s in r["series"]: + s["filter"] = helper.old_search_payload_to_flat(s["filter"]) rows = helper.list_to_camel_case(rows) return rows @@ -177,7 +180,7 @@ def delete(project_id, metric_id, user_id): return {"state": "success"} -def get(metric_id, project_id, user_id): +def get(metric_id, project_id, user_id, flatten=True): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify( @@ -197,7 +200,12 @@ def get(metric_id, project_id, user_id): ) ) row = cur.fetchone() + if row is None: + return None row["created_at"] = TimeUTC.datetime_to_timestamp(row["created_at"]) + if flatten: + for s in row["series"]: + s["filter"] = helper.old_search_payload_to_flat(s["filter"]) return helper.dict_to_camel_case(row) diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py index f8ce9fab5..6887fa5da 100644 --- a/api/chalicelib/utils/helper.py +++ b/api/chalicelib/utils/helper.py @@ -366,3 +366,14 @@ def has_smtp(): def get_edition(): return "ee" if "ee" in config("ENTERPRISE_BUILD", default="").lower() else "foss" + + +def old_search_payload_to_flat(values): + # in case the old search body was passed + if values.get("events") is not None: + for v in values["events"]: + v["isEvent"] = True + for v in values.get("filters", []): + v["isEvent"] = False + values["filters"] = values.pop("events") + values.get("filters", []) + return values diff --git a/api/schemas.py b/api/schemas.py index 949ab0dff..c5623f972 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -629,7 +629,8 @@ class MobileSignPayloadSchema(BaseModel): keys: List[str] = Field(...) -class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): +class CustomMetricSeriesFilterSchema(FlatSessionsSearchPayloadSchema): +# class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): startDate: Optional[int] = Field(None) endDate: Optional[int] = Field(None) sort: Optional[str] = Field(None) From 7658d9224bf2c21904def98d730e5b9d66080415 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 15:36:27 +0100 Subject: [PATCH 17/33] feat(api): flat funnels 1/2 --- api/chalicelib/core/funnels.py | 17 +++++++++++------ api/schemas.py | 11 +++++++---- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 835a655f4..b9db6c631 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -99,7 +99,8 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date # row["filter"]["events"] = filter_stages(row["filter"]["events"]) get_start_end_time(filter_d=row["filter"], range_value=range_value, start_date=start_date, end_date=end_date) - counts = sessions.search2_pg(data=row["filter"], project_id=project_id, user_id=None, count_only=True) + counts = sessions.search2_pg(data=schemas.SessionsSearchPayloadSchema.parse_obj(row["filter"]), + project_id=project_id, user_id=None, count_only=True) row["sessionsCount"] = counts["countSessions"] row["usersCount"] = counts["countUsers"] overview = significance.get_overview(filter_d=row["filter"], project_id=project_id) @@ -110,6 +111,7 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date row["criticalIssuesCount"] = overview["criticalIssuesCount"] row["missedConversions"] = 0 if len(row["stages"]) < 2 \ else row["stages"][0]["sessionsCount"] - row["stages"][-1]["sessionsCount"] + row["filter"] = helper.old_search_payload_to_flat(row["filter"]) return rows @@ -147,11 +149,12 @@ def delete(project_id, funnel_id, user_id): def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) - return sessions.search2_pg(data=f["filter"], project_id=project_id, user_id=user_id) + return sessions.search2_pg(data=schemas.SessionsSearchPayloadSchema.parse_obj(f["filter"]), project_id=project_id, + user_id=user_id) def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.FunnelSearchPayloadSchema): @@ -168,7 +171,7 @@ def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.Funnel def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -196,7 +199,7 @@ def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data): def get_issues(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -224,7 +227,7 @@ def get_issues_on_the_fly(funnel_id, user_id, project_id, data): last_stage=last_stage))} -def get(funnel_id, project_id, user_id): +def get(funnel_id, project_id, user_id, flatten=True): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify( @@ -246,6 +249,8 @@ def get(funnel_id, project_id, user_id): f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) # f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) + if flatten: + f["filter"] = helper.old_search_payload_to_flat(f["filter"]) return f diff --git a/api/schemas.py b/api/schemas.py index c5623f972..d66b92444 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -570,12 +570,14 @@ class FlatSessionsSearchPayloadSchema(SessionsSearchPayloadSchema): return values -class SessionsSearchCountSchema(SessionsSearchPayloadSchema): +class SessionsSearchCountSchema(FlatSessionsSearchPayloadSchema): + # class SessionsSearchCountSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(default=None) order: Optional[str] = Field(default=None) -class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema): +class FunnelSearchPayloadSchema(FlatSessionsSearchPayloadSchema): + # class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema): range_value: Optional[str] = Field(None) sort: Optional[str] = Field(None) order: Optional[str] = Field(None) @@ -599,7 +601,8 @@ class UpdateFunnelSchema(FunnelSchema): is_public: Optional[bool] = Field(None) -class FunnelInsightsPayloadSchema(SessionsSearchPayloadSchema): +class FunnelInsightsPayloadSchema(FlatSessionsSearchPayloadSchema): + # class FunnelInsightsPayloadSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(None) order: Optional[str] = Field(None) @@ -630,7 +633,7 @@ class MobileSignPayloadSchema(BaseModel): class CustomMetricSeriesFilterSchema(FlatSessionsSearchPayloadSchema): -# class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): + # class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): startDate: Optional[int] = Field(None) endDate: Optional[int] = Field(None) sort: Optional[str] = Field(None) From 1f379435ecdd29707e8426cd852a13a6300b6484 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 21 Jan 2022 22:34:32 +0530 Subject: [PATCH 18/33] chore(helm): variable naming convention Signed-off-by: Rajesh Rajendran --- .../openreplay/charts/chalice/templates/deployment.yaml | 4 ++-- .../openreplay/charts/storage/templates/deployment.yaml | 4 ++-- scripts/helmcharts/vars.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index 41df5e2f5..25ec6c387 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -69,9 +69,9 @@ spec: - name: sessions_region value: '{{ .Values.global.s3.region }}' - name: sessions_bucket - value: '{{ .Values.global.s3.recordings_bucket }}' + value: '{{ .Values.global.s3.recordingsBucket }}' - name: sourcemaps_bucket - value: '{{ .Values.global.s3.sourcemaps_bucket }}' + value: '{{ .Values.global.s3.sourcemapsBucket }}' - name: js_cache_bucket value: '{{ .Values.global.s3.assetsBucket }}' - name: EMAIL_HOST diff --git a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml index 4ca3be6e2..c5ec92cf4 100644 --- a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml @@ -49,9 +49,9 @@ spec: - name: AWS_REGION_IOS value: '{{ .Values.global.s3.region }}' - name: S3_BUCKET_WEB - value: '{{ .Values.global.s3.recordings_bucket }}' + value: '{{ .Values.global.s3.recordingsBucket }}' - name: S3_BUCKET_IOS - value: '{{ .Values.global.s3.recordings_bucket }}' + value: '{{ .Values.global.s3.recordingsBucket }}' - name: REDIS_STRING value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - name: LICENSE_KEY diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 46d52911c..85901873f 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -43,8 +43,8 @@ global: region: "us-east-1" endpoint: "http://minio.db.svc.cluster.local:9000" assetsBucket: "sessions-assets" - recordings_bucket: "mobs" - sourcemaps_bucket: "sourcemaps" + recordingsBucket: "mobs" + sourcemapsBucket: "sourcemaps" # if you're using one node installation, where # you're using local s3, make sure these variables # are same as minio.global.minio.accesskey and secretKey From 76f925beed0affadb1df38bc1fb6ec168da1c44c Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 21 Jan 2022 23:21:42 +0530 Subject: [PATCH 19/33] chore(helm): example for db upgrade Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/vars.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 85901873f..a19ab96ae 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -8,6 +8,13 @@ postgresql: &postgres postgresqlPort: "5432" postgresqlUser: "postgres" postgresqlDatabase: "postgres" + # resources: + # requests: + # memory: 256Mi + # cpu: 250m + # limits: + # memory: 3000Mi + # cpu: 2 clickhouse: {} # For enterpriseEdition @@ -91,6 +98,8 @@ chalice: ## Changes to nginx # # nginx-ingress: +# sslKey: site.crt +# sslCert: site.crt # customServerConfigs: | # # Redirecting http to https # return 301 https://$host$request_uri; From ef8eb2e9940b74a1b28686f1291e5a2aaacaea8e Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 18:46:02 +0100 Subject: [PATCH 20/33] Update vars.yaml --- scripts/helmcharts/vars.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index a19ab96ae..5fd626d48 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -90,10 +90,14 @@ chalice: # For example: # -# alerts: +# http: # resources: # limits: -# cpu: 1 +# cpu: 1024m +# memory: 4096Mi +# requests: +# cpu: 512m +# memory: 2056Mi ## Changes to nginx # From 29a37918f571a6e8d3b2dbd0c2e8c81f82a87c06 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 18:46:17 +0100 Subject: [PATCH 21/33] Update vars.yaml --- scripts/helmcharts/vars.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 5fd626d48..9ad685642 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -108,20 +108,3 @@ chalice: # # Redirecting http to https # return 301 https://$host$request_uri; # -### Change the ssl certificates -### Public certificate ( content from site.crt, mind the indentation ) -# ssl_certificate: |- -# -----BEGIN CERTIFICATE----- -# MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL -# BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3 -# NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh -# -----END CERTIFICATE----- -# -### Private certificate ( content from site.key, mind the indentation. ) -# ssl_privatekey: |- -# -----BEGIN PRIVATE KEY----- -# TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T -# e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP -# 3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 -# 7qINKoGovqXB1oAdopIl1z64e7MWVE4= -# -----END PRIVATE KEY----- From 56c4308b20a96b66132f6252640ab17f680af55c Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 18:02:36 +0100 Subject: [PATCH 22/33] Update vars.yaml --- scripts/helmcharts/vars.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 9ad685642..62482dd50 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -74,6 +74,8 @@ global: chalice: env: jwt_secret: "SetARandomStringHere" + # captcha_server: '' + # captcha_key: '' # SAML2_MD_URL: '' # idp_entityId: '' # idp_sso_url: '' From e474ecd0bf4c5619b4c566a1ba0e51bba066fb18 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 22 Jan 2022 21:12:06 +0530 Subject: [PATCH 23/33] chore(helm): read nginx ssl from file Signed-off-by: Rajesh Rajendran --- .../charts/nginx-ingress/files/site.crt | 1 + .../charts/nginx-ingress/files/site.key | 1 + .../nginx-ingress/templates/secrets.yaml | 6 +- .../charts/nginx-ingress/values.yaml | 87 ------------------- scripts/helmcharts/openreplay/files/site.crt | 30 +++++++ scripts/helmcharts/openreplay/files/site.key | 52 +++++++++++ scripts/helmcharts/vars.yaml | 2 - 7 files changed, 87 insertions(+), 92 deletions(-) create mode 120000 scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt create mode 120000 scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key create mode 100644 scripts/helmcharts/openreplay/files/site.crt create mode 100644 scripts/helmcharts/openreplay/files/site.key diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt new file mode 120000 index 000000000..12e23824a --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt @@ -0,0 +1 @@ +../../../files/site.crt \ No newline at end of file diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key new file mode 120000 index 000000000..3805a27d1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key @@ -0,0 +1 @@ +../../../files/site.key \ No newline at end of file diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml index e5e4d7dd9..91b7cc09c 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml @@ -4,6 +4,6 @@ kind: Secret metadata: name: ssl data: - ca.crt: "" - site.crt: "{{ .Values.ssl_certificate | b64enc }}" - site.key: "{{ .Values.ssl_privatekey | b64enc }}" + ca.crt: '' + site.crt: '{{ .Files.Get "files/site.crt" | b64enc }}' + site.key: '{{ .Files.Get "files/site.key" | b64enc }}' diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml index 8c595aafa..821ad9e3c 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml @@ -84,90 +84,3 @@ nodeSelector: {} tolerations: [] affinity: {} - - -ssl_certificate: |- - -----BEGIN CERTIFICATE----- - MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL - BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3 - NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh - bC5ob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyXTX6RwqNVM+ - LSvc5TkuBnxlw1sHxtkkojwpbwavr6ccSdtoYB7KYwcufh0zz3LaSDgPNqStOf6w - hAWV830bxvOvU6yJ7MgP8/htfY1KWIoNS6ducoct4VhgshWXWwQtrtWZJku+cyds - QTkr2BziSX+Y7/1rALKbOU4CIRCKtJ2jeaI+c4kcXXB+ARauDlqB7+CS4B+wjlfX - sOoC2bWgZOxyZnHolb3hKMLfBswLwYq0DRjjNMDqX8xS6V1AgoTrCxl1DqPLw47o - immbSKZ4voot60cSBYVK4qOX5Nqw5RmqwELb9Ib4QPVCt9HjbYQp77EcOonkgE4l - fYabvvOeM/U6vdtZhI2CJg0tkytuJ4+Hb7i7nRK2SRMppmtP7yDDXpMGoAXK2bVZ - ipZBRct0onxLifH5vdrUNbOlXItjWLQMfiHlDeG48kbXbKaJPv3tRvU0Gix2X8SJ - OlRNezNNz8pce0Bbgx3YoQhrRTad4CC6cIpRjgTt/pww3BoF7jDLl6RNI1cXfy4u - tkSlMqAQV6x0aig9Ldg1VFM2oCaEyvzx0BWDm/jmbZcyVizlb+uQQ/huNSJXT++p - CmPNG7rP6eYNTh7+7DDWvKBQQFWOPaVfwvrhzvb7q2B2Bmc33bDXeRuF4MJE6syA - YUCV2Ztw65uI864PRDIKO4ru1UQgx5sCAwEAAaNTMFEwHQYDVR0OBBYEFIdNQGn2 - z3xmJfExKAUeohFnLjzsMB8GA1UdIwQYMBaAFIdNQGn2z3xmJfExKAUeohFnLjzs - MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJvkSX+RoYGOcU0z - qEWlAN6Jy0CzLsTp/VoSZY9gd+hLFd/lK7SjXhAnNyEELpeanGs04ytJpXnSH1bA - dGs0UeWYalOxrHSN4ln5SNzWVE8kODM1zcyNePllI/PVcXLmujQz5wNbNoC5Qt8p - 0RoZ2wInmyh2wTQcflIPUtncsw84ozVVuebmc6jiuPxnxdTAXeYOwKUF25t8rSp6 - 5n23F0GP8Ypu7vjT7N2RpUe0zkutaij+uISBHZw50ohrelPlV4V9qhp6MV+h9xuh - 0z8OEyq2vK4KNn96A97mSRuqqt6Ajb2MHdErTr6fgj5/CtSD337oIK3froRmID8s - /JXADsNnBEqQBfcM6gSaw1M/fHDPNZzwVv6yAN+bKrI+KEmKJD31Tm2G55oPvLTP - XZdmVIAqxIu89v/GOJ2J29vC+h9pTjTze31DFg0niwLcr1aNawiC2d4n2wdDwKwc - HnCnflELyYcn4KgvpLNz5wEKEHTAQ3JF5VIel1/uqYN9cosw1vjRskPK/g3nIEPG - T247naj+JbW244P0jxb57VWiD/7IJ4ZErA1KrvqR/y1NnGxrgXoRjwmhCv/4YIYi - qgnvF7IkwGozdoLPiBMmvjNq/AmVLrfZNPxZjHL3nIW+PBEeBD/lkH36mcakg/1S - w7yMPvE+TIh6+HwDZc2jNLkv/8tY - -----END CERTIFICATE----- - -ssl_privatekey: |- - -----BEGIN PRIVATE KEY----- - MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDJdNfpHCo1Uz4t - K9zlOS4GfGXDWwfG2SSiPClvBq+vpxxJ22hgHspjBy5+HTPPctpIOA82pK05/rCE - BZXzfRvG869TrInsyA/z+G19jUpYig1Lp25yhy3hWGCyFZdbBC2u1ZkmS75zJ2xB - OSvYHOJJf5jv/WsAsps5TgIhEIq0naN5oj5ziRxdcH4BFq4OWoHv4JLgH7COV9ew - 6gLZtaBk7HJmceiVveEowt8GzAvBirQNGOM0wOpfzFLpXUCChOsLGXUOo8vDjuiK - aZtIpni+ii3rRxIFhUrio5fk2rDlGarAQtv0hvhA9UK30eNthCnvsRw6ieSATiV9 - hpu+854z9Tq921mEjYImDS2TK24nj4dvuLudErZJEymma0/vIMNekwagBcrZtVmK - lkFFy3SifEuJ8fm92tQ1s6Vci2NYtAx+IeUN4bjyRtdspok+/e1G9TQaLHZfxIk6 - VE17M03Pylx7QFuDHdihCGtFNp3gILpwilGOBO3+nDDcGgXuMMuXpE0jVxd/Li62 - RKUyoBBXrHRqKD0t2DVUUzagJoTK/PHQFYOb+OZtlzJWLOVv65BD+G41IldP76kK - Y80bus/p5g1OHv7sMNa8oFBAVY49pV/C+uHO9vurYHYGZzfdsNd5G4XgwkTqzIBh - QJXZm3Drm4jzrg9EMgo7iu7VRCDHmwIDAQABAoICAQCebjlupiu7jB+Vvq0VyAYe - K66MGAbhpttcixu6qPN5nF5u5xIKpaxcfMVfgO/B8X0g1pWAT7m7pkSDTzFCL92s - dPApScOeZyfEolbZKkiRoOAb4yzE/PJkCfDhnIFPntWebXTn3SGFxjcohCGq7+w2 - CRbphc6k2dGhG2wpPK0YpfBuM94RVn7sLQ+rI3724s7VKzPW9pUPHJ4QD7j2JhRh - ymGdl29mc9GjEL38xnNoXgCDXFMypZSsii+aPzAAdS+zpu2b+czBmp3eXHc2h1Tl - 5B2Arn/Jv63I1wcZf7MmOS1DzlDU2WBbFYbGsVW+RvYD/rFIiDEfhlWNhlLttQFw - TJ9xk+EePK9VQuWzN5tG1lEjGcNWtPLUp3IxZTqaei5lWu6zyA6HVsxjyArzmfNk - x0fRpZU+VZYzbkgj0ROq3wg7QEEMQ8SPo9vvLF1ZNnndzs/ziPA1CodUuSwa6B2c - Zeref4s0B//q3U1SDQE08OD9iuZODwtkO4wQtW2DP33gC6VIts94jg87z8SRDp2g - DcT3D8ZhV5B2VPelluQZ/scWKGWKAvPVRjq51EiMeZtFBVyM6+o0xW2+MxxZdjbj - OWexc+dw8QfwIlFRm0v8Tfvljk1prqYEMLV4s9JD8up5X1h3Yg5uAsQpdZ+1JkGm - 5UvvQQVQgxkC1NFXxqYyQQKCAQEA95r3oYm+bnQXHXX4dBQO98+XwOOfnhMk815p - /CAuxCzbPNafjqyAxRgmp5D/IkdLzbitDL3uo73ot1RwB4CEN/Ovp3a/+CklBnnA - 0bKAtsGE2XWaqFeguVIy25WEWKaTxKGX0g6KHkOvGt0DNo4wUJUk+2sAqIvXU1Q6 - tUbd+8YRYxO7i6+92K7kxoZega6qiA/L3akZ2uTzFf+IskfqmDUoF2ZaEOFluG8E - ASX3KoVFfraV3DBEN0ionvfpaRIidr2IsuC848zHFBtAXA0mL55BCuf++HmAZnpy - HFN7owVVgqbEw+GGbNdRLt5zV00DmX/sHsIZU/gCLRPsfPUAqQKCAQEA0ElWWiS4 - IA91lWbzCwswFrHvjpcogh67fNd9kJCcFCUHabev7SSrIEcjqDP2m6HPvp/FwxYA - PEo1/vDZ884v9roft2J13OvpvXoqtRZLGo1E76sECBrcto4nhCiTeRQg9uRpHG+Q - p77QC/4eRBLGykFRJET6913x7JzpjAO0QLLLzilj1yBkbF5U01Up5KbIuNeXNvEO - GVGpbryIXxwR6Qhyv7C54xpjRdu9EOT1frRqdIs0qOGafnLXWAXKfvWUzz1wSiiw - 1p7xqYZrawXAr7XEkGA2aeqt/iqo2X2G9oYA0apJVwfR4WhuS2hPkSy405bsrGzZ - cjMs9bnJSYP8owKCAQEAxCTSvfisDjuQhBoL84hgQxcEFB09OK/ZuaC1PLER2v3d - vtgWFaO5bmivVlaahcEM367IByv+e1/ZlkEhbg/0rY4xO+vqLuAJIJQalwNcy2mJ - n+p11Z11CNmAyEotSTzMGhwYdKJn74mWkSU7gmApDezYGwKsxtfgf3Zd+3RkLSq+ - Y0oia4mQTrJdMJcJDpobJSW+TZ3DiY+MsYR3+SLXSDPzynWeK3kiZ3QqK+6zWc+x - OavSE1d48oJwcV3aXQ2sl3uVan51o894dQkRdtpDwb0PsWAOry8w8/1Tn/TSIFX9 - Yz5Q6Qsivd3jxckafbHYhCS+G6+O+OGid6ssz+AV4QKCAQAqK78ND0QsUZT4A9kP - kltRLQOYtiggeEJzm1mz7GN9fKXMlMFM3VC8f0rL4oF6rz9VlBnBTvILQuc9z9wB - De0OIk8LnSbJ7QXtNA/zjCj2nkWn1NNDJNUtLQj5LBH3wMiP1F0nwbrjC7Ipy3Cr - TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T - e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP - 3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 - WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa - 8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb - esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn - PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6 - PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5 - 7qINKoGovqXB1oAdopIl1z64e7MWVE4= - -----END PRIVATE KEY----- diff --git a/scripts/helmcharts/openreplay/files/site.crt b/scripts/helmcharts/openreplay/files/site.crt new file mode 100644 index 000000000..90f6a68c2 --- /dev/null +++ b/scripts/helmcharts/openreplay/files/site.crt @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL +BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3 +NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh +bC5ob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyXTX6RwqNVM+ +LSvc5TkuBnxlw1sHxtkkojwpbwavr6ccSdtoYB7KYwcufh0zz3LaSDgPNqStOf6w +hAWV830bxvOvU6yJ7MgP8/htfY1KWIoNS6ducoct4VhgshWXWwQtrtWZJku+cyds +QTkr2BziSX+Y7/1rALKbOU4CIRCKtJ2jeaI+c4kcXXB+ARauDlqB7+CS4B+wjlfX +sOoC2bWgZOxyZnHolb3hKMLfBswLwYq0DRjjNMDqX8xS6V1AgoTrCxl1DqPLw47o +immbSKZ4voot60cSBYVK4qOX5Nqw5RmqwELb9Ib4QPVCt9HjbYQp77EcOonkgE4l +fYabvvOeM/U6vdtZhI2CJg0tkytuJ4+Hb7i7nRK2SRMppmtP7yDDXpMGoAXK2bVZ +ipZBRct0onxLifH5vdrUNbOlXItjWLQMfiHlDeG48kbXbKaJPv3tRvU0Gix2X8SJ +OlRNezNNz8pce0Bbgx3YoQhrRTad4CC6cIpRjgTt/pww3BoF7jDLl6RNI1cXfy4u +tkSlMqAQV6x0aig9Ldg1VFM2oCaEyvzx0BWDm/jmbZcyVizlb+uQQ/huNSJXT++p +CmPNG7rP6eYNTh7+7DDWvKBQQFWOPaVfwvrhzvb7q2B2Bmc33bDXeRuF4MJE6syA +YUCV2Ztw65uI864PRDIKO4ru1UQgx5sCAwEAAaNTMFEwHQYDVR0OBBYEFIdNQGn2 +z3xmJfExKAUeohFnLjzsMB8GA1UdIwQYMBaAFIdNQGn2z3xmJfExKAUeohFnLjzs +MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJvkSX+RoYGOcU0z +qEWlAN6Jy0CzLsTp/VoSZY9gd+hLFd/lK7SjXhAnNyEELpeanGs04ytJpXnSH1bA +dGs0UeWYalOxrHSN4ln5SNzWVE8kODM1zcyNePllI/PVcXLmujQz5wNbNoC5Qt8p +0RoZ2wInmyh2wTQcflIPUtncsw84ozVVuebmc6jiuPxnxdTAXeYOwKUF25t8rSp6 +5n23F0GP8Ypu7vjT7N2RpUe0zkutaij+uISBHZw50ohrelPlV4V9qhp6MV+h9xuh +0z8OEyq2vK4KNn96A97mSRuqqt6Ajb2MHdErTr6fgj5/CtSD337oIK3froRmID8s +/JXADsNnBEqQBfcM6gSaw1M/fHDPNZzwVv6yAN+bKrI+KEmKJD31Tm2G55oPvLTP +XZdmVIAqxIu89v/GOJ2J29vC+h9pTjTze31DFg0niwLcr1aNawiC2d4n2wdDwKwc +HnCnflELyYcn4KgvpLNz5wEKEHTAQ3JF5VIel1/uqYN9cosw1vjRskPK/g3nIEPG +T247naj+JbW244P0jxb57VWiD/7IJ4ZErA1KrvqR/y1NnGxrgXoRjwmhCv/4YIYi +qgnvF7IkwGozdoLPiBMmvjNq/AmVLrfZNPxZjHL3nIW+PBEeBD/lkH36mcakg/1S +w7yMPvE+TIh6+HwDZc2jNLkv/8tY +-----END CERTIFICATE----- diff --git a/scripts/helmcharts/openreplay/files/site.key b/scripts/helmcharts/openreplay/files/site.key new file mode 100644 index 000000000..641fd1b7b --- /dev/null +++ b/scripts/helmcharts/openreplay/files/site.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDJdNfpHCo1Uz4t +K9zlOS4GfGXDWwfG2SSiPClvBq+vpxxJ22hgHspjBy5+HTPPctpIOA82pK05/rCE +BZXzfRvG869TrInsyA/z+G19jUpYig1Lp25yhy3hWGCyFZdbBC2u1ZkmS75zJ2xB +OSvYHOJJf5jv/WsAsps5TgIhEIq0naN5oj5ziRxdcH4BFq4OWoHv4JLgH7COV9ew +6gLZtaBk7HJmceiVveEowt8GzAvBirQNGOM0wOpfzFLpXUCChOsLGXUOo8vDjuiK +aZtIpni+ii3rRxIFhUrio5fk2rDlGarAQtv0hvhA9UK30eNthCnvsRw6ieSATiV9 +hpu+854z9Tq921mEjYImDS2TK24nj4dvuLudErZJEymma0/vIMNekwagBcrZtVmK +lkFFy3SifEuJ8fm92tQ1s6Vci2NYtAx+IeUN4bjyRtdspok+/e1G9TQaLHZfxIk6 +VE17M03Pylx7QFuDHdihCGtFNp3gILpwilGOBO3+nDDcGgXuMMuXpE0jVxd/Li62 +RKUyoBBXrHRqKD0t2DVUUzagJoTK/PHQFYOb+OZtlzJWLOVv65BD+G41IldP76kK +Y80bus/p5g1OHv7sMNa8oFBAVY49pV/C+uHO9vurYHYGZzfdsNd5G4XgwkTqzIBh +QJXZm3Drm4jzrg9EMgo7iu7VRCDHmwIDAQABAoICAQCebjlupiu7jB+Vvq0VyAYe +K66MGAbhpttcixu6qPN5nF5u5xIKpaxcfMVfgO/B8X0g1pWAT7m7pkSDTzFCL92s +dPApScOeZyfEolbZKkiRoOAb4yzE/PJkCfDhnIFPntWebXTn3SGFxjcohCGq7+w2 +CRbphc6k2dGhG2wpPK0YpfBuM94RVn7sLQ+rI3724s7VKzPW9pUPHJ4QD7j2JhRh +ymGdl29mc9GjEL38xnNoXgCDXFMypZSsii+aPzAAdS+zpu2b+czBmp3eXHc2h1Tl +5B2Arn/Jv63I1wcZf7MmOS1DzlDU2WBbFYbGsVW+RvYD/rFIiDEfhlWNhlLttQFw +TJ9xk+EePK9VQuWzN5tG1lEjGcNWtPLUp3IxZTqaei5lWu6zyA6HVsxjyArzmfNk +x0fRpZU+VZYzbkgj0ROq3wg7QEEMQ8SPo9vvLF1ZNnndzs/ziPA1CodUuSwa6B2c +Zeref4s0B//q3U1SDQE08OD9iuZODwtkO4wQtW2DP33gC6VIts94jg87z8SRDp2g +DcT3D8ZhV5B2VPelluQZ/scWKGWKAvPVRjq51EiMeZtFBVyM6+o0xW2+MxxZdjbj +OWexc+dw8QfwIlFRm0v8Tfvljk1prqYEMLV4s9JD8up5X1h3Yg5uAsQpdZ+1JkGm +5UvvQQVQgxkC1NFXxqYyQQKCAQEA95r3oYm+bnQXHXX4dBQO98+XwOOfnhMk815p +/CAuxCzbPNafjqyAxRgmp5D/IkdLzbitDL3uo73ot1RwB4CEN/Ovp3a/+CklBnnA +0bKAtsGE2XWaqFeguVIy25WEWKaTxKGX0g6KHkOvGt0DNo4wUJUk+2sAqIvXU1Q6 +tUbd+8YRYxO7i6+92K7kxoZega6qiA/L3akZ2uTzFf+IskfqmDUoF2ZaEOFluG8E +ASX3KoVFfraV3DBEN0ionvfpaRIidr2IsuC848zHFBtAXA0mL55BCuf++HmAZnpy +HFN7owVVgqbEw+GGbNdRLt5zV00DmX/sHsIZU/gCLRPsfPUAqQKCAQEA0ElWWiS4 +IA91lWbzCwswFrHvjpcogh67fNd9kJCcFCUHabev7SSrIEcjqDP2m6HPvp/FwxYA +PEo1/vDZ884v9roft2J13OvpvXoqtRZLGo1E76sECBrcto4nhCiTeRQg9uRpHG+Q +p77QC/4eRBLGykFRJET6913x7JzpjAO0QLLLzilj1yBkbF5U01Up5KbIuNeXNvEO +GVGpbryIXxwR6Qhyv7C54xpjRdu9EOT1frRqdIs0qOGafnLXWAXKfvWUzz1wSiiw +1p7xqYZrawXAr7XEkGA2aeqt/iqo2X2G9oYA0apJVwfR4WhuS2hPkSy405bsrGzZ +cjMs9bnJSYP8owKCAQEAxCTSvfisDjuQhBoL84hgQxcEFB09OK/ZuaC1PLER2v3d +vtgWFaO5bmivVlaahcEM367IByv+e1/ZlkEhbg/0rY4xO+vqLuAJIJQalwNcy2mJ +n+p11Z11CNmAyEotSTzMGhwYdKJn74mWkSU7gmApDezYGwKsxtfgf3Zd+3RkLSq+ +Y0oia4mQTrJdMJcJDpobJSW+TZ3DiY+MsYR3+SLXSDPzynWeK3kiZ3QqK+6zWc+x +OavSE1d48oJwcV3aXQ2sl3uVan51o894dQkRdtpDwb0PsWAOry8w8/1Tn/TSIFX9 +Yz5Q6Qsivd3jxckafbHYhCS+G6+O+OGid6ssz+AV4QKCAQAqK78ND0QsUZT4A9kP +kltRLQOYtiggeEJzm1mz7GN9fKXMlMFM3VC8f0rL4oF6rz9VlBnBTvILQuc9z9wB +De0OIk8LnSbJ7QXtNA/zjCj2nkWn1NNDJNUtLQj5LBH3wMiP1F0nwbrjC7Ipy3Cr +TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T +e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP +3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 +WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa +8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb +esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn +PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6 +PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5 +7qINKoGovqXB1oAdopIl1z64e7MWVE4= +-----END PRIVATE KEY----- diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 62482dd50..9fbd08c98 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -104,8 +104,6 @@ chalice: ## Changes to nginx # # nginx-ingress: -# sslKey: site.crt -# sslCert: site.crt # customServerConfigs: | # # Redirecting http to https # return 301 https://$host$request_uri; From 7b4adc92bd57c3ef3ffda3a4c8edad7390ad7af2 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 22 Jan 2022 21:31:12 +0530 Subject: [PATCH 24/33] chore(helm): create vars_template for variable migration --- scripts/helmcharts/vars_template.yaml | 111 ++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 scripts/helmcharts/vars_template.yaml diff --git a/scripts/helmcharts/vars_template.yaml b/scripts/helmcharts/vars_template.yaml new file mode 100644 index 000000000..84ce36ee6 --- /dev/null +++ b/scripts/helmcharts/vars_template.yaml @@ -0,0 +1,111 @@ +fromVersion: "{{ openreplay_version }}" +# Databases specific variables +postgresql: &postgres + # For generating passwords + # `openssl rand -hex 20` + postgresqlPassword: "{{ postgres_db_password }}" + postgresqlHost: "{{ postgres_endpoint }}" + postgresqlPort: "5432" + postgresqlUser: "{{ postgres_db_user }}" + postgresqlDatabase: "{{ postgres_db_name }}" + # resources: + # requests: + # memory: 256Mi + # cpu: 250m + # limits: + # memory: 3000Mi + # cpu: 2 + +clickhouse: {} + # For enterpriseEdition + # enabled: true + +kafka: &kafka + # For enterpriseEdition + # enabled: true + + kafkaHost: "{{ kafka_endpoint }}" + kafkaPort: "{{ kafka_endpoint.split(':')[-1] }}" + kafkaUseSsl: "{{ kafka_ssl }}" + +redis: &redis + # For enterpriseEdition + # enabled: false + redisHost: "{{ redis_endpoint }}" + redisPort: "{{ redis_endpoint.split(':')[-1] }}" + +minio: + global: + minio: + # For generating passwords + # `openssl rand -hex 20` + accessKey: "{{ minio_access_key }}" + secretKey: "{{ minio_secret_key }}" + +# Application specific variables +global: + postgresql: *postgres + kafka: *kafka + redis: *redis + s3: + region: "us-east-1" + endpoint: "http://minio.db.svc.cluster.local:9000" + assetsBucket: "sessions-assets" + recordingsBucket: "mobs" + sourcemapsBucket: "sourcemaps" + # if you're using one node installation, where + # you're using local s3, make sure these variables + # are same as minio.global.minio.accesskey and secretKey + accessKey: "{{ minio_access_key }}" + secretKey: "{{ minio_secret_key }}" + email: + emailHost: '{{ email_host }}' + emailPort: '{{ email_port }}' + emailUser: '{{ email_user }}' + emailPassword: '{{ email_password }}' + emailUseTls: '{{ email_use_tls }}' + emailUseSsl: '{{ email_use_ssl }}' + emailSslKey: '{{ email_ssl_key }}' + emailSslCert: '{{ email_ssl_cert }}' + emailFrom: '{{ email_from }}' + + enterpriseEditionLicense: "{{ enterprise_edition_license }}" + domainName: "{{ domain_name }}" + +chalice: + env: + jwt_secret: "{{ jwt_secret_key }}" + # captcha_server: '' + # captcha_key: '' + # SAML2_MD_URL: '' + # idp_entityId: '' + # idp_sso_url: '' + # idp_x509cert: '' + # idp_sls_url: '' + # idp_name: '' + # idp_tenantKey: '' + + +# If you want to override something +# chartname: +# filedFrom chart/Values.yaml: +# key: value + +# For example: +# +# http: +# resources: +# limits: +# cpu: 1024m +# memory: 4096Mi +# requests: +# cpu: 512m +# memory: 2056Mi + +## Changes to nginx +# +# nginx-ingress: +# customServerConfigs: | +# # Redirecting http to https +# return 301 https://$host$request_uri; +# From a2a34e90f6bc4b1a0ae805b41d51f9fc2dc43e0f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 18:20:16 +0100 Subject: [PATCH 25/33] feat(api): flat funnels --- api/chalicelib/core/funnels.py | 3 ++- api/chalicelib/core/significance.py | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index b9db6c631..c33bed586 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -103,6 +103,7 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date project_id=project_id, user_id=None, count_only=True) row["sessionsCount"] = counts["countSessions"] row["usersCount"] = counts["countUsers"] + filter_clone = dict(row["filter"]) overview = significance.get_overview(filter_d=row["filter"], project_id=project_id) row["stages"] = overview["stages"] row.pop("filter") @@ -111,7 +112,7 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date row["criticalIssuesCount"] = overview["criticalIssuesCount"] row["missedConversions"] = 0 if len(row["stages"]) < 2 \ else row["stages"][0]["sessionsCount"] - row["stages"][-1]["sessionsCount"] - row["filter"] = helper.old_search_payload_to_flat(row["filter"]) + row["filter"] = helper.old_search_payload_to_flat(filter_clone) return rows diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index d81378ddb..f2261ce59 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -2,7 +2,7 @@ __author__ = "AZNAUROV David" __maintainer__ = "KRAIEM Taha Yassine" import schemas -from chalicelib.core import events, sessions_metas, metadata, sessions +from chalicelib.core import events, metadata, sessions from chalicelib.utils import dev """ @@ -617,6 +617,15 @@ def get_overview(filter_d, project_id, first_stage=None, last_stage=None): # The result of the multi-stage query rows = get_stages_and_events(filter_d=filter_d, project_id=project_id) if len(rows) == 0: + # PS: not sure what to return if rows are empty + output["stages"] = [{ + "type": stages[0]["type"], + "value": stages[0]["value"], + "sessionsCount": None, + "dropPercentage": None, + "usersCount": None + }] + output['criticalIssuesCount'] = 0 return output # Obtain the first part of the output stages_list = get_stages(stages, rows) From b8caca0223c172288c42589a4d0f8ef4f5c8456b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 19:56:32 +0100 Subject: [PATCH 26/33] feat(api): flat saved_search --- api/chalicelib/core/saved_search.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/chalicelib/core/saved_search.py b/api/chalicelib/core/saved_search.py index dfa9a1dcf..732fc1596 100644 --- a/api/chalicelib/core/saved_search.py +++ b/api/chalicelib/core/saved_search.py @@ -18,6 +18,7 @@ def create(project_id, user_id, data: schemas.SavedSearchSchema): ) r = cur.fetchone() r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r["filter"] = helper.old_search_payload_to_flat(r["filter"]) r = helper.dict_to_camel_case(r) return {"data": r} @@ -40,6 +41,7 @@ def update(search_id, project_id, user_id, data: schemas.SavedSearchSchema): ) r = cur.fetchone() r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r["filter"] = helper.old_search_payload_to_flat(r["filter"]) r = helper.dict_to_camel_case(r) # r["filter"]["startDate"], r["filter"]["endDate"] = TimeUTC.get_start_end_from_range(r["filter"]["rangeValue"]) return r @@ -74,6 +76,8 @@ def get_all(project_id, user_id, details=False): rows = helper.list_to_camel_case(rows) for row in rows: row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"]) + if details: + row["filter"] = helper.old_search_payload_to_flat(row["filter"]) return rows @@ -112,4 +116,5 @@ def get(search_id, project_id, user_id): return None f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) + f["filter"] = helper.old_search_payload_to_flat(f["filter"]) return f From 1b62d4dd7497dee729da0ebcd364aecaeb5fd383 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Sun, 23 Jan 2022 11:46:43 +0100 Subject: [PATCH 27/33] Delete README.md --- scripts/helmcharts/README.md | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 scripts/helmcharts/README.md diff --git a/scripts/helmcharts/README.md b/scripts/helmcharts/README.md deleted file mode 100644 index b54ca9652..000000000 --- a/scripts/helmcharts/README.md +++ /dev/null @@ -1,7 +0,0 @@ -- Initialize databases - - we've to pass the --wait flag, else the db installation won't be complete. and it'll break the db init. - -## Installation -helm upgrade --install databases ./databases -n db --create-namespace --wait -f ./values.yaml --atomic - -helm upgrade --install openreplay ./openreplay -n app --create-namespace --wait -f ./values.yaml --atomic From 5e82278b4d6fd4a68aba88ba16e6cb48510b424e Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Sun, 23 Jan 2022 11:56:47 +0100 Subject: [PATCH 28/33] Update vars.yaml --- scripts/helmcharts/vars.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 9fbd08c98..d6b403408 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -89,9 +89,8 @@ chalice: # chartname: # filedFrom chart/Values.yaml: # key: value - -# For example: # +# For example (http): # http: # resources: # limits: @@ -104,6 +103,10 @@ chalice: ## Changes to nginx # # nginx-ingress: +# # Key and certificate files must be named site.key and site.crt +# # and copied to ../openreplay/files/ +# sslKey: site.key +# sslCert: site.crt # customServerConfigs: | # # Redirecting http to https # return 301 https://$host$request_uri; From 7196b819791f138f7b37434a45727c12e425a601 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 23 Jan 2022 17:52:58 +0100 Subject: [PATCH 29/33] feat(tracker-assist): 3.4.16: improve scroll, ignore Fetch, peerjs logs --- tracker/tracker-assist/package.json | 2 +- tracker/tracker-assist/src/BufferingConnection.ts | 6 +++++- tracker/tracker-assist/src/Mouse.ts | 5 +++-- tracker/tracker-assist/src/index.ts | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 506f05e2d..4d327410b 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "3.4.13", + "version": "3.4.16", "keywords": [ "WebRTC", "assistance", diff --git a/tracker/tracker-assist/src/BufferingConnection.ts b/tracker/tracker-assist/src/BufferingConnection.ts index e90970c21..5fb3b7349 100644 --- a/tracker/tracker-assist/src/BufferingConnection.ts +++ b/tracker/tracker-assist/src/BufferingConnection.ts @@ -6,12 +6,13 @@ interface Message { } // 16kb should be max according to specification +// 64kb chrome const crOrFf: boolean = typeof navigator !== "undefined" && (navigator.userAgent.indexOf("Chrom") !== -1 || // Chrome && Chromium navigator.userAgent.indexOf("Firefox") !== -1); -const MESSAGES_PER_SEND = crOrFf ? 500 : 100 +const MESSAGES_PER_SEND = crOrFf ? 200 : 50 // Bffering required in case of webRTC export default class BufferingConnection { @@ -34,7 +35,10 @@ export default class BufferingConnection { send(messages: Message[]) { if (!this.conn.open) { return; } let i = 0; + //@ts-ignore + messages=messages.filter(m => m._id !== 39) while (i < messages.length) { + this.buffer.push(messages.slice(i, i+=this.msgsPerSend)) } if (!this.buffering) { diff --git a/tracker/tracker-assist/src/Mouse.ts b/tracker/tracker-assist/src/Mouse.ts index d2c89cfe6..b183413bd 100644 --- a/tracker/tracker-assist/src/Mouse.ts +++ b/tracker/tracker-assist/src/Mouse.ts @@ -47,11 +47,12 @@ export default class Mouse { } } - private readonly pScrEl = document.scrollingElement || document.documentElement + private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct private lastScrEl: Element | "window" | null = null private resetLastScrEl = () => { this.lastScrEl = null } private handleWScroll = e => { - if (e.target !== this.lastScrEl) { + if (e.target !== this.lastScrEl && + this.lastScrEl !== "window") { this.resetLastScrEl() } } diff --git a/tracker/tracker-assist/src/index.ts b/tracker/tracker-assist/src/index.ts index 74514378a..d2067ff91 100644 --- a/tracker/tracker-assist/src/index.ts +++ b/tracker/tracker-assist/src/index.ts @@ -27,7 +27,7 @@ enum CallingState { }; //@ts-ignore peerjs hack for webpack5 (?!) TODO: ES/node modules; -//Peer = Peer.default || Peer; +Peer = Peer.default || Peer; // type IncomeMessages = // "call_end" | @@ -86,7 +86,7 @@ export default function(opts?: Partial) { host: app.getHost(), path: '/assist', port: location.protocol === 'http:' && appOptions.__DISABLE_SECURE_MODE ? 80 : 443, - //debug: // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs. + debug: appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs. } if (options.config) { _opt['config'] = options.config From b3fba224cc5d248ceaaf6364dc8ae1e7e0285f65 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 23 Jan 2022 18:28:22 +0100 Subject: [PATCH 30/33] frat(tracker): 3.4.16: observer for shRoot, topObserver & sanitizer refactor; 3.4.17:autoResetOnWindowOpen; userID onStart option; restart fix --- tracker/tracker/package-lock.json | 2 +- tracker/tracker/package.json | 2 +- tracker/tracker/src/main/app/context.ts | 72 +++ tracker/tracker/src/main/app/index.ts | 246 +++++---- tracker/tracker/src/main/app/observer.ts | 484 ------------------ .../src/main/app/observer/iframe_observer.ts | 19 + .../tracker/src/main/app/observer/observer.ts | 353 +++++++++++++ .../main/app/observer/shadow_root_observer.ts | 18 + .../src/main/app/observer/top_observer.ts | 98 ++++ tracker/tracker/src/main/app/sanitizer.ts | 66 +++ tracker/tracker/src/main/index.ts | 65 ++- tracker/tracker/src/main/modules/exception.ts | 8 +- tracker/tracker/src/main/modules/img.ts | 19 +- tracker/tracker/src/main/modules/input.ts | 20 +- tracker/tracker/src/main/modules/mouse.ts | 2 +- tracker/tracker/src/webworker/index.ts | 1 + 16 files changed, 853 insertions(+), 622 deletions(-) create mode 100644 tracker/tracker/src/main/app/context.ts delete mode 100644 tracker/tracker/src/main/app/observer.ts create mode 100644 tracker/tracker/src/main/app/observer/iframe_observer.ts create mode 100644 tracker/tracker/src/main/app/observer/observer.ts create mode 100644 tracker/tracker/src/main/app/observer/shadow_root_observer.ts create mode 100644 tracker/tracker/src/main/app/observer/top_observer.ts create mode 100644 tracker/tracker/src/main/app/sanitizer.ts diff --git a/tracker/tracker/package-lock.json b/tracker/tracker/package-lock.json index 287203b30..6dcbc4e81 100644 --- a/tracker/tracker/package-lock.json +++ b/tracker/tracker/package-lock.json @@ -1,6 +1,6 @@ { "name": "@openreplay/tracker", - "version": "3.4.7", + "version": "3.4.12", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index a2dbc60f1..16b10b8f4 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "3.4.12", + "version": "3.4.17", "keywords": [ "logging", "replay" diff --git a/tracker/tracker/src/main/app/context.ts b/tracker/tracker/src/main/app/context.ts new file mode 100644 index 000000000..aa9a5dfb3 --- /dev/null +++ b/tracker/tracker/src/main/app/context.ts @@ -0,0 +1,72 @@ +// TODO: global type +export interface Window extends globalThis.Window { + HTMLInputElement: typeof HTMLInputElement, + HTMLLinkElement: typeof HTMLLinkElement, + HTMLStyleElement: typeof HTMLStyleElement, + SVGStyleElement: typeof SVGStyleElement, + HTMLIFrameElement: typeof HTMLIFrameElement, + Text: typeof Text, + Element: typeof Element, + ShadowRoot: typeof ShadowRoot, + //parent: Window, +} + +type WindowConstructor = + Document | + Element | + Text | + ShadowRoot | + HTMLInputElement | + HTMLLinkElement | + HTMLStyleElement | + HTMLIFrameElement + +// type ConstructorNames = +// 'Element' | +// 'Text' | +// 'HTMLInputElement' | +// 'HTMLLinkElement' | +// 'HTMLStyleElement' | +// 'HTMLIFrameElement' +type Constructor = { new (...args: any[]): T , name: string }; + + // TODO: we need a type expert here so we won't have to ignore the lines + // TODO: use it everywhere (static function; export from which file? <-- global Window typing required) +export function isInstance(node: Node, constr: Constructor): node is T { + const doc = node.ownerDocument; + if (!doc) { // null if Document + return constr.name === 'Document'; + } + let context: Window = + // @ts-ignore (for EI, Safary) + doc.parentWindow || + doc.defaultView; // TODO: smart global typing for Window object + while(context.parent && context.parent !== context) { + // @ts-ignore + if (node instanceof context[constr.name]) { + return true + } + // @ts-ignore + context = context.parent + } + // @ts-ignore + return node instanceof context[constr.name] +} + +export function inDocument(node: Node): boolean { + const doc = node.ownerDocument + if (!doc) { return false } + if (doc.contains(node)) { return true } + let context: Window = + // @ts-ignore (for EI, Safary) + doc.parentWindow || + doc.defaultView; + while(context.parent && context.parent !== context) { + if (context.document.contains(node)) { + return true + } + // @ts-ignore + context = context.parent + } + return false; +} diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 54fe9050f..b02d15f91 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -2,12 +2,15 @@ import { timestamp, log, warn } from "../utils.js"; import { Timestamp, PageClose } from "../../messages/index.js"; import Message from "../../messages/message.js"; import Nodes from "./nodes.js"; -import Observer from "./observer.js"; +import Observer from "./observer/top_observer.js"; +import Sanitizer from "./sanitizer.js"; import Ticker from "./ticker.js"; import { deviceMemory, jsHeapSizeLimit } from "../modules/performance.js"; -import type { Options as ObserverOptions } from "./observer.js"; +import type { Options as ObserverOptions } from "./observer/top_observer.js"; +import type { Options as SanitizerOptions } from "./sanitizer.js"; + import type { Options as WebworkerOptions, WorkerMessageData } from "../../messages/webworker.js"; @@ -17,11 +20,17 @@ export interface OnStartInfo { userUUID: string, } -export type Options = { +export interface StartOptions { + userID?: string, + forceNew: boolean, +} + +type AppOptions = { revID: string; node_id: string; session_token_key: string; session_pageno_key: string; + session_reset_key: string; local_uuid_key: string; ingestPoint: string; resourceBaseHref: string | null, // resourceHref? @@ -30,7 +39,9 @@ export type Options = { __debug_report_edp: string | null; __debug_log: boolean; onStart?: (info: OnStartInfo) => void; -} & ObserverOptions & WebworkerOptions; +} & WebworkerOptions; + +export type Options = AppOptions & ObserverOptions & SanitizerOptions type Callback = () => void; type CommitCallback = (messages: Array) => void; @@ -43,21 +54,23 @@ export default class App { readonly nodes: Nodes; readonly ticker: Ticker; readonly projectKey: string; + readonly sanitizer: Sanitizer; private readonly messages: Array = []; - /*private*/ readonly observer: Observer; // temp, for fast security fix. TODO: separate security/obscure module with nodeCallback that incapsulates `textMasked` functionality from Observer + private readonly observer: Observer; private readonly startCallbacks: Array = []; private readonly stopCallbacks: Array = []; private readonly commitCallbacks: Array = []; - private readonly options: Options; + private readonly options: AppOptions; private readonly revID: string; private _sessionID: string | null = null; + private _userID: string | undefined; private isActive = false; private version = 'TRACKER_VERSION'; private readonly worker?: Worker; constructor( projectKey: string, sessionToken: string | null | undefined, - opts: Partial, + options: Partial, ) { this.projectKey = projectKey; this.options = Object.assign( @@ -66,24 +79,23 @@ export default class App { node_id: '__openreplay_id', session_token_key: '__openreplay_token', session_pageno_key: '__openreplay_pageno', + session_reset_key: '__openreplay_reset', local_uuid_key: '__openreplay_uuid', ingestPoint: DEFAULT_INGEST_POINT, resourceBaseHref: null, __is_snippet: false, __debug_report_edp: null, __debug_log: false, - obscureTextEmails: true, - obscureTextNumbers: false, - captureIFrames: false, }, - opts, + options, ); if (sessionToken != null) { sessionStorage.setItem(this.options.session_token_key, sessionToken); } this.revID = this.options.revID; + this.sanitizer = new Sanitizer(this, options); this.nodes = new Nodes(this.options.node_id); - this.observer = new Observer(this, this.options); + this.observer = new Observer(this, options); this.ticker = new Ticker(this); this.ticker.attach(() => this.commit()); try { @@ -102,7 +114,10 @@ export default class App { this.stop(); } else if (data === "restart") { this.stop(); - this.start(true); + this.start({ + forceNew: true, + userID: this._userID, + }); } }; const alertWorker = () => { @@ -244,104 +259,132 @@ export default class App { active(): boolean { return this.isActive; } - private _start(reset: boolean): Promise { - if (!this.isActive) { - if (!this.worker) { - return Promise.reject("No worker found: perhaps, CSP is not set."); - } - this.isActive = true; - let pageNo: number = 0; - const pageNoStr = sessionStorage.getItem(this.options.session_pageno_key); - if (pageNoStr != null) { - pageNo = parseInt(pageNoStr); - pageNo++; - } - sessionStorage.setItem(this.options.session_pageno_key, pageNo.toString()); - const startTimestamp = timestamp(); - - const messageData: WorkerMessageData = { - ingestPoint: this.options.ingestPoint, - pageNo, - startTimestamp, - connAttemptCount: this.options.connAttemptCount, - connAttemptGap: this.options.connAttemptGap, - } - this.worker.postMessage(messageData); // brings delay of 10th ms? - return window.fetch(this.options.ingestPoint + '/v1/web/start', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - token: sessionStorage.getItem(this.options.session_token_key), - userUUID: localStorage.getItem(this.options.local_uuid_key), - projectKey: this.projectKey, - revID: this.revID, - timestamp: startTimestamp, - trackerVersion: this.version, - isSnippet: this.options.__is_snippet, - deviceMemory, - jsHeapSizeLimit, - reset, - }), - }) - .then(r => { - if (r.status === 200) { - return r.json() - } else { // TODO: handle canceling && 403 - return r.text().then(text => { - throw new Error(`Server error: ${r.status}. ${text}`); - }); - } - }) - .then(r => { - const { token, userUUID, sessionID, beaconSizeLimit } = r; - if (typeof token !== 'string' || - typeof userUUID !== 'string' || - (typeof beaconSizeLimit !== 'number' && typeof beaconSizeLimit !== 'undefined')) { - throw new Error(`Incorrect server response: ${ JSON.stringify(r) }`); - } - sessionStorage.setItem(this.options.session_token_key, token); - localStorage.setItem(this.options.local_uuid_key, userUUID); - if (typeof sessionID === 'string') { - this._sessionID = sessionID; - } - if (!this.worker) { - throw new Error("no worker found after start request (this might not happen)"); - } - this.worker.postMessage({ token, beaconSizeLimit }); - this.startCallbacks.forEach((cb) => cb()); - this.observer.observe(); - this.ticker.start(); - - log("OpenReplay tracking started."); - const onStartInfo = { sessionToken: token, userUUID, sessionID }; - if (typeof this.options.onStart === 'function') { - this.options.onStart(onStartInfo); - } - return onStartInfo; - }) - .catch(e => { - sessionStorage.removeItem(this.options.session_token_key) - this.stop() - warn("OpenReplay was unable to start. ", e) - this._debug("session_start", e); - throw e - }) + resetNextPageSession(flag: boolean) { + if (flag) { + sessionStorage.setItem(this.options.session_reset_key, 't'); + } else { + sessionStorage.removeItem(this.options.session_reset_key); } - return Promise.reject("Player is already active"); + } + private _start(startOpts: StartOptions): Promise { + if (!this.worker) { + return Promise.reject("No worker found: perhaps, CSP is not set."); + } + if (this.isActive) { + return Promise.reject("OpenReplay: trying to call `start()` on the instance that has been started already.") + } + this.isActive = true; + + let pageNo: number = 0; + const pageNoStr = sessionStorage.getItem(this.options.session_pageno_key); + if (pageNoStr != null) { + pageNo = parseInt(pageNoStr); + pageNo++; + } + sessionStorage.setItem(this.options.session_pageno_key, pageNo.toString()); + const startTimestamp = timestamp(); + + const messageData: WorkerMessageData = { + ingestPoint: this.options.ingestPoint, + pageNo, + startTimestamp, + connAttemptCount: this.options.connAttemptCount, + connAttemptGap: this.options.connAttemptGap, + } + this.worker.postMessage(messageData); // brings delay of 10th ms? + + + // let token = sessionStorage.getItem(this.options.session_token_key) + // const tokenIsActive = localStorage.getItem("__or_at_" + token) + // if (tokenIsActive) { + // token = null + // } + + const sReset = sessionStorage.getItem(this.options.session_reset_key); + sessionStorage.removeItem(this.options.session_reset_key); + + this._userID = startOpts.userID || undefined + return window.fetch(this.options.ingestPoint + '/v1/web/start', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + token: sessionStorage.getItem(this.options.session_token_key), + userUUID: localStorage.getItem(this.options.local_uuid_key), + projectKey: this.projectKey, + revID: this.revID, + timestamp: startTimestamp, + trackerVersion: this.version, + isSnippet: this.options.__is_snippet, + deviceMemory, + jsHeapSizeLimit, + reset: startOpts.forceNew || sReset !== null, + userID: this._userID, + }), + }) + .then(r => { + if (r.status === 200) { + return r.json() + } else { // TODO: handle canceling && 403 + return r.text().then(text => { + throw new Error(`Server error: ${r.status}. ${text}`); + }); + } + }) + .then(r => { + const { token, userUUID, sessionID, beaconSizeLimit } = r; + if (typeof token !== 'string' || + typeof userUUID !== 'string' || + (typeof beaconSizeLimit !== 'number' && typeof beaconSizeLimit !== 'undefined')) { + throw new Error(`Incorrect server response: ${ JSON.stringify(r) }`); + } + sessionStorage.setItem(this.options.session_token_key, token); + localStorage.setItem(this.options.local_uuid_key, userUUID); + // localStorage.setItem("__or_at_" + token, "true") + // this.attachEventListener(window, 'beforeunload', ()=>{ + // localStorage.removeItem("__or_at_" + token) + // }, false); + // this.attachEventListener(window, 'pagehide', ()=>{ + // localStorage.removeItem("__or_at_" + token) + // }, false); + if (typeof sessionID === 'string') { + this._sessionID = sessionID; + } + if (!this.worker) { + throw new Error("no worker found after start request (this might not happen)"); + } + this.worker.postMessage({ token, beaconSizeLimit }); + this.startCallbacks.forEach((cb) => cb()); + this.observer.observe(); + this.ticker.start(); + + log("OpenReplay tracking started."); + const onStartInfo = { sessionToken: token, userUUID, sessionID }; + if (typeof this.options.onStart === 'function') { + this.options.onStart(onStartInfo); + } + return onStartInfo; + }) + .catch(e => { + sessionStorage.removeItem(this.options.session_token_key) + this.stop() + warn("OpenReplay was unable to start. ", e) + this._debug("session_start", e); + throw e + }) } - start(reset: boolean = false): Promise { + start(options: StartOptions = { forceNew: false }): Promise { if (!document.hidden) { - return this._start(reset); + return this._start(options); } else { return new Promise((resolve) => { const onVisibilityChange = () => { if (!document.hidden) { document.removeEventListener("visibilitychange", onVisibilityChange); - resolve(this._start(reset)); + resolve(this._start(options)); } } document.addEventListener("visibilitychange", onVisibilityChange); @@ -354,6 +397,7 @@ export default class App { if (this.worker) { this.worker.postMessage("stop"); } + this.sanitizer.clear(); this.observer.disconnect(); this.nodes.clear(); this.ticker.stop(); diff --git a/tracker/tracker/src/main/app/observer.ts b/tracker/tracker/src/main/app/observer.ts deleted file mode 100644 index 3ed5088af..000000000 --- a/tracker/tracker/src/main/app/observer.ts +++ /dev/null @@ -1,484 +0,0 @@ -import { stars, hasOpenreplayAttribute } from "../utils.js"; -import { - CreateDocument, - CreateElementNode, - CreateTextNode, - SetNodeData, - SetCSSDataURLBased, - SetNodeAttribute, - SetNodeAttributeURLBased, - RemoveNodeAttribute, - MoveNode, - RemoveNode, - CreateIFrameDocument, -} from "../../messages/index.js"; -import App from "./index.js"; - -interface Window extends WindowProxy { - HTMLInputElement: typeof HTMLInputElement, - HTMLLinkElement: typeof HTMLLinkElement, - HTMLStyleElement: typeof HTMLStyleElement, - SVGStyleElement: typeof SVGStyleElement, - HTMLIFrameElement: typeof HTMLIFrameElement, - Text: typeof Text, - Element: typeof Element, - //parent: Window, -} - - -type WindowConstructor = - Document | - Element | - Text | - HTMLInputElement | - HTMLLinkElement | - HTMLStyleElement | - HTMLIFrameElement - -// type ConstructorNames = -// 'Element' | -// 'Text' | -// 'HTMLInputElement' | -// 'HTMLLinkElement' | -// 'HTMLStyleElement' | -// 'HTMLIFrameElement' -type Constructor = { new (...args: any[]): T , name: string }; - - -function isSVGElement(node: Element): node is SVGElement { - return node.namespaceURI === 'http://www.w3.org/2000/svg'; -} - -export interface Options { - obscureTextEmails: boolean; - obscureTextNumbers: boolean; - captureIFrames: boolean; -} - -export default class Observer { - private readonly observer: MutationObserver; - private readonly commited: Array; - private readonly recents: Array; - private readonly indexes: Array; - private readonly attributesList: Array | undefined>; - private readonly textSet: Set; - private readonly textMasked: Set; - constructor(private readonly app: App, private readonly options: Options, private readonly context: Window = window) { - this.observer = new MutationObserver( - this.app.safe((mutations) => { - for (const mutation of mutations) { - const target = mutation.target; - const type = mutation.type; - - // Special case - // Document 'childList' might happen in case of iframe. - // TODO: generalize as much as possible - if (this.isInstance(target, Document) - && type === 'childList' - //&& new Array(mutation.addedNodes).some(node => this.isInstance(node, HTMLHtmlElement)) - ) { - const parentFrame = target.defaultView?.frameElement - if (!parentFrame) { continue } - this.bindTree(target.documentElement) - const frameID = this.app.nodes.getID(parentFrame) - const docID = this.app.nodes.getID(target.documentElement) - if (frameID === undefined || docID === undefined) { continue } - this.app.send(CreateIFrameDocument(frameID, docID)); - continue; - } - - if (this.isIgnored(target) || !context.document.contains(target)) { - continue; - } - if (type === 'childList') { - for (let i = 0; i < mutation.removedNodes.length; i++) { - this.bindTree(mutation.removedNodes[i]); - } - for (let i = 0; i < mutation.addedNodes.length; i++) { - this.bindTree(mutation.addedNodes[i]); - } - continue; - } - const id = this.app.nodes.getID(target); - if (id === undefined) { - continue; - } - if (id >= this.recents.length) { - this.recents[id] = undefined; - } - if (type === 'attributes') { - const name = mutation.attributeName; - if (name === null) { - continue; - } - let attr = this.attributesList[id]; - if (attr === undefined) { - this.attributesList[id] = attr = new Set(); - } - attr.add(name); - continue; - } - if (type === 'characterData') { - this.textSet.add(id); - continue; - } - } - this.commitNodes(); - }), - ); - this.commited = []; - this.recents = []; - this.indexes = [0]; - this.attributesList = []; - this.textSet = new Set(); - this.textMasked = new Set(); - } - private clear(): void { - this.commited.length = 0; - this.recents.length = 0; - this.indexes.length = 1; - this.attributesList.length = 0; - this.textSet.clear(); - this.textMasked.clear(); - } - - // TODO: we need a type expert here so we won't have to ignore the lines - private isInstance(node: Node, constr: Constructor): node is T { - let context = this.context; - while(context.parent && context.parent !== context) { - // @ts-ignore - if (node instanceof context[constr.name]) { - return true - } - // @ts-ignore - context = context.parent - } - // @ts-ignore - return node instanceof context[constr.name] - } - - private isIgnored(node: Node): boolean { - if (this.isInstance(node, Text)) { - return false; - } - if (!this.isInstance(node, Element)) { - return true; - } - const tag = node.tagName.toUpperCase(); - if (tag === 'LINK') { - const rel = node.getAttribute('rel'); - const as = node.getAttribute('as'); - return !(rel?.includes('stylesheet') || as === "style" || as === "font"); - } - return ( - tag === 'SCRIPT' || - tag === 'NOSCRIPT' || - tag === 'META' || - tag === 'TITLE' || - tag === 'BASE' - ); - } - - private sendNodeAttribute( - id: number, - node: Element, - name: string, - value: string | null, - ): void { - if (isSVGElement(node)) { - if (name.substr(0, 6) === 'xlink:') { - name = name.substr(6); - } - if (value === null) { - this.app.send(new RemoveNodeAttribute(id, name)); - } else if (name === 'href') { - if (value.length > 1e5) { - value = ''; - } - this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); - } else { - this.app.send(new SetNodeAttribute(id, name, value)); - } - return; - } - if ( - name === 'src' || - name === 'srcset' || - name === 'integrity' || - name === 'crossorigin' || - name === 'autocomplete' || - name.substr(0, 2) === 'on' - ) { - return; - } - if ( - name === 'value' && - this.isInstance(node, HTMLInputElement) && - node.type !== 'button' && - node.type !== 'reset' && - node.type !== 'submit' - ) { - return; - } - if (value === null) { - this.app.send(new RemoveNodeAttribute(id, name)); - return; - } - if (name === 'style' || name === 'href' && this.isInstance(node, HTMLLinkElement)) { - this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); - return; - } - if (name === 'href' || value.length > 1e5) { - value = ''; - } - this.app.send(new SetNodeAttribute(id, name, value)); - } - - /* TODO: abstract sanitation */ - getInnerTextSecure(el: HTMLElement): string { - const id = this.app.nodes.getID(el) - if (!id) { return '' } - return this.checkObscure(id, el.innerText) - - } - - private checkObscure(id: number, data: string): string { - if (this.textMasked.has(id)) { - return data.replace( - /[^\f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/g, - '█', - ); - } - if (this.options.obscureTextNumbers) { - data = data.replace(/\d/g, '0'); - } - if (this.options.obscureTextEmails) { - data = data.replace( - /([^\s]+)@([^\s]+)\.([^\s]+)/g, - (...f: Array) => - stars(f[1]) + '@' + stars(f[2]) + '.' + stars(f[3]), - ); - } - return data - } - - private sendNodeData(id: number, parentElement: Element, data: string): void { - if (this.isInstance(parentElement, HTMLStyleElement) || this.isInstance(parentElement, SVGStyleElement)) { - this.app.send(new SetCSSDataURLBased(id, data, this.app.getBaseHref())); - return; - } - data = this.checkObscure(id, data) - this.app.send(new SetNodeData(id, data)); - } - /* end TODO: abstract sanitation */ - - private bindNode(node: Node): void { - const r = this.app.nodes.registerNode(node); - const id = r[0]; - this.recents[id] = r[1] || this.recents[id] || false; - } - - private bindTree(node: Node): void { - if (this.isIgnored(node)) { - return; - } - this.bindNode(node); - const walker = document.createTreeWalker( - node, - NodeFilter.SHOW_ELEMENT + NodeFilter.SHOW_TEXT, - { - acceptNode: (node) => - this.isIgnored(node) || this.app.nodes.getID(node) !== undefined - ? NodeFilter.FILTER_REJECT - : NodeFilter.FILTER_ACCEPT, - }, - // @ts-ignore - false, - ); - while (walker.nextNode()) { - this.bindNode(walker.currentNode); - } - } - - private unbindNode(node: Node): void { - const id = this.app.nodes.unregisterNode(node); - if (id !== undefined && this.recents[id] === false) { - this.app.send(new RemoveNode(id)); - } - } - - private _commitNode(id: number, node: Node): boolean { - const parent = node.parentNode; - let parentID: number | undefined; - if (this.isInstance(node, HTMLHtmlElement)) { - this.indexes[id] = 0 - } else { - if (parent === null) { - this.unbindNode(node); - return false; - } - parentID = this.app.nodes.getID(parent); - if (parentID === undefined) { - this.unbindNode(node); - return false; - } - if (!this.commitNode(parentID)) { - this.unbindNode(node); - return false; - } - if ( - this.textMasked.has(parentID) || - (this.isInstance(node, Element) && hasOpenreplayAttribute(node, 'masked')) - ) { - this.textMasked.add(id); - } - let sibling = node.previousSibling; - while (sibling !== null) { - const siblingID = this.app.nodes.getID(sibling); - if (siblingID !== undefined) { - this.commitNode(siblingID); - this.indexes[id] = this.indexes[siblingID] + 1; - break; - } - sibling = sibling.previousSibling; - } - if (sibling === null) { - this.indexes[id] = 0; - } - } - const isNew = this.recents[id]; - const index = this.indexes[id]; - if (index === undefined) { - throw 'commitNode: missing node index'; - } - if (isNew === true) { - if (this.isInstance(node, Element)) { - if (parentID !== undefined) { - this.app.send(new - CreateElementNode( - id, - parentID, - index, - node.tagName, - isSVGElement(node), - ), - ); - } - for (let i = 0; i < node.attributes.length; i++) { - const attr = node.attributes[i]; - this.sendNodeAttribute(id, node, attr.nodeName, attr.value); - } - - if (this.isInstance(node, HTMLIFrameElement) && - (this.options.captureIFrames || node.getAttribute("data-openreplay-capture"))) { - this.handleIframe(node); - } - } else if (this.isInstance(node, Text)) { - // for text node id != 0, hence parentID !== undefined and parent is Element - this.app.send(new CreateTextNode(id, parentID as number, index)); - this.sendNodeData(id, parent as Element, node.data); - } - return true; - } - if (isNew === false && parentID !== undefined) { - this.app.send(new MoveNode(id, parentID, index)); - } - const attr = this.attributesList[id]; - if (attr !== undefined) { - if (!this.isInstance(node, Element)) { - throw 'commitNode: node is not an element'; - } - for (const name of attr) { - this.sendNodeAttribute(id, node, name, node.getAttribute(name)); - } - } - if (this.textSet.has(id)) { - if (!this.isInstance(node, Text)) { - throw 'commitNode: node is not a text'; - } - // for text node id != 0, hence parent is Element - this.sendNodeData(id, parent as Element, node.data); - } - return true; - } - private commitNode(id: number): boolean { - const node = this.app.nodes.getNode(id); - if (node === undefined) { - return false; - } - const cmt = this.commited[id]; - if (cmt !== undefined) { - return cmt; - } - return (this.commited[id] = this._commitNode(id, node)); - } - private commitNodes(): void { - let node; - for (let id = 0; id < this.recents.length; id++) { - this.commitNode(id); - if (this.recents[id] === true && (node = this.app.nodes.getNode(id))) { - this.app.nodes.callNodeCallbacks(node); - } - } - this.clear(); - } - - private iframeObservers: Observer[] = []; - private handleIframe(iframe: HTMLIFrameElement): void { - let context: Window | null = null - const handle = this.app.safe(() => { - const id = this.app.nodes.getID(iframe) - if (id === undefined) { return } - if (iframe.contentWindow === context) { return } - context = iframe.contentWindow as Window | null; - if (!context) { return } - const observer = new Observer(this.app, this.options, context) - this.iframeObservers.push(observer) - observer.observeIframe(id, context) - }) - this.app.attachEventListener(iframe, "load", handle) - handle() - } - - // TODO: abstract common functionality, separate FrameObserver - private observeIframe(id: number, context: Window) { - const doc = context.document; - this.observer.observe(doc, { - childList: true, - attributes: true, - characterData: true, - subtree: true, - attributeOldValue: false, - characterDataOldValue: false, - }); - this.bindTree(doc.documentElement); - const docID = this.app.nodes.getID(doc.documentElement); - if (docID === undefined) { - console.log("Wrong") - return; - } - this.app.send(CreateIFrameDocument(id,docID)); - this.commitNodes(); - } - - observe(): void { - this.observer.observe(this.context.document, { - childList: true, - attributes: true, - characterData: true, - subtree: true, - attributeOldValue: false, - characterDataOldValue: false, - }); - this.app.send(new CreateDocument()); - this.bindTree(this.context.document.documentElement); - this.commitNodes(); - } - - disconnect(): void { - this.iframeObservers.forEach(o => o.disconnect()); - this.iframeObservers = []; - this.observer.disconnect(); - this.clear(); - } -} diff --git a/tracker/tracker/src/main/app/observer/iframe_observer.ts b/tracker/tracker/src/main/app/observer/iframe_observer.ts new file mode 100644 index 000000000..be0a7182c --- /dev/null +++ b/tracker/tracker/src/main/app/observer/iframe_observer.ts @@ -0,0 +1,19 @@ +import Observer from "./observer.js"; +import { CreateIFrameDocument } from "../../../messages/index.js"; + +export default class IFrameObserver extends Observer { + observe(iframe: HTMLIFrameElement) { + const doc = iframe.contentDocument; + const hostID = this.app.nodes.getID(iframe); + if (!doc || hostID === undefined) { return } //log TODO common app.logger + // Have to observe document, because the inner might be changed + this.observeRoot(doc, (docID) => { + if (docID === undefined) { + console.log("OpenReplay: Iframe document not bound") + return; + } + this.app.send(CreateIFrameDocument(hostID, docID)); + }); + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/app/observer/observer.ts b/tracker/tracker/src/main/app/observer/observer.ts new file mode 100644 index 000000000..0f4ff2994 --- /dev/null +++ b/tracker/tracker/src/main/app/observer/observer.ts @@ -0,0 +1,353 @@ +import { hasOpenreplayAttribute } from "../../utils.js"; +import { + RemoveNodeAttribute, + SetNodeAttribute, + SetNodeAttributeURLBased, + SetCSSDataURLBased, + SetNodeData, + CreateTextNode, + CreateElementNode, + MoveNode, + RemoveNode, +} from "../../../messages/index.js"; +import App from "../index.js"; +import { isInstance, inDocument } from "../context.js"; + + +function isSVGElement(node: Element): node is SVGElement { + return node.namespaceURI === 'http://www.w3.org/2000/svg'; +} + +function isIgnored(node: Node): boolean { + if (isInstance(node, Text)) { + return false; + } + if (!isInstance(node, Element)) { + return true; + } + const tag = node.tagName.toUpperCase(); + if (tag === 'LINK') { + const rel = node.getAttribute('rel'); + const as = node.getAttribute('as'); + return !(rel?.includes('stylesheet') || as === "style" || as === "font"); + } + return ( + tag === 'SCRIPT' || + tag === 'NOSCRIPT' || + tag === 'META' || + tag === 'TITLE' || + tag === 'BASE' + ); +} + +function isRootNode(node: Node): boolean { + return isInstance(node, Document) || isInstance(node, ShadowRoot); +} + +function isObservable(node: Node): boolean { + if (isRootNode(node)) { + return true; + } + return !isIgnored(node); +} + +export default abstract class Observer { + private readonly observer: MutationObserver; + private readonly commited: Array = []; + private readonly recents: Array = []; + private readonly myNodes: Array = []; + private readonly indexes: Array = []; + private readonly attributesList: Array | undefined> = []; + private readonly textSet: Set = new Set(); + private readonly inUpperContext: boolean; + constructor(protected readonly app: App, protected readonly context: Window = window) { + this.inUpperContext = context.parent === context //TODO: get rid of context here + this.observer = new MutationObserver( + this.app.safe((mutations) => { + for (const mutation of mutations) { + const target = mutation.target; + const type = mutation.type; + + if (!isObservable(target) || !inDocument(target)) { + continue; + } + if (type === 'childList') { + for (let i = 0; i < mutation.removedNodes.length; i++) { + this.bindTree(mutation.removedNodes[i]); + } + for (let i = 0; i < mutation.addedNodes.length; i++) { + this.bindTree(mutation.addedNodes[i]); + } + continue; + } + const id = this.app.nodes.getID(target); + if (id === undefined) { + continue; + } + if (id >= this.recents.length) { // TODO: something more convinient + this.recents[id] = undefined; + } + if (type === 'attributes') { + const name = mutation.attributeName; + if (name === null) { + continue; + } + let attr = this.attributesList[id]; + if (attr === undefined) { + this.attributesList[id] = attr = new Set(); + } + attr.add(name); + continue; + } + if (type === 'characterData') { + this.textSet.add(id); + continue; + } + } + this.commitNodes(); + }), + ); + } + private clear(): void { + this.commited.length = 0; + this.recents.length = 0; + this.indexes.length = 1; + this.attributesList.length = 0; + this.textSet.clear(); + } + + private sendNodeAttribute( + id: number, + node: Element, + name: string, + value: string | null, + ): void { + if (isSVGElement(node)) { + if (name.substr(0, 6) === 'xlink:') { + name = name.substr(6); + } + if (value === null) { + this.app.send(new RemoveNodeAttribute(id, name)); + } else if (name === 'href') { + if (value.length > 1e5) { + value = ''; + } + this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); + } else { + this.app.send(new SetNodeAttribute(id, name, value)); + } + return; + } + if ( + name === 'src' || + name === 'srcset' || + name === 'integrity' || + name === 'crossorigin' || + name === 'autocomplete' || + name.substr(0, 2) === 'on' + ) { + return; + } + if ( + name === 'value' && + isInstance(node, HTMLInputElement) && + node.type !== 'button' && + node.type !== 'reset' && + node.type !== 'submit' + ) { + return; + } + if (value === null) { + this.app.send(new RemoveNodeAttribute(id, name)); + return; + } + if (name === 'style' || name === 'href' && isInstance(node, HTMLLinkElement)) { + this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); + return; + } + if (name === 'href' || value.length > 1e5) { + value = ''; + } + this.app.send(new SetNodeAttribute(id, name, value)); + } + + private sendNodeData(id: number, parentElement: Element, data: string): void { + if (isInstance(parentElement, HTMLStyleElement) || isInstance(parentElement, SVGStyleElement)) { + this.app.send(new SetCSSDataURLBased(id, data, this.app.getBaseHref())); + return; + } + data = this.app.sanitizer.sanitize(id, data) + this.app.send(new SetNodeData(id, data)); + } + + private bindNode(node: Node): void { + const r = this.app.nodes.registerNode(node); + const id = r[0]; + this.recents[id] = r[1] || this.recents[id] || false; + + this.myNodes[id] = true; + } + + private bindTree(node: Node): void { + if (!isObservable(node)) { + return + } + this.bindNode(node); + const walker = document.createTreeWalker( + node, + NodeFilter.SHOW_ELEMENT + NodeFilter.SHOW_TEXT, + { + acceptNode: (node) => + isIgnored(node) || this.app.nodes.getID(node) !== undefined + ? NodeFilter.FILTER_REJECT + : NodeFilter.FILTER_ACCEPT, + }, + // @ts-ignore + false, + ); + while (walker.nextNode()) { + this.bindNode(walker.currentNode); + } + } + + private unbindNode(node: Node): void { + const id = this.app.nodes.unregisterNode(node); + if (id !== undefined && this.recents[id] === false) { + this.app.send(new RemoveNode(id)); + } + } + + private _commitNode(id: number, node: Node): boolean { + if (isRootNode(node)) { + return true; + } + const parent = node.parentNode; + let parentID: number | undefined; + // Disable parent check for the upper context HTMLHtmlElement, because it is root there... (before) + // TODO: get rid of "special" cases (there is an issue with CreateDocument altered behaviour though) + // TODO: Clean the logic (though now it workd fine) + if (!isInstance(node, HTMLHtmlElement) || !this.inUpperContext) { + if (parent === null) { + this.unbindNode(node); + return false; + } + parentID = this.app.nodes.getID(parent); + if (parentID === undefined) { + this.unbindNode(node); + return false; + } + if (!this.commitNode(parentID)) { + this.unbindNode(node); + return false; + } + this.app.sanitizer.handleNode(id, parentID, node); + } + let sibling = node.previousSibling; + while (sibling !== null) { + const siblingID = this.app.nodes.getID(sibling); + if (siblingID !== undefined) { + this.commitNode(siblingID); + this.indexes[id] = this.indexes[siblingID] + 1; + break; + } + sibling = sibling.previousSibling; + } + if (sibling === null) { + this.indexes[id] = 0; // + } + const isNew = this.recents[id]; + const index = this.indexes[id]; + if (index === undefined) { + throw 'commitNode: missing node index'; + } + if (isNew === true) { + if (isInstance(node, Element)) { + if (parentID !== undefined) { + this.app.send(new + CreateElementNode( + id, + parentID, + index, + node.tagName, + isSVGElement(node), + ), + ); + } + for (let i = 0; i < node.attributes.length; i++) { + const attr = node.attributes[i]; + this.sendNodeAttribute(id, node, attr.nodeName, attr.value); + } + } else if (isInstance(node, Text)) { + // for text node id != 0, hence parentID !== undefined and parent is Element + this.app.send(new CreateTextNode(id, parentID as number, index)); + this.sendNodeData(id, parent as Element, node.data); + } + return true; + } + if (isNew === false && parentID !== undefined) { + this.app.send(new MoveNode(id, parentID, index)); + } + const attr = this.attributesList[id]; + if (attr !== undefined) { + if (!isInstance(node, Element)) { + throw 'commitNode: node is not an element'; + } + for (const name of attr) { + this.sendNodeAttribute(id, node, name, node.getAttribute(name)); + } + } + if (this.textSet.has(id)) { + if (!isInstance(node, Text)) { + throw 'commitNode: node is not a text'; + } + // for text node id != 0, hence parent is Element + this.sendNodeData(id, parent as Element, node.data); + } + return true; + } + private commitNode(id: number): boolean { + const node = this.app.nodes.getNode(id); + if (node === undefined) { + return false; + } + const cmt = this.commited[id]; + if (cmt !== undefined) { + return cmt; + } + return (this.commited[id] = this._commitNode(id, node)); + } + private commitNodes(): void { + let node; + for (let id = 0; id < this.recents.length; id++) { + // TODO: make things/logic nice here. + // commit required in any case if recents[id] true or false (in case of unbinding) or undefined (in case of attr change). + if (!this.myNodes[id]) { continue } + this.commitNode(id); + if (this.recents[id] === true && (node = this.app.nodes.getNode(id))) { + this.app.nodes.callNodeCallbacks(node); + } + } + this.clear(); + } + + // ISSSUE + protected observeRoot(node: Node, beforeCommit: (id?: number) => unknown, nodeToBind: Node = node) { + this.observer.observe(node, { + childList: true, + attributes: true, + characterData: true, + subtree: true, + attributeOldValue: false, + characterDataOldValue: false, + }); + this.bindTree(nodeToBind); + beforeCommit(this.app.nodes.getID(node)) + this.commitNodes(); + } + + disconnect(): void { + this.observer.disconnect(); + this.clear(); + this.myNodes.length = 0; + } +} diff --git a/tracker/tracker/src/main/app/observer/shadow_root_observer.ts b/tracker/tracker/src/main/app/observer/shadow_root_observer.ts new file mode 100644 index 000000000..244348ea1 --- /dev/null +++ b/tracker/tracker/src/main/app/observer/shadow_root_observer.ts @@ -0,0 +1,18 @@ +import Observer from "./observer.js"; +import { CreateIFrameDocument } from "../../../messages/index.js"; + +export default class ShadowRootObserver extends Observer { + observe(el: Element) { + const shRoot = el.shadowRoot; + const hostID = this.app.nodes.getID(el); + if (!shRoot || hostID === undefined) { return } // log + this.observeRoot(shRoot, (rootID) => { + if (rootID === undefined) { + console.log("OpenReplay: Shadow Root was not bound") + return; + } + this.app.send(CreateIFrameDocument(hostID,rootID)); + }); + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/app/observer/top_observer.ts b/tracker/tracker/src/main/app/observer/top_observer.ts new file mode 100644 index 000000000..b35f5d901 --- /dev/null +++ b/tracker/tracker/src/main/app/observer/top_observer.ts @@ -0,0 +1,98 @@ +import Observer from "./observer.js"; +import { isInstance } from "../context.js"; +import type { Window } from "../context.js"; +import IFrameObserver from "./iframe_observer.js"; +import ShadowRootObserver from "./shadow_root_observer.js"; + +import { CreateDocument } from "../../../messages/index.js"; +import App from "../index.js"; +import { IN_BROWSER } from '../../utils.js' + +export interface Options { + captureIFrames: boolean +} + +const attachShadowNativeFn = IN_BROWSER ? Element.prototype.attachShadow : ()=>new ShadowRoot(); + +export default class TopObserver extends Observer { + private readonly options: Options; + constructor(app: App, options: Partial) { + super(app); + this.options = Object.assign({ + captureIFrames: false + }, options); + + // IFrames + this.app.nodes.attachNodeCallback(node => { + if (isInstance(node, HTMLIFrameElement) && + (this.options.captureIFrames || node.getAttribute("data-openreplay-capture")) + ) { + this.handleIframe(node) + } + }) + + // ShadowDOM + this.app.nodes.attachNodeCallback(node => { + if (isInstance(node, Element) && node.shadowRoot !== null) { + this.handleShadowRoot(node.shadowRoot) + } + }) + } + + + private iframeObservers: IFrameObserver[] = []; + private handleIframe(iframe: HTMLIFrameElement): void { + let context: Window | null = null + const handle = this.app.safe(() => { + const id = this.app.nodes.getID(iframe) + if (id === undefined) { return } //log + if (iframe.contentWindow === context) { return } //Does this happen frequently? + context = iframe.contentWindow as Window | null; + if (!context) { return } + const observer = new IFrameObserver(this.app, context) + + this.iframeObservers.push(observer) + observer.observe(iframe) + }) + this.app.attachEventListener(iframe, "load", handle) + handle() + } + + private shadowRootObservers: ShadowRootObserver[] = [] + private handleShadowRoot(shRoot: ShadowRoot) { + const observer = new ShadowRootObserver(this.app, this.context) + + this.shadowRootObservers.push(observer) + observer.observe(shRoot.host) + } + + observe(): void { + // Protection from several subsequent calls? + const observer = this; + Element.prototype.attachShadow = function() { + const shadow = attachShadowNativeFn.apply(this, arguments) + observer.handleShadowRoot(shadow) + return shadow + } + + // Can observe documentElement () here, because it is not supposed to be changing. + // However, it is possible in some exotic cases and may cause an ignorance of the newly created + // In this case context.document have to be observed, but this will cause + // the change in the re-player behaviour caused by CreateDocument message: + // the 0-node ("fRoot") will become #document rather than documentElement as it is now. + // Alternatively - observe(#document) then bindNode(documentElement) + this.observeRoot(this.context.document, () => { + this.app.send(new CreateDocument()) + }, this.context.document.documentElement); + } + + disconnect() { + Element.prototype.attachShadow = attachShadowNativeFn + this.iframeObservers.forEach(o => o.disconnect()) + this.iframeObservers = [] + this.shadowRootObservers.forEach(o => o.disconnect()) + this.shadowRootObservers = [] + super.disconnect() + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/app/sanitizer.ts b/tracker/tracker/src/main/app/sanitizer.ts new file mode 100644 index 000000000..d085b5739 --- /dev/null +++ b/tracker/tracker/src/main/app/sanitizer.ts @@ -0,0 +1,66 @@ +import { stars, hasOpenreplayAttribute } from "../utils.js"; +import App from "./index.js"; +import { isInstance } from "./context.js"; + +export interface Options { + obscureTextEmails: boolean; + obscureTextNumbers: boolean; +} + +export default class Sanitizer { + private readonly masked: Set = new Set(); + private readonly options: Options; + + constructor(private readonly app: App, options: Partial) { + this.options = Object.assign({ + obscureTextEmails: true, + obscureTextNumbers: false, + }, options); + } + + handleNode(id: number, parentID: number, node: Node) { + if ( + this.masked.has(parentID) || + (isInstance(node, Element) && hasOpenreplayAttribute(node, 'masked')) + ) { + this.masked.add(id); + } + } + + sanitize(id: number, data: string): string { + if (this.masked.has(id)) { + // TODO: is it the best place to put trim() ? Might trimmed spaces be considered in layout in certain cases? + return data.trim().replace( + /[^\f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/g, + '█', + ); + } + if (this.options.obscureTextNumbers) { + data = data.replace(/\d/g, '0'); + } + if (this.options.obscureTextEmails) { + data = data.replace( + /([^\s]+)@([^\s]+)\.([^\s]+)/g, + (...f: Array) => + stars(f[1]) + '@' + stars(f[2]) + '.' + stars(f[3]), + ); + } + return data + } + + isMasked(id: number): boolean { + return this.masked.has(id); + } + + getInnerTextSecure(el: HTMLElement): string { + const id = this.app.nodes.getID(el) + if (!id) { return '' } + return this.sanitize(id, el.innerText) + + } + + clear(): void { + this.masked.clear(); + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/index.ts b/tracker/tracker/src/main/index.ts index 6af325e57..75d195e50 100644 --- a/tracker/tracker/src/main/index.ts +++ b/tracker/tracker/src/main/index.ts @@ -19,14 +19,15 @@ import Longtasks from "./modules/longtasks.js"; import CSSRules from "./modules/cssrules.js"; import { IN_BROWSER, deprecationWarn, DOCS_HOST } from "./utils.js"; -import { Options as AppOptions } from "./app/index.js"; -import { Options as ConsoleOptions } from "./modules/console.js"; -import { Options as ExceptionOptions } from "./modules/exception.js"; -import { Options as InputOptions } from "./modules/input.js"; -import { Options as PerformanceOptions } from "./modules/performance.js"; -import { Options as TimingOptions } from "./modules/timing.js"; - -export type { OnStartInfo } from './app/index.js'; +import type { Options as AppOptions } from "./app/index.js"; +import type { Options as ConsoleOptions } from "./modules/console.js"; +import type { Options as ExceptionOptions } from "./modules/exception.js"; +import type { Options as InputOptions } from "./modules/input.js"; +import type { Options as PerformanceOptions } from "./modules/performance.js"; +import type { Options as TimingOptions } from "./modules/timing.js"; +import type { StartOptions } from './app/index.js' +//TODO: unique options init +import type { OnStartInfo } from './app/index.js'; export type Options = Partial< AppOptions & ConsoleOptions & ExceptionOptions & InputOptions & PerformanceOptions & TimingOptions @@ -35,6 +36,7 @@ export type Options = Partial< projectKey: string; sessionToken?: string; respectDoNotTrack?: boolean; + autoResetOnWindowOpen?: boolean; // dev only __DISABLE_SECURE_MODE?: boolean; }; @@ -84,7 +86,7 @@ export default class API { (navigator.doNotTrack == '1' // @ts-ignore || window.doNotTrack == '1'); - this.app = doNotTrack || + const app = this.app = doNotTrack || !('Map' in window) || !('Set' in window) || !('MutationObserver' in window) || @@ -95,20 +97,35 @@ export default class API { !('Worker' in window) ? null : new App(options.projectKey, options.sessionToken, options); - if (this.app !== null) { - Viewport(this.app); - CSSRules(this.app); - Connection(this.app); - Console(this.app, options); - Exception(this.app, options); - Img(this.app); - Input(this.app, options); - Mouse(this.app); - Timing(this.app, options); - Performance(this.app, options); - Scroll(this.app); - Longtasks(this.app); + if (app !== null) { + Viewport(app); + CSSRules(app); + Connection(app); + Console(app, options); + Exception(app, options); + Img(app); + Input(app, options); + Mouse(app); + Timing(app, options); + Performance(app, options); + Scroll(app); + Longtasks(app); (window as any).__OPENREPLAY__ = this; + + if (options.autoResetOnWindowOpen) { + const wOpen = window.open; + app.attachStartCallback(() => { + // @ts-ignore ? + window.open = function(...args) { + app.resetNextPageSession(true) + wOpen.call(window, ...args) + app.resetNextPageSession(false) + } + }) + app.attachStopCallback(() => { + window.open = wOpen; + }) + } } else { console.log("OpenReplay: browser doesn't support API required for tracking or doNotTrack is set to 1.") const req = new XMLHttpRequest(); @@ -140,7 +157,7 @@ export default class API { return this.isActive(); } - start() /*: Promise*/ { + start(startOpts?: StartOptions) : Promise { if (!IN_BROWSER) { console.error(`OpenReplay: you are trying to start Tracker on a node.js environment. If you want to use OpenReplay with SSR, please, use componentDidMount or useEffect API for placing the \`tracker.start()\` line. Check documentation on ${DOCS_HOST}${DOCS_SETUP}`) return Promise.reject("Trying to start not in browser."); @@ -148,7 +165,7 @@ export default class API { if (this.app === null) { return Promise.reject("Browser doesn't support required api, or doNotTrack is active."); } - return this.app.start(); + return this.app.start(startOpts); } stop(): void { if (this.app === null) { diff --git a/tracker/tracker/src/main/modules/exception.ts b/tracker/tracker/src/main/modules/exception.ts index 45fe37465..848df03be 100644 --- a/tracker/tracker/src/main/modules/exception.ts +++ b/tracker/tracker/src/main/modules/exception.ts @@ -50,7 +50,13 @@ export function getExceptionMessageFromEvent(e: ErrorEvent | PromiseRejectionEve if (e.reason instanceof Error) { return getExceptionMessage(e.reason, []) } else { - return new JSException('Unhandled Promise Rejection', String(e.reason), '[]'); + let message: string; + try { + message = JSON.stringify(e.reason) + } catch(_) { + message = String(e.reason) + } + return new JSException('Unhandled Promise Rejection', message, '[]'); } } return null; diff --git a/tracker/tracker/src/main/modules/img.ts b/tracker/tracker/src/main/modules/img.ts index 61e793b89..8c0f911a8 100644 --- a/tracker/tracker/src/main/modules/img.ts +++ b/tracker/tracker/src/main/modules/img.ts @@ -1,8 +1,21 @@ import { timestamp, isURL } from "../utils.js"; import App from "../app/index.js"; -import { ResourceTiming, SetNodeAttributeURLBased } from "../../messages/index.js"; +import { ResourceTiming, SetNodeAttributeURLBased, SetNodeAttribute } from "../../messages/index.js"; + +const PLACEHOLDER_SRC = "https://static.openreplay.com/tracker/placeholder.jpeg"; export default function (app: App): void { + function sendPlaceholder(id: number, node: HTMLImageElement): void { + app.send(new SetNodeAttribute(id, "src", PLACEHOLDER_SRC)) + const { width, height } = node.getBoundingClientRect(); + if (!node.hasAttribute("width")){ + app.send(new SetNodeAttribute(id, "width", String(width))) + } + if (!node.hasAttribute("height")){ + app.send(new SetNodeAttribute(id, "height", String(height))) + } + } + const sendImgSrc = app.safe(function (this: HTMLImageElement): void { const id = app.nodes.getID(this); if (id === undefined) { @@ -16,7 +29,9 @@ export default function (app: App): void { if (src != null && isURL(src)) { // TODO: How about relative urls ? Src type is null sometimes. app.send(new ResourceTiming(timestamp(), 0, 0, 0, 0, 0, src, 'img')); } - } else if (src.length < 1e5) { + } else if (src.length >= 1e5 || app.sanitizer.isMasked(id)) { + sendPlaceholder(id, this) + } else { app.send(new SetNodeAttributeURLBased(id, 'src', src, app.getBaseHref())); } }); diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 746c26f8f..ad8cda673 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -2,7 +2,12 @@ import { normSpaces, IN_BROWSER, getLabelAttribute, hasOpenreplayAttribute } fro import App from "../app/index.js"; import { SetInputTarget, SetInputValue, SetInputChecked } from "../../messages/index.js"; -function isInput(node: any): node is HTMLInputElement { +// TODO: take into consideration "contenteditable" attribute +type TextEditableElement = HTMLInputElement | HTMLTextAreaElement +function isTextEditable(node: any): node is TextEditableElement { + if (node instanceof HTMLTextAreaElement) { + return true; + } if (!(node instanceof HTMLInputElement)) { return false; } @@ -16,6 +21,7 @@ function isInput(node: any): node is HTMLInputElement { type === 'range' ); } + function isCheckable(node: any): node is HTMLInputElement { if (!(node instanceof HTMLInputElement)) { return false; @@ -25,7 +31,7 @@ function isCheckable(node: any): node is HTMLInputElement { } const labelElementFor: ( - node: HTMLInputElement, + node: TextEditableElement, ) => HTMLLabelElement | undefined = IN_BROWSER && 'labels' in HTMLInputElement.prototype ? (node): HTMLLabelElement | undefined => { @@ -56,7 +62,7 @@ const labelElementFor: ( } }; -export function getInputLabel(node: HTMLInputElement): string { +export function getInputLabel(node: TextEditableElement): string { let label = getLabelAttribute(node); if (label === null) { const labelElement = labelElementFor(node); @@ -89,13 +95,13 @@ export default function (app: App, opts: Partial): void { }, opts, ); - function sendInputTarget(id: number, node: HTMLInputElement): void { + function sendInputTarget(id: number, node: TextEditableElement): void { const label = getInputLabel(node); if (label !== '') { app.send(new SetInputTarget(id, label)); } } - function sendInputValue(id: number, node: HTMLInputElement): void { + function sendInputValue(id: number, node: TextEditableElement): void { let value = node.value; let inputMode: InputMode = options.defaultInputMode; if (node.type === 'password' || hasOpenreplayAttribute(node, 'hidden')) { @@ -136,7 +142,7 @@ export default function (app: App, opts: Partial): void { app.ticker.attach((): void => { inputValues.forEach((value, id) => { const node = app.nodes.getNode(id); - if (!isInput(node)) { + if (!isTextEditable(node)) { inputValues.delete(id); return; } @@ -169,7 +175,7 @@ export default function (app: App, opts: Partial): void { if (id === undefined) { return; } - if (isInput(node)) { + if (isTextEditable(node)) { inputValues.set(id, node.value); sendInputValue(id, node); return; diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index 3ec70e844..0089fa37f 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -92,7 +92,7 @@ export default function (app: App): void { (target as HTMLElement).onclick != null || target.getAttribute('role') === 'button' ) { - const label: string = app.observer.getInnerTextSecure(target as HTMLElement); + const label: string = app.sanitizer.getInnerTextSecure(target as HTMLElement); return normSpaces(label).slice(0, 100); } return ''; diff --git a/tracker/tracker/src/webworker/index.ts b/tracker/tracker/src/webworker/index.ts index 723008c52..cf0d1586a 100644 --- a/tracker/tracker/src/webworker/index.ts +++ b/tracker/tracker/src/webworker/index.ts @@ -47,6 +47,7 @@ function sendBatch(batch: Uint8Array):void { return; // happens simultaneously with onerror TODO: clear codeflow } if (this.status >= 400) { // TODO: test workflow. After 400+ it calls /start for some reason + busy = false; reset(); sendQueue.length = 0; if (this.status === 401) { // Unauthorised (Token expired) From 12b414793521a5dfe5f83c8316ff802ac6e5f89a Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 23 Jan 2022 18:57:06 +0100 Subject: [PATCH 31/33] feat (backend): insert userID on SessionStart --- backend/pkg/db/postgres/messages_common.go | 7 +++++-- backend/pkg/messages/messages.go | 4 +++- backend/pkg/messages/read_message.go | 1 + backend/services/http/handlers_web.go | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/pkg/db/postgres/messages_common.go b/backend/pkg/db/postgres/messages_common.go index a6f624651..df539e05c 100644 --- a/backend/pkg/db/postgres/messages_common.go +++ b/backend/pkg/db/postgres/messages_common.go @@ -47,7 +47,8 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error { rev_id, tracker_version, issue_score, platform, - user_agent, user_browser, user_browser_version, user_device_memory_size, user_device_heap_size + user_agent, user_browser, user_browser_version, user_device_memory_size, user_device_heap_size, + user_id ) VALUES ( $1, $2, $3, $4, $5, $6, $7, @@ -55,7 +56,8 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error { NULLIF($10, ''), $11, $12, $13, - NULLIF($14, ''), NULLIF($15, ''), NULLIF($16, ''), NULLIF($17, 0), NULLIF($18, 0::bigint) + NULLIF($14, ''), NULLIF($15, ''), NULLIF($16, ''), NULLIF($17, 0), NULLIF($18, 0::bigint), + NULLIF($19, '') )`, sessionID, s.ProjectID, s.Timestamp, s.UserUUID, s.UserDevice, s.UserDeviceType, s.UserCountry, @@ -64,6 +66,7 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error { s.TrackerVersion, s.Timestamp/1000, s.Platform, s.UserAgent, s.UserBrowser, s.UserBrowserVersion, s.UserDeviceMemorySize, s.UserDeviceHeapSize, + s.UserID, ); err != nil { return err; } diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 3d8bae7f6..cdff71e1d 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -63,9 +63,10 @@ UserDeviceType string UserDeviceMemorySize uint64 UserDeviceHeapSize uint64 UserCountry string +UserID string } func (msg *SessionStart) Encode() []byte{ - buf := make([]byte, 151 + len(msg.TrackerVersion)+ len(msg.RevID)+ len(msg.UserUUID)+ len(msg.UserAgent)+ len(msg.UserOS)+ len(msg.UserOSVersion)+ len(msg.UserBrowser)+ len(msg.UserBrowserVersion)+ len(msg.UserDevice)+ len(msg.UserDeviceType)+ len(msg.UserCountry)) + buf := make([]byte, 161 + len(msg.TrackerVersion)+ len(msg.RevID)+ len(msg.UserUUID)+ len(msg.UserAgent)+ len(msg.UserOS)+ len(msg.UserOSVersion)+ len(msg.UserBrowser)+ len(msg.UserBrowserVersion)+ len(msg.UserDevice)+ len(msg.UserDeviceType)+ len(msg.UserCountry)+ len(msg.UserID)) buf[0] = 1 p := 1 p = WriteUint(msg.Timestamp, buf, p) @@ -83,6 +84,7 @@ p = WriteString(msg.UserDeviceType, buf, p) p = WriteUint(msg.UserDeviceMemorySize, buf, p) p = WriteUint(msg.UserDeviceHeapSize, buf, p) p = WriteString(msg.UserCountry, buf, p) +p = WriteString(msg.UserID, buf, p) return buf[:p] } diff --git a/backend/pkg/messages/read_message.go b/backend/pkg/messages/read_message.go index d0148bbc6..c226df728 100644 --- a/backend/pkg/messages/read_message.go +++ b/backend/pkg/messages/read_message.go @@ -42,6 +42,7 @@ if msg.UserDeviceType, err = ReadString(reader); err != nil { return nil, err } if msg.UserDeviceMemorySize, err = ReadUint(reader); err != nil { return nil, err } if msg.UserDeviceHeapSize, err = ReadUint(reader); err != nil { return nil, err } if msg.UserCountry, err = ReadString(reader); err != nil { return nil, err } +if msg.UserID, err = ReadString(reader); err != nil { return nil, err } return msg, nil case 2: diff --git a/backend/services/http/handlers_web.go b/backend/services/http/handlers_web.go index 5e144f1cc..09d2511d8 100644 --- a/backend/services/http/handlers_web.go +++ b/backend/services/http/handlers_web.go @@ -27,6 +27,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { JsHeapSizeLimit uint64 `json:"jsHeapSizeLimit"` ProjectKey *string `json:"projectKey"` Reset bool `json:"reset"` + UserID string `json:"userID"` } type response struct { Timestamp int64 `json:"timestamp"` @@ -101,6 +102,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { UserCountry: country, UserDeviceMemorySize: req.DeviceMemory, UserDeviceHeapSize: req.JsHeapSizeLimit, + UserID: req.UserID, })) } From 41cd2aba7b37efab3c4b1d86c0eb83f723257374 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 23 Jan 2022 19:31:13 +0100 Subject: [PATCH 32/33] feat-fix (backend): insert userID on SessionStart (2) --- backend/pkg/db/cache/messages_web.go | 1 + backend/pkg/db/types/session.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/pkg/db/cache/messages_web.go b/backend/pkg/db/cache/messages_web.go index 3afae8592..b259e49da 100644 --- a/backend/pkg/db/cache/messages_web.go +++ b/backend/pkg/db/cache/messages_web.go @@ -30,6 +30,7 @@ func (c *PGCache) InsertWebSessionStart(sessionID uint64, s *SessionStart) error UserDeviceType: s.UserDeviceType, UserDeviceMemorySize: s.UserDeviceMemorySize, UserDeviceHeapSize: s.UserDeviceHeapSize, + UserID: s.UserID, } if err := c.Conn.InsertSessionStart(sessionID, c.sessions[ sessionID ]); err != nil { c.sessions[ sessionID ] = nil diff --git a/backend/pkg/db/types/session.go b/backend/pkg/db/types/session.go index 0205f76f3..d354b0cd2 100644 --- a/backend/pkg/db/types/session.go +++ b/backend/pkg/db/types/session.go @@ -16,7 +16,7 @@ type Session struct { PagesCount int EventsCount int ErrorsCount int - UserID *string + UserID string // pointer?? UserAnonymousID *string Metadata1 *string Metadata2 *string From b4f441132ff63672392168e23312b4e3bcc5c598 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 23 Jan 2022 19:38:37 +0100 Subject: [PATCH 33/33] feat-fix (backend): insert userID on SessionStart (3) --- backend/pkg/db/cache/messages_common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/db/cache/messages_common.go b/backend/pkg/db/cache/messages_common.go index c05422cb2..dcf860835 100644 --- a/backend/pkg/db/cache/messages_common.go +++ b/backend/pkg/db/cache/messages_common.go @@ -38,7 +38,7 @@ func (c *PGCache) InsertUserID(sessionID uint64, userID *IOSUserID) error { if err != nil { return err } - session.UserID = &userID.Value + session.UserID = userID.Value return nil }