diff --git a/api/chalicelib/core/canvas.py b/api/chalicelib/core/canvas.py new file mode 100644 index 000000000..b3f1a3a0d --- /dev/null +++ b/api/chalicelib/core/canvas.py @@ -0,0 +1,23 @@ +from chalicelib.utils import pg_client, helper +from chalicelib.utils.storage import StorageClient +from decouple import config + + +def get_canvas_presigned_urls(session_id, project_id): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify("""\ + SELECT * + FROM events.canvas_recordings + WHERE session_id = %(session_id)s + ORDER BY timestamp;""", + {"project_id": project_id, "session_id": session_id}) + ) + rows = cur.fetchall() + + for i in range(len(rows)): + rows.append(StorageClient.get_presigned_url_for_sharing( + bucket=config("CANVAS_BUCKET", default=config("sessions_bucket")), + expires_in=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900), + key=rows[i] + )) + return rows diff --git a/api/chalicelib/core/sessions_replay.py b/api/chalicelib/core/sessions_replay.py index 694ff57e7..52c2a58fa 100644 --- a/api/chalicelib/core/sessions_replay.py +++ b/api/chalicelib/core/sessions_replay.py @@ -1,6 +1,6 @@ import schemas from chalicelib.core import events, metadata, events_ios, \ - sessions_mobs, issues, resources, assist, sessions_devtool, sessions_notes + sessions_mobs, issues, resources, assist, sessions_devtool, sessions_notes, canvas from chalicelib.utils import errors_helper from chalicelib.utils import pg_client, helper @@ -131,6 +131,7 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False) data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, check_existence=False) + data['canvasURL'] = canvas.get_canvas_presigned_urls(session_id=session_id, project_id=project_id) data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, diff --git a/api/env.default b/api/env.default index 1d88927f0..4fbe56f6f 100644 --- a/api/env.default +++ b/api/env.default @@ -1,62 +1,62 @@ +announcement_url= +assist=/sockets-live +ASSIST_JWT_EXPIRATION=144000 +ASSIST_JWT_SECRET= +ASSIST_KEY= +ASSIST_URL=http://assist-openreplay.app.svc.cluster.local:9001/assist/%s +assistList=/sockets-list +CANVAS_PATTERN=%(sessionId)s/%(recordingId)s.mp4 +captcha_key= +captcha_server= +change_password_link=/reset-password?invitation=%s&&pass=%s +DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob +EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)sdevtools +EFS_SESSION_MOB_PATTERN=%(sessionId)s EMAIL_FROM=OpenReplay EMAIL_HOST= EMAIL_PASSWORD= EMAIL_PORT=587 EMAIL_SSL_CERT= EMAIL_SSL_KEY= -EMAIL_USER= EMAIL_USE_SSL=false EMAIL_USE_TLS=true -S3_HOST= -S3_KEY= -S3_SECRET= -S3_DISABLE_SSL_VERIFY= -SITE_URL= -announcement_url= -captcha_key= -captcha_server= -change_password_link=/reset-password?invitation=%s&&pass=%s +EMAIL_USER= +FS_DIR=/mnt/efs invitation_link=/api/users/invitation?token=%s +IOS_VIDEO_BUCKET=mobs js_cache_bucket=sessions-assets jwt_algorithm=HS512 JWT_EXPIRATION=86400 -JWT_REFRESH_EXPIRATION=604800 JWT_ISSUER=openreplay-oss -jwt_secret="SET A RANDOM STRING HERE" +JWT_REFRESH_EXPIRATION=604800 JWT_REFRESH_SECRET="SET A RANDOM STRING HERE" -ASSIST_URL=http://assist-openreplay.app.svc.cluster.local:9001/assist/%s -ASSIST_KEY= -assist=/sockets-live -assistList=/sockets-list +jwt_secret="SET A RANDOM STRING HERE" pg_dbname=postgres pg_host=postgresql.db.svc.cluster.local -pg_password=asayerPostgres -pg_port=5432 -pg_user=postgres -PG_TIMEOUT=30 -PG_MINCONN=20 PG_MAXCONN=50 -PG_RETRY_MAX=50 -PG_RETRY_INTERVAL=2 +PG_MINCONN=20 +pg_password=asayerPostgres PG_POOL=true +pg_port=5432 +PG_RETRY_INTERVAL=2 +PG_RETRY_MAX=50 +PG_TIMEOUT=30 +pg_user=postgres +PRESIGNED_URL_EXPIRATION=3600 +PYTHONUNBUFFERED=1 +REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 +S3_DISABLE_SSL_VERIFY= +S3_HOST= +S3_KEY= +S3_SECRET= +SCH_DELETE_DAYS=30 +SESSION_IOS_VIDEO_PATTERN=%(sessionId)s/replay.mp4 +SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe +SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs sessions_bucket=mobs sessions_region=us-east-1 +SITE_URL= sourcemaps_bucket=sourcemaps sourcemaps_reader=http://sourcemapreader-openreplay.app.svc.cluster.local:9000/sourcemaps/%s/sourcemaps STAGE=default-foss -version_number=1.4.0 -FS_DIR=/mnt/efs -EFS_SESSION_MOB_PATTERN=%(sessionId)s -EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)sdevtools -SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs -SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe -DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob -SESSION_IOS_VIDEO_PATTERN=%(sessionId)s/replay.mp4 -PRESIGNED_URL_EXPIRATION=3600 -ASSIST_JWT_EXPIRATION=144000 -ASSIST_JWT_SECRET= -PYTHONUNBUFFERED=1 -REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 -SCH_DELETE_DAYS=30 -IOS_VIDEO_BUCKET=mobs TZ=UTC \ No newline at end of file diff --git a/api/env.dev b/api/env.dev index 63721c24a..45650fd8a 100644 --- a/api/env.dev +++ b/api/env.dev @@ -1,66 +1,63 @@ +announcement_url= +assist=/sockets-live +ASSIST_JWT_EXPIRATION=14400 +ASSIST_JWT_SECRET=secret +ASSIST_KEY=abc +ASSIST_URL=http://127.0.0.1:9001/assist/%s +assistList=/sockets-list +CANVAS_PATTERN=%(sessionId)s/%(recordingId)s.mp4 +captcha_key= +captcha_server= +change_password_link=/changepassword?invitation=%s&&pass=%s +DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mobs +docs_url=/docs +docs_url=/docs +EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob +EFS_SESSION_MOB_PATTERN=%(sessionId)s/dom.mob EMAIL_FROM=Openreplay-dev EMAIL_HOST= EMAIL_PASSWORD= EMAIL_PORT=587 EMAIL_SSL_CERT= EMAIL_SSL_KEY= -EMAIL_USER= EMAIL_USE_SSL=false EMAIL_USE_TLS=true -S3_HOST= -S3_KEY= -S3_SECRET= -SITE_URL=http://127.0.0.1:3333 -announcement_url= -captcha_key= -captcha_server= -change_password_link=/changepassword?invitation=%s&&pass=%s +EMAIL_USER= +FS_DIR= invitation_link=/users/invitation?token=%s +IOS_VIDEO_BUCKET=mobs js_cache_bucket= jwt_algorithm=HS512 JWT_EXPIRATION=6000 -JWT_REFRESH_EXPIRATION=60 JWT_ISSUER=openReplay-dev -jwt_secret=SECRET +JWT_REFRESH_EXPIRATION=60 JWT_REFRESH_SECRET=SECRET2 -ASSIST_URL=http://127.0.0.1:9001/assist/%s -assist=/sockets-live -assistList=/sockets-list - -# FOSS +jwt_secret=SECRET +LOCAL_DEV=true +LOGLEVEL=INFO pg_dbname=postgres pg_host=127.0.0.1 -pg_password=password -pg_port=5420 -pg_user=postgres - -PG_TIMEOUT=20 -PG_MINCONN=2 PG_MAXCONN=5 -PG_RETRY_MAX=50 -PG_RETRY_INTERVAL=2 +PG_MINCONN=2 +pg_password=password PG_POOL=true +pg_port=5420 +PG_RETRY_INTERVAL=2 +PG_RETRY_MAX=50 +PG_TIMEOUT=20 +pg_user=postgres +PRESIGNED_URL_EXPIRATION=3600 +REDIS_STRING=redis://127.0.0.1:6379 +root_path='' +S3_HOST= +S3_KEY= +S3_SECRET= +SESSION_IOS_VIDEO_PATTERN=%(sessionId)s/replay.mp4 +SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe +SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs sessions_bucket=mobs sessions_region=us-east-1 +SITE_URL=http://127.0.0.1:3333 sourcemaps_bucket= sourcemaps_reader=http://127.0.0.1:3000/sourcemaps -LOGLEVEL=INFO -FS_DIR= - -ASSIST_KEY=abc -EFS_SESSION_MOB_PATTERN=%(sessionId)s/dom.mob -EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob -SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs -SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe -DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mobs -PRESIGNED_URL_EXPIRATION=3600 -ASSIST_JWT_EXPIRATION=14400 -ASSIST_JWT_SECRET=secret -REDIS_STRING=redis://127.0.0.1:6379 -LOCAL_DEV=true -TZ=UTC -docs_url=/docs -root_path='' -docs_url=/docs -IOS_VIDEO_BUCKET=mobs -SESSION_IOS_VIDEO_PATTERN=%(sessionId)s/replay.mp4 \ No newline at end of file +TZ=UTC \ No newline at end of file diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 1f2a2cd1c..79b230dbb 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -178,11 +178,18 @@ README/* Pipfile.lock .local/* - +/app_alerts.py +/auth/__init__.py +/auth/auth_apikey.py +/build.sh +/build_alerts.sh +/build_crons.sh /chalicelib/core/alerts.py -#exp /chalicelib/core/alerts_processor.py /chalicelib/core/announcements.py +/chalicelib/core/assist.py +/chalicelib/core/authorizers.py /chalicelib/core/autocomplete.py +/chalicelib/core/canvas.py /chalicelib/core/click_maps.py /chalicelib/core/collaboration_base.py /chalicelib/core/collaboration_msteams.py @@ -190,11 +197,11 @@ Pipfile.lock /chalicelib/core/countries.py /chalicelib/core/custom_metrics_predefined.py /chalicelib/core/dashboards.py -#exp /chalicelib/core/errors.py /chalicelib/core/errors_favorite.py -#exp /chalicelib/core/events.py /chalicelib/core/events_ios.py +/chalicelib/core/feature_flags.py /chalicelib/core/funnels.py +/chalicelib/core/heatmaps.py /chalicelib/core/integration_base.py /chalicelib/core/integration_base_issue.py /chalicelib/core/integration_github.py @@ -215,17 +222,15 @@ Pipfile.lock /chalicelib/core/log_tool_sumologic.py /chalicelib/core/metadata.py /chalicelib/core/mobile.py +/chalicelib/core/performance_event.py +/chalicelib/core/saved_search.py /chalicelib/core/sessions.py /chalicelib/core/sessions_assignments.py -#exp /chalicelib/core/sessions_metas.py /chalicelib/core/sessions_mobs.py -#exp /chalicelib/core/significance.py /chalicelib/core/socket_ios.py /chalicelib/core/sourcemaps.py /chalicelib/core/sourcemaps_parser.py -/chalicelib/core/feature_flags.py /chalicelib/saml -/chalicelib/utils/html/ /chalicelib/utils/__init__.py /chalicelib/utils/args_transformer.py /chalicelib/utils/captcha.py @@ -236,43 +241,31 @@ Pipfile.lock /chalicelib/utils/event_filter_definition.py /chalicelib/utils/github_client_v3.py /chalicelib/utils/helper.py +/chalicelib/utils/html/ /chalicelib/utils/jira_client.py /chalicelib/utils/metrics_helper.py /chalicelib/utils/pg_client.py /chalicelib/utils/smtp.py /chalicelib/utils/sql_helper.py -/chalicelib/utils/strings.py -/chalicelib/utils/TimeUTC.py /chalicelib/utils/storage/generators.py /chalicelib/utils/storage/interface.py /chalicelib/utils/storage/s3.py -/routers/app/__init__.py +/chalicelib/utils/strings.py +/chalicelib/utils/TimeUTC.py /crons/__init__.py -/routers/subs/__init__.py -/routers/__init__.py -/chalicelib/core/assist.py -/auth/__init__.py -/auth/auth_apikey.py -/build.sh -/routers/base.py -/routers/core.py /crons/core_crons.py /db_changes.sql /Dockerfile_bundle /entrypoint.bundle.sh -/chalicelib/core/heatmaps.py -#exp /chalicelib/core/custom_metrics.py -/chalicelib/core/performance_event.py -/chalicelib/core/saved_search.py -/app_alerts.py -/build_alerts.sh -/build_crons.sh -/run-dev.sh -/run-alerts-dev.sh +/routers/__init__.py +/routers/app/__init__.py +/routers/base.py +/routers/core.py +/routers/subs/__init__.py /routers/subs/v1_api.py -#exp /chalicelib/core/dashboards.py +/run-alerts-dev.sh +/run-dev.sh /schemas/overrides.py /schemas/schemas.py -/chalicelib/core/authorizers.py /schemas/transformers_validators.py /test/ diff --git a/ee/api/chalicelib/core/sessions_replay.py b/ee/api/chalicelib/core/sessions_replay.py index 8015f5995..6027c7f9e 100644 --- a/ee/api/chalicelib/core/sessions_replay.py +++ b/ee/api/chalicelib/core/sessions_replay.py @@ -1,6 +1,6 @@ import schemas from chalicelib.core import events, metadata, events_ios, \ - sessions_mobs, issues, resources, assist, sessions_devtool, sessions_notes + sessions_mobs, issues, resources, assist, sessions_devtool, sessions_notes, canvas from chalicelib.utils import errors_helper from chalicelib.utils import pg_client, helper @@ -139,6 +139,7 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False) data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, context=context, check_existence=False) + data['canvasURL'] = canvas.get_canvas_presigned_urls(session_id=session_id, project_id=project_id) data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index 37913fe91..3cf7be7cc 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -1,23 +1,28 @@ #!/bin/bash +rm -rf ./app_alerts.py +rm -rf ./auth/__init__.py +rm -rf ./auth/auth_apikey.py +rm -rf ./build.sh +rm -rf ./build_alerts.sh +rm -rf ./build_crons.sh rm -rf ./chalicelib/core/alerts.py -#exp rm -rf ./chalicelib/core/alerts_processor.py rm -rf ./chalicelib/core/announcements.py -rm -rf ./chalicelib/core/autocomplete.py +rm -rf ./chalicelib/core/assist.py rm -rf ./chalicelib/core/authorizers.py +rm -rf ./chalicelib/core/autocomplete.py rm -rf ./chalicelib/core/click_maps.py rm -rf ./chalicelib/core/collaboration_base.py rm -rf ./chalicelib/core/collaboration_msteams.py rm -rf ./chalicelib/core/collaboration_slack.py rm -rf ./chalicelib/core/countries.py rm -rf ./chalicelib/core/custom_metrics_predefined.py -rm -rf ./chalicelib/core/feature_flags.py -#exp rm -rf ./chalicelib/core/errors.py -rm -rf ./chalicelib/core/errors_favorite.py -#exp rm -rf ./chalicelib/core/events.py -rm -rf ./chalicelib/core/events_ios.py rm -rf ./chalicelib/core/dashboards.py +rm -rf ./chalicelib/core/errors_favorite.py +rm -rf ./chalicelib/core/events_ios.py +rm -rf ./chalicelib/core/feature_flags.py rm -rf ./chalicelib/core/funnels.py +rm -rf ./chalicelib/core/heatmaps.py rm -rf ./chalicelib/core/integration_base.py rm -rf ./chalicelib/core/integration_base_issue.py rm -rf ./chalicelib/core/integration_github.py @@ -38,18 +43,18 @@ rm -rf ./chalicelib/core/log_tool_stackdriver.py rm -rf ./chalicelib/core/log_tool_sumologic.py rm -rf ./chalicelib/core/metadata.py rm -rf ./chalicelib/core/mobile.py +rm -rf ./chalicelib/core/performance_event.py +rm -rf ./chalicelib/core/saved_search.py rm -rf ./chalicelib/core/sessions.py rm -rf ./chalicelib/core/sessions_assignments.py -#exp rm -rf ./chalicelib/core/sessions_metas.py rm -rf ./chalicelib/core/sessions_mobs.py -#exp rm -rf ./chalicelib/core/significance.py rm -rf ./chalicelib/core/socket_ios.py rm -rf ./chalicelib/core/sourcemaps.py rm -rf ./chalicelib/core/sourcemaps_parser.py rm -rf ./chalicelib/saml -rm -rf ./chalicelib/utils/html/ rm -rf ./chalicelib/utils/__init__.py rm -rf ./chalicelib/utils/args_transformer.py +rm -rf ./chalicelib/utils/canvas.py rm -rf ./chalicelib/utils/captcha.py rm -rf ./chalicelib/utils/dev.py rm -rf ./chalicelib/utils/email_handler.py @@ -58,40 +63,30 @@ rm -rf ./chalicelib/utils/errors_helper.py rm -rf ./chalicelib/utils/event_filter_definition.py rm -rf ./chalicelib/utils/github_client_v3.py rm -rf ./chalicelib/utils/helper.py +rm -rf ./chalicelib/utils/html/ rm -rf ./chalicelib/utils/jira_client.py rm -rf ./chalicelib/utils/metrics_helper.py rm -rf ./chalicelib/utils/pg_client.py rm -rf ./chalicelib/utils/smtp.py rm -rf ./chalicelib/utils/sql_helper.py -rm -rf ./chalicelib/utils/strings.py -rm -rf ./chalicelib/utils/TimeUTC.py rm -rf ./chalicelib/utils/storage/generators.py rm -rf ./chalicelib/utils/storage/interface.py rm -rf ./chalicelib/utils/storage/s3.py -rm -rf ./routers/app/__init__.py +rm -rf ./chalicelib/utils/strings.py +rm -rf ./chalicelib/utils/TimeUTC.py rm -rf ./crons/__init__.py -rm -rf ./routers/subs/__init__.py -rm -rf ./routers/__init__.py -rm -rf ./chalicelib/core/assist.py -rm -rf ./auth/__init__.py -rm -rf ./auth/auth_apikey.py -rm -rf ./build.sh -rm -rf ./build_crons.sh -rm -rf ./routers/base.py -rm -rf ./routers/core.py rm -rf ./crons/core_crons.py rm -rf ./db_changes.sql rm -rf ./Dockerfile_bundle rm -rf ./entrypoint.bundle.sh -rm -rf ./chalicelib/core/heatmaps.py +rm -rf ./routers/__init__.py +rm -rf ./routers/app/__init__.py +rm -rf ./routers/base.py +rm -rf ./routers/core.py +rm -rf ./routers/subs/__init__.py rm -rf ./routers/subs/v1_api.py -#exp rm -rf ./chalicelib/core/custom_metrics.py -rm -rf ./chalicelib/core/performance_event.py -rm -rf ./chalicelib/core/saved_search.py -rm -rf ./app_alerts.py -rm -rf ./build_alerts.sh -rm -rf ./run-dev.sh rm -rf ./run-alerts-dev.sh +rm -rf ./run-dev.sh rm -rf ./schemas/overrides.py rm -rf ./schemas/schemas.py rm -rf ./schemas/transformers_validators.py \ No newline at end of file diff --git a/ee/api/env.default b/ee/api/env.default index bf7b6fee7..4fc8c80c7 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -1,81 +1,81 @@ +announcement_url= +assist=/sockets-live +ASSIST_JWT_EXPIRATION=144000 +ASSIST_JWT_SECRET= +ASSIST_KEY= +ASSIST_RECORDS_BUCKET=records +ASSIST_URL=http://assist-openreplay.app.svc.cluster.local:9001/assist/%s +assistList=/sockets-list +CANVAS_PATTERN=%(sessionId)s/%(recordingId)s.mp4 +captcha_key= +captcha_server= +CH_COMPRESSION=true +ch_host= +ch_port= +ch_receive_timeout=10 +ch_timeout=30 +change_password_link=/reset-password?invitation=%s&&pass=%s +DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob +EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)sdevtools +EFS_SESSION_MOB_PATTERN=%(sessionId)s EMAIL_FROM=OpenReplay EMAIL_HOST= EMAIL_PASSWORD= EMAIL_PORT=587 EMAIL_SSL_CERT= EMAIL_SSL_KEY= -EMAIL_USER= EMAIL_USE_SSL=false EMAIL_USE_TLS=true -LICENSE_KEY= -S3_HOST= -S3_KEY= -S3_SECRET= -S3_DISABLE_SSL_VERIFY= -SAML2_MD_URL= -SITE_URL= -announcement_url= -captcha_key= -captcha_server= -ch_host= -ch_port= -ch_timeout=30 -ch_receive_timeout=10 -change_password_link=/reset-password?invitation=%s&&pass=%s +EMAIL_USER= +EXP_7D_MV=false +EXP_ALERTS=false +EXP_AUTOCOMPLETE=false +EXP_ERRORS_GET=false +EXP_ERRORS_SEARCH=false +EXP_FUNNELS=false +EXP_RESOURCES=true +EXP_SESSIONS_SEARCH=false +FS_DIR=/mnt/efs idp_entityId= idp_sls_url= idp_sso_url= idp_x509cert= invitation_link=/api/users/invitation?token=%s +IOS_VIDEO_BUCKET=mobs js_cache_bucket=sessions-assets jwt_algorithm=HS512 JWT_EXPIRATION=86400 -JWT_REFRESH_EXPIRATION=604800 JWT_ISSUER=openreplay-oss -jwt_secret="SET A RANDOM STRING HERE" +JWT_REFRESH_EXPIRATION=604800 JWT_REFRESH_SECRET="SET A RANDOM STRING HERE" -ASSIST_URL=http://assist-openreplay.app.svc.cluster.local:9001/assist/%s -ASSIST_KEY= -assist=/sockets-live -assistList=/sockets-list -pg_dbname=postgres -pg_host=postgresql.db.svc.cluster.local -pg_password=asayerPostgres -pg_port=5432 -pg_user=postgres -PG_TIMEOUT=30 -PG_MINCONN=20 -PG_MAXCONN=50 -PG_RETRY_MAX=50 -PG_RETRY_INTERVAL=2 -PG_POOL=true -ASSIST_RECORDS_BUCKET=records -sessions_bucket=mobs -sessions_region=us-east-1 -sourcemaps_bucket=sourcemaps -sourcemaps_reader=http://sourcemapreader-openreplay.app.svc.cluster.local:9000/sourcemaps/%s/sourcemaps -version_number=1.0.0 -FS_DIR=/mnt/efs -EXP_SESSIONS_SEARCH=false -EXP_AUTOCOMPLETE=false -EXP_ERRORS_SEARCH=false -EXP_ERRORS_GET=false -EXP_7D_MV=false -EXP_ALERTS=false -EXP_FUNNELS=false -EXP_RESOURCES=true -TRACE_PERIOD=300 -EFS_SESSION_MOB_PATTERN=%(sessionId)s -EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)sdevtools -SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs -SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe -DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob -PRESIGNED_URL_EXPIRATION=3600 -ASSIST_JWT_EXPIRATION=144000 -ASSIST_JWT_SECRET= +jwt_secret="SET A RANDOM STRING HERE" KAFKA_SERVERS=kafka.db.svc.cluster.local:9092 KAFKA_USE_SSL=false +LICENSE_KEY= +pg_dbname=postgres +pg_host=postgresql.db.svc.cluster.local +PG_MAXCONN=50 +PG_MINCONN=20 +pg_password=asayerPostgres +PG_POOL=true +pg_port=5432 +PG_RETRY_INTERVAL=2 +PG_RETRY_MAX=50 +PG_TIMEOUT=30 +pg_user=postgres +PRESIGNED_URL_EXPIRATION=3600 +S3_DISABLE_SSL_VERIFY= +S3_HOST= +S3_KEY= +S3_SECRET= +SAML2_MD_URL= SCH_DELETE_DAYS=30 -TZ=UTC -CH_COMPRESSION=true -IOS_VIDEO_BUCKET=mobs \ No newline at end of file +SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe +SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs +sessions_bucket=mobs +sessions_region=us-east-1 +SITE_URL= +sourcemaps_bucket=sourcemaps +sourcemaps_reader=http://sourcemapreader-openreplay.app.svc.cluster.local:9000/sourcemaps/%s/sourcemaps +TRACE_PERIOD=300 +TZ=UTC \ No newline at end of file diff --git a/ee/api/env.dev b/ee/api/env.dev index 1aa609137..65b69a055 100644 --- a/ee/api/env.dev +++ b/ee/api/env.dev @@ -1,83 +1,80 @@ +announcement_url= +assist=/sockets-live +ASSIST_JWT_EXPIRATION=14400 +ASSIST_JWT_SECRET=secret +ASSIST_KEY=abc +ASSIST_URL=http://127.0.0.1:9001/assist/%s +assistList=/sockets-list +CANVAS_PATTERN=%(sessionId)s/%(recordingId)s.mp4 +captcha_key= +captcha_server= +change_password_link=/changepassword?invitation=%s&&pass=%s +DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mobs +docs_url=/docs +docs_url=/docs +EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob +EFS_SESSION_MOB_PATTERN=%(sessionId)s/dom.mob EMAIL_FROM=Openreplay-dev EMAIL_HOST= EMAIL_PASSWORD= EMAIL_PORT=587 EMAIL_SSL_CERT= EMAIL_SSL_KEY= -EMAIL_USER= EMAIL_USE_SSL=false EMAIL_USE_TLS=true -S3_HOST= -S3_KEY= -S3_SECRET= -SITE_URL=http://127.0.0.1:3333 -announcement_url= -captcha_key= -captcha_server= -change_password_link=/changepassword?invitation=%s&&pass=%s +EMAIL_USER= +ENABLE_SSO=false +EXP_7D_MV=false +EXP_ALERTS=false +EXP_AUTOCOMPLETE=true +EXP_ERRORS_GET=false +EXP_ERRORS_SEARCH=false +EXP_FUNNELS=false +EXP_RESOURCES=true +EXP_SESSIONS_SEARCH=false +EXP_SESSIONS_SEARCH_METRIC=true +FS_DIR= +idp_entityId= +idp_name=okta +idp_sls_url='' +idp_sso_url= +idp_x509cert= invitation_link=/users/invitation?token=%s +IOS_VIDEO_BUCKET=mobs js_cache_bucket= jwt_algorithm=HS512 JWT_EXPIRATION=6000 -JWT_REFRESH_EXPIRATION=60 JWT_ISSUER=openReplay-dev -jwt_secret=SECRET +JWT_REFRESH_EXPIRATION=60 JWT_REFRESH_SECRET=SECRET2 -ASSIST_URL=http://127.0.0.1:9001/assist/%s -assist=/sockets-live -assistList=/sockets-list - -# EE -pg_dbname=postgres -pg_host=127.0.0.1 -pg_password=password -pg_port=5420 -pg_user=postgres - -PG_TIMEOUT=20 -PG_MINCONN=2 -PG_MAXCONN=5 -PG_RETRY_MAX=50 -PG_RETRY_INTERVAL=2 -PG_POOL=true -sessions_bucket=mobs -sessions_region=us-east-1 -sourcemaps_bucket= -sourcemaps_reader=http://127.0.0.1:3000/sourcemaps -LOGLEVEL=INFO -FS_DIR= - -idp_entityId= -idp_sso_url= -idp_sls_url='' -idp_name=okta -idp_x509cert= -EXP_SESSIONS_SEARCH=false -EXP_AUTOCOMPLETE=true -EXP_ERRORS_SEARCH=false -EXP_ERRORS_GET=false -EXP_7D_MV=false -EXP_ALERTS=false -EXP_FUNNELS=false -EXP_RESOURCES=true -EXP_SESSIONS_SEARCH_METRIC=true -ASSIST_KEY=abc -EFS_SESSION_MOB_PATTERN=%(sessionId)s/dom.mob -EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob -SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs -SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe -DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mobs -PRESIGNED_URL_EXPIRATION=3600 -ASSIST_JWT_EXPIRATION=14400 -ASSIST_JWT_SECRET=secret -REDIS_STRING=redis://127.0.0.1:6379 +jwt_secret=SECRET KAFKA_SERVERS=127.0.0.1:9092 KAFKA_USE_SSL=false LOCAL_DEV=true -ENABLE_SSO=false -TZ=UTC -docs_url=/docs +LOGLEVEL=INFO +pg_dbname=postgres +pg_host=127.0.0.1 +PG_MAXCONN=5 +PG_MINCONN=2 +pg_password=password +PG_POOL=true +pg_port=5420 +PG_RETRY_INTERVAL=2 +PG_RETRY_MAX=50 +PG_TIMEOUT=20 +pg_user=postgres +PRESIGNED_URL_EXPIRATION=3600 +REDIS_STRING=redis://127.0.0.1:6379 root_path='' -docs_url=/docs -IOS_VIDEO_BUCKET=mobs -SESSION_IOS_VIDEO_PATTERN=%(sessionId)s/replay.mp4 \ No newline at end of file +S3_HOST= +S3_KEY= +S3_SECRET= +SESSION_IOS_VIDEO_PATTERN=%(sessionId)s/replay.mp4 +SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe +SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs +sessions_bucket=mobs +sessions_region=us-east-1 +SITE_URL=http://127.0.0.1:3333 +sourcemaps_bucket= +sourcemaps_reader=http://127.0.0.1:3000/sourcemaps +TZ=UTC \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.16.0/1.16.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.16.0/1.16.0.sql new file mode 100644 index 000000000..3d93c5d23 --- /dev/null +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.16.0/1.16.0.sql @@ -0,0 +1,36 @@ +\set previous_version 'v1.15.0' +\set next_version 'v1.16.0' +SELECT openreplay_version() AS current_version, + openreplay_version() = :'previous_version' AS valid_previous, + openreplay_version() = :'next_version' AS is_next +\gset + +\if :valid_previous +\echo valid previous DB version :'previous_version', starting DB upgrade to :'next_version' +BEGIN; +SELECT format($fn_def$ +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT '%1$s' +$$ LANGUAGE sql IMMUTABLE; +$fn_def$, :'next_version') +\gexec + +-- + +CREATE TABLE IF NOT EXISTS events.canvas_recordings +( + session_id bigint NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE, + recording_id text NOT NULL, + timestamp bigint NOT NULL +); +CREATE INDEX IF NOT EXISTS canvas_recordings_session_id_idx ON events.canvas_recordings (session_id); + +COMMIT; + +\elif :is_next +\echo new version detected :'next_version', nothing to do +\else +\warn skipping DB upgrade of :'next_version', expected previous version :'previous_version', found :'current_version' +\endif \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index 724af507a..fbdbea147 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -973,13 +973,13 @@ $$ CREATE TABLE IF NOT EXISTS public.assist_events ( - event_id varchar NOT NULL PRIMARY KEY, - project_id integer NOT NULL, - session_id varchar NOT NULL, - event_type varchar NOT NULL, - timestamp bigint NOT NULL, - duration integer, - agent_id integer + event_id varchar NOT NULL PRIMARY KEY, + project_id integer NOT NULL, + session_id varchar NOT NULL, + event_type varchar NOT NULL, + timestamp bigint NOT NULL, + duration integer, + agent_id integer ); CREATE TABLE IF NOT EXISTS public.assist_events_aggregates @@ -1019,7 +1019,8 @@ $$ ('pages'), ('performance'), ('resources'), - ('state_actions')) + ('state_actions'), + ('canvas_recordings')) select bool_and(exists(select * from information_schema.tables t where table_schema = 'events' @@ -1267,6 +1268,15 @@ $$ CREATE INDEX IF NOT EXISTS performance_session_id_timestamp_idx ON events.performance (session_id, timestamp); CREATE INDEX IF NOT EXISTS performance_avg_cpu_gt0_idx ON events.performance (avg_cpu) WHERE avg_cpu > 0; CREATE INDEX IF NOT EXISTS performance_avg_used_js_heap_size_gt0_idx ON events.performance (avg_used_js_heap_size) WHERE avg_used_js_heap_size > 0; + + CREATE TABLE IF NOT EXISTS events.canvas_recordings + ( + session_id bigint NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE, + recording_id text NOT NULL, + timestamp bigint NOT NULL + ); + CREATE INDEX IF NOT EXISTS canvas_recordings_session_id_idx ON events.canvas_recordings (session_id); + END IF; END; $$ diff --git a/ee/scripts/schema/db/rollback_dbs/postgresql/1.16.0/1.16.0.sql b/ee/scripts/schema/db/rollback_dbs/postgresql/1.16.0/1.16.0.sql new file mode 100644 index 000000000..0bd9577a8 --- /dev/null +++ b/ee/scripts/schema/db/rollback_dbs/postgresql/1.16.0/1.16.0.sql @@ -0,0 +1,30 @@ +\set previous_version 'v1.16.0-ee' +\set next_version 'v1.15.0-ee' +SELECT openreplay_version() AS current_version, + openreplay_version() = :'previous_version' AS valid_previous, + openreplay_version() = :'next_version' AS is_next +\gset + +\if :valid_previous +\echo valid previous DB version :'previous_version', starting DB downgrade to :'next_version' +BEGIN; +SELECT format($fn_def$ +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT '%1$s' +$$ LANGUAGE sql IMMUTABLE; +$fn_def$, :'next_version') +\gexec + +-- + +DROP TABLE IF EXISTS events.canvas_recordings; + +COMMIT; + +\elif :is_next +\echo new version detected :'next_version', nothing to do +\else +\warn skipping DB downgrade of :'next_version', expected previous version :'previous_version', found :'current_version' +\endif \ No newline at end of file diff --git a/scripts/schema/db/init_dbs/postgresql/1.16.0/1.16.0.sql b/scripts/schema/db/init_dbs/postgresql/1.16.0/1.16.0.sql new file mode 100644 index 000000000..2416358b8 --- /dev/null +++ b/scripts/schema/db/init_dbs/postgresql/1.16.0/1.16.0.sql @@ -0,0 +1,36 @@ +\set previous_version 'v1.15.0-ee' +\set next_version 'v1.16.0-ee' +SELECT openreplay_version() AS current_version, + openreplay_version() = :'previous_version' AS valid_previous, + openreplay_version() = :'next_version' AS is_next +\gset + +\if :valid_previous +\echo valid previous DB version :'previous_version', starting DB upgrade to :'next_version' +BEGIN; +SELECT format($fn_def$ +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT '%1$s' +$$ LANGUAGE sql IMMUTABLE; +$fn_def$, :'next_version') +\gexec + +-- + +CREATE TABLE IF NOT EXISTS events.canvas_recordings +( + session_id bigint NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE, + recording_id text NOT NULL, + timestamp bigint NOT NULL +); +CREATE INDEX IF NOT EXISTS canvas_recordings_session_id_idx ON events.canvas_recordings (session_id); + +COMMIT; + +\elif :is_next +\echo new version detected :'next_version', nothing to do +\else +\warn skipping DB upgrade of :'next_version', expected previous version :'previous_version', found :'current_version' +\endif \ No newline at end of file diff --git a/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/scripts/schema/db/init_dbs/postgresql/init_schema.sql index f80545d2b..e26c4da30 100644 --- a/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -1116,6 +1116,14 @@ $$ CREATE INDEX swipes_timestamp_idx ON events_ios.swipes (timestamp); CREATE INDEX swipes_label_session_id_timestamp_idx ON events_ios.swipes (label, session_id, timestamp); + CREATE TABLE events.canvas_recordings + ( + session_id bigint NOT NULL REFERENCES public.sessions (session_id) ON DELETE CASCADE, + recording_id text NOT NULL, + timestamp bigint NOT NULL + ); + CREATE INDEX canvas_recordings_session_id_idx ON events.canvas_recordings (session_id); + raise notice 'DB created'; END IF; END; diff --git a/scripts/schema/db/rollback_dbs/postgresql/1.16.0/1.16.0.sql b/scripts/schema/db/rollback_dbs/postgresql/1.16.0/1.16.0.sql new file mode 100644 index 000000000..351e672ea --- /dev/null +++ b/scripts/schema/db/rollback_dbs/postgresql/1.16.0/1.16.0.sql @@ -0,0 +1,30 @@ +\set previous_version 'v1.16.0' +\set next_version 'v1.15.0' +SELECT openreplay_version() AS current_version, + openreplay_version() = :'previous_version' AS valid_previous, + openreplay_version() = :'next_version' AS is_next +\gset + +\if :valid_previous +\echo valid previous DB version :'previous_version', starting DB downgrade to :'next_version' +BEGIN; +SELECT format($fn_def$ +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT '%1$s' +$$ LANGUAGE sql IMMUTABLE; +$fn_def$, :'next_version') +\gexec + +-- + +DROP TABLE IF EXISTS events.canvas_recordings; + +COMMIT; + +\elif :is_next +\echo new version detected :'next_version', nothing to do +\else +\warn skipping DB downgrade of :'next_version', expected previous version :'previous_version', found :'current_version' +\endif \ No newline at end of file