From 6e156e0bdb2f4f3ebaae47bc8ac4d88a0b653cde Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Sat, 4 Dec 2021 15:24:35 +0100 Subject: [PATCH 01/12] Delete test.md --- scripts/helm/db/test.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 scripts/helm/db/test.md diff --git a/scripts/helm/db/test.md b/scripts/helm/db/test.md deleted file mode 100644 index 8b1378917..000000000 --- a/scripts/helm/db/test.md +++ /dev/null @@ -1 +0,0 @@ - From df0ea888e2620df2b91259efa648f90812185f35 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Sat, 4 Dec 2021 15:27:28 +0100 Subject: [PATCH 02/12] Update 1.3.6.sql --- .../db/init_dbs/postgresql/1.3.6/1.3.6.sql | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/helm/db/init_dbs/postgresql/1.3.6/1.3.6.sql b/scripts/helm/db/init_dbs/postgresql/1.3.6/1.3.6.sql index cc4704f26..7638a1433 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.3.6/1.3.6.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.3.6/1.3.6.sql @@ -1,16 +1,16 @@ BEGIN; -CREATE INDEX sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; -CREATE INDEX sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; -CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; +CREATE INDEX IF NOT EXISTS sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; +CREATE INDEX IF NOT EXISTS pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; -CREATE INDEX clicks_session_id_timestamp_idx ON events.clicks (session_id, timestamp); -CREATE INDEX errors_error_id_idx ON errors (error_id); -CREATE INDEX errors_error_id_idx ON events.errors (error_id); +CREATE INDEX IF NOT EXISTS clicks_session_id_timestamp_idx ON events.clicks (session_id, timestamp); +CREATE INDEX IF NOT EXISTS errors_error_id_idx ON errors (error_id); +CREATE INDEX IF NOT EXISTS errors_error_id_idx ON events.errors (error_id); -CREATE INDEX issues_issue_id_timestamp_idx ON events_common.issues(issue_id,timestamp); -CREATE INDEX issues_timestamp_idx ON events_common.issues (timestamp); -CREATE INDEX issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); +CREATE INDEX IF NOT EXISTS issues_issue_id_timestamp_idx ON events_common.issues(issue_id,timestamp); +CREATE INDEX IF NOT EXISTS issues_timestamp_idx ON events_common.issues (timestamp); +CREATE INDEX IF NOT EXISTS issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); -COMMIT; \ No newline at end of file +COMMIT; From 9069e989d75f35711edd2ab05eb18b09042637f0 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Sat, 4 Dec 2021 15:28:27 +0100 Subject: [PATCH 03/12] Update 1.3.6.sql --- .../helm/db/init_dbs/postgresql/1.3.6/1.3.6.sql | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.3.6/1.3.6.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.3.6/1.3.6.sql index 15ce18803..34eeb5dc9 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.3.6/1.3.6.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.3.6/1.3.6.sql @@ -1,13 +1,13 @@ BEGIN; -CREATE INDEX sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; -CREATE INDEX sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; -CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; +CREATE INDEX IF NOT EXISTS sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; +CREATE INDEX IF NOT EXISTS pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; -CREATE INDEX users_tenant_id_deleted_at_N_idx ON users (tenant_id) WHERE deleted_at ISNULL; -CREATE INDEX issues_issue_id_timestamp_idx ON events_common.issues (issue_id, timestamp); -CREATE INDEX issues_timestamp_idx ON events_common.issues (timestamp); -CREATE INDEX issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); +CREATE INDEX IF NOT EXISTS users_tenant_id_deleted_at_N_idx ON users (tenant_id) WHERE deleted_at ISNULL; +CREATE INDEX IF NOT EXISTS issues_issue_id_timestamp_idx ON events_common.issues (issue_id, timestamp); +CREATE INDEX IF NOT EXISTS issues_timestamp_idx ON events_common.issues (timestamp); +CREATE INDEX IF NOT EXISTS issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); CREATE TABLE roles ( @@ -69,4 +69,4 @@ ALTER TABLE public.users ALTER TABLE public.users ALTER COLUMN origin TYPE text; DROP TYPE IF EXISTS user_origin; -COMMIT; \ No newline at end of file +COMMIT; From c8c6e98286b3dea861969124f3d5222828a4081f Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 4 Dec 2021 19:04:06 +0530 Subject: [PATCH 04/12] chore(upgrade): remove sed template / as this is a valid password string. Signed-off-by: Rajesh Rajendran --- scripts/helm/upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helm/upgrade.sh b/scripts/helm/upgrade.sh index 60f161721..a62ddf446 100644 --- a/scripts/helm/upgrade.sh +++ b/scripts/helm/upgrade.sh @@ -66,7 +66,7 @@ patch(){ for var in ${vars[@]};do # Get old value old_val=`grep $var ${openreplay_old_dir}/scripts/helm/app/chalice.yaml|xargs` - sed -i "s/${var}.*/$old_val/g" app/chalice.yaml + sed -i "s#${var}.*#$old_val#g" app/chalice.yaml done } From 1b6f92c3d497003c415fc432773812f18e1fa55e Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 4 Dec 2021 19:30:05 +0530 Subject: [PATCH 05/12] chore(vars): updating variables to vars_template.yaml Signed-off-by: Rajesh Rajendran --- scripts/helm/upgrade.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/helm/upgrade.sh b/scripts/helm/upgrade.sh index a62ddf446..8dd94fef3 100644 --- a/scripts/helm/upgrade.sh +++ b/scripts/helm/upgrade.sh @@ -65,8 +65,11 @@ patch(){ ) for var in ${vars[@]};do # Get old value - old_val=`grep $var ${openreplay_old_dir}/scripts/helm/app/chalice.yaml|xargs` - sed -i "s#${var}.*#$old_val#g" app/chalice.yaml + old_val=`grep $var ${openreplay_old_dir}/scripts/helm/app/chalice.yaml| cutl -d" " f4|xargs` + # Coverting caps env var to small ansible variable. + # In chalice EMAIL_HOST translates to email_host in vars.yaml + # Ref: https://stackoverflow.com/questions/2264428/how-to-convert-a-string-to-lower-case-in-bash + sed -i "s#${var,,}.*#${var,,}: \"$old_val\"#g" vars_template.yaml done } From 4dd1a8aece536edd997e183be37cabc6c2a0d315 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 4 Dec 2021 19:32:54 +0530 Subject: [PATCH 06/12] chore(upgrade): fail if error Signed-off-by: Rajesh Rajendran --- scripts/helm/upgrade.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/helm/upgrade.sh b/scripts/helm/upgrade.sh index 8dd94fef3..10880087e 100644 --- a/scripts/helm/upgrade.sh +++ b/scripts/helm/upgrade.sh @@ -2,6 +2,8 @@ # upgrade.sh v1.10 +set -e + cwd=$PWD openreplay_old_dir=$1 vars_file_path=${openreplay_old_dir}/scripts/helm/vars.yaml From 49c1fb6e099bfcfbdb6e074b399130db761d4d37 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 4 Dec 2021 20:09:28 +0530 Subject: [PATCH 07/12] chore(upgrade): ignore create folder in db, which is used for clickhouse Signed-off-by: Rajesh Rajendran --- scripts/helm/upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helm/upgrade.sh b/scripts/helm/upgrade.sh index 10880087e..5b2123f0e 100644 --- a/scripts/helm/upgrade.sh +++ b/scripts/helm/upgrade.sh @@ -37,7 +37,7 @@ migration(){ # Ref: https://stackoverflow.com/questions/1527049/how-can-i-join-elements-of-an-array-in-bash # Creating an array of versions to migrate. db=$1 - migration_versions=(`ls -l db/init_dbs/$db | grep -E ^d | awk -v number=${old_version} '$NF > number {print $NF}'`) + migration_versions=(`ls -l db/init_dbs/$db | grep -E ^d | awk -v number=${old_version} '$NF > number {print $NF}' | grep -v create`) # Can't pass the space seperated array to ansible for migration. So joining them with , joined_migration_versions=$(IFS=, ; echo "${migration_versions[*]}") From 9be1809274b7f04506331d13107585069d2d9d00 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 4 Dec 2021 20:14:07 +0530 Subject: [PATCH 08/12] chore(upgrade): enable clickhouse migration Signed-off-by: Rajesh Rajendran --- scripts/helm/migration.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/helm/migration.yaml b/scripts/helm/migration.yaml index 4d7c2ce64..abd4e571a 100644 --- a/scripts/helm/migration.yaml +++ b/scripts/helm/migration.yaml @@ -30,18 +30,17 @@ db_path: [] - name: generating migration db paths set_fact: - db_path: "{{dst_list | default([])}} + [ '{{ item[0] }}/*.sql' ]" + db_path: "{{ db_path }} + [ '{{playbook_dir}}/db/init_dbs/clickhouse/{{ item }}/*.sql' ]" with_items: "{{ migration_versions.split(',') }}" - name: Restoring clickhouse data shell: | file="{{ item|basename }}" kubectl exec -n db clickhouse-0 -- /bin/bash -c "rm -rf /tmp/$file" - kubectl cp -n db $file clickhouse-0:/tmp/ + kubectl cp -n db {{ item }} clickhouse-0:/tmp/ kubectl exec -n db clickhouse-0 -- /bin/bash -c "clickhouse-client < /tmp/$file" 2>&1 | tee -a "{{ playbook_dir }}"/clickhouse_init.log args: - chdir: db/init_dbs/clickhouse/create - with_fileglob: - - "{{ db_path }}" + chdir: db/init_dbs/clickhouse/ + with_fileglob: "{{ db_path }}" retries: 3 delay: 60 register: result From 9704c9ee5a7689cc2faed53d80c57481c69bcde6 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 4 Dec 2021 20:21:44 +0530 Subject: [PATCH 09/12] chore(upgrade): Support enterprise data migration Signed-off-by: Rajesh Rajendran --- scripts/helm/upgrade.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/helm/upgrade.sh b/scripts/helm/upgrade.sh index 5b2123f0e..3a5df210a 100644 --- a/scripts/helm/upgrade.sh +++ b/scripts/helm/upgrade.sh @@ -79,6 +79,11 @@ patch(){ patch installation_type=1 +if [[ ${ENTERPRISE} -eq 1 ]]; then + cp -rf ../../ee/scripts/helm/db/* db/ + echo -e "Migrating clickhouse" + migration clickhouse +fi echo -e "Migrating postgresql" migration postgresql From 57af83dd4872fdadad9882d2106f369f47240912 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 4 Dec 2021 20:27:57 +0530 Subject: [PATCH 10/12] chore(upgrade): updating vars.yaml for fragmented values Signed-off-by: Rajesh Rajendran --- scripts/helm/upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helm/upgrade.sh b/scripts/helm/upgrade.sh index 3a5df210a..1610811fd 100644 --- a/scripts/helm/upgrade.sh +++ b/scripts/helm/upgrade.sh @@ -71,7 +71,7 @@ patch(){ # Coverting caps env var to small ansible variable. # In chalice EMAIL_HOST translates to email_host in vars.yaml # Ref: https://stackoverflow.com/questions/2264428/how-to-convert-a-string-to-lower-case-in-bash - sed -i "s#${var,,}.*#${var,,}: \"$old_val\"#g" vars_template.yaml + sed -i "s#${var,,}.*#${var,,}: \"$old_val\"#g" vars.yaml done } From 14641591c7e6f98671010ebb4b25cb2e87b80b47 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 4 Dec 2021 19:01:59 +0530 Subject: [PATCH 11/12] chore(vars): reverting the template to default values. Adding default values as we've changed the template format and older versions doesn't have this variable format. Signed-off-by: Rajesh Rajendran --- scripts/helm/vars_template.yaml | 44 ++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/scripts/helm/vars_template.yaml b/scripts/helm/vars_template.yaml index d20d8aaa0..78272fbe9 100644 --- a/scripts/helm/vars_template.yaml +++ b/scripts/helm/vars_template.yaml @@ -88,25 +88,25 @@ db_resource_override: clickhouse: {{ db_resource_override.clickhouse|default({}) }} ## Sane defaults -s3_endpoint: "{{ s3_endpoint }}" -aws_region: "{{ aws_region }}" -assets_bucket: "{{ assets_bucket }}" -recordings_bucket: "{{ recordings_bucket }}" -sourcemaps_bucket: "{{ sourcemaps_bucket }}" -kafka_endpoint: "{{ kafka_endpoint }}" -kafka_ssl: "{{ kafka_ssl }}" -postgres_endpoint: "{{ postgres_endpoint }}" -postgres_port: "{{ postgres_port }}" -postgres_db_name: "{{ postgres_db_name }}" -postgres_db_user: "{{ postgres_db_user }}" -postgres_db_password: "{{ postgres_db_password }}" -redis_endpoint: "{{ redis_endpoint }}" -email_host: "{{ email_host }}" -email_port: "{{ email_port }}" -email_user: "{{ email_user }}" -email_password: "{{ email_password }}" -email_use_tls: "{{ email_use_tls }}" -email_use_ssl: "{{ email_use_ssl }}" -email_ssl_key: "{{ email_ssl_key }}" -email_ssl_cert: "{{ email_ssl_cert }}" -email_from: "{{ email_from }}" +s3_endpoint: "http://minio.db.svc.cluster.local:9000" +aws_region: "us-east-1" +assets_bucket: sessions-assets +recordings_bucket: mobs +sourcemaps_bucket: sourcemaps +kafka_endpoint: kafka.db.svc.cluster.local:9042 +kafka_ssl: 'false' +postgres_endpoint: postgresql.db.svc.cluster.local +postgres_port: 5432 +postgres_db_name: postgres +postgres_db_user: postgres +postgres_db_password: asayerPostgres +redis_endpoint: redis-master.db.svc.cluster.local:6379 +email_host: '' +email_port: '587' +email_user: '' +email_password: '' +email_use_tls: 'true' +email_use_ssl: 'false' +email_ssl_key: '' +email_ssl_cert: '' +email_from: OpenReplay From 74097d12d3ebb9f5bede1fd0afbaec283fb507cb Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 4 Dec 2021 20:37:59 +0530 Subject: [PATCH 12/12] fix(upgrade): parse previous values Signed-off-by: Rajesh Rajendran --- scripts/helm/upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helm/upgrade.sh b/scripts/helm/upgrade.sh index 1610811fd..23526cd20 100644 --- a/scripts/helm/upgrade.sh +++ b/scripts/helm/upgrade.sh @@ -67,7 +67,7 @@ patch(){ ) for var in ${vars[@]};do # Get old value - old_val=`grep $var ${openreplay_old_dir}/scripts/helm/app/chalice.yaml| cutl -d" " f4|xargs` + old_val=`grep $var ${openreplay_old_dir}/scripts/helm/app/chalice.yaml| cut -d" " -f4|xargs` # Coverting caps env var to small ansible variable. # In chalice EMAIL_HOST translates to email_host in vars.yaml # Ref: https://stackoverflow.com/questions/2264428/how-to-convert-a-string-to-lower-case-in-bash