jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

clickhouse password (#972)

Browse source 
* chore(helm): Adding clickhouse password
* chore(backend): Support clickhouse password
* chore(api): Adding clickhouse password support
* chore(helm): inject clickhouse password
* chore(helm): Updating clickhouse image
* fix(clickhouse): Removing duplicate country code

---------

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>
This commit is contained in:
Rajesh Rajendran 2023-02-08 18:47:32 +01:00 • committed by GitHub
parent 61210506d4
commit 93f27fe459
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 61 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
backend/Dockerfile
View file

@ -79,7 +79,10 @@ ENV TZ=UTC \
TOPIC_STORAGE_FAILOVER=storage-failover \
SERVICE_NAME=$SERVICE_NAME \
PROFILER_ENABLED=false \
COMPRESSION_TYPE=zstd
COMPRESSION_TYPE=zstd \
CH_USERNAME="default" \
CH_PASSWORD="" \
CH_DATABASE="default"
RUN if [ "$SERVICE_NAME" = "http" ]; then \

3
ee/api/chalicelib/utils/ch_client.py
View file

@ -20,7 +20,8 @@ class ClickHouseClient:
def __init__(self):
self.__client = clickhouse_driver.Client(host=config("ch_host"),
database="default",
database=config("ch_database",default="default", cast=str),
password=config("ch_password",default="", cast=str),
port=config("ch_port", cast=int),
settings=settings) \
if self.__client is None else self.__client

24
ee/backend/pkg/db/clickhouse/connector.go
View file

@ -3,16 +3,18 @@ package clickhouse
import (
"errors"
"fmt"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"log"
"openreplay/backend/pkg/db/types"
"openreplay/backend/pkg/hashid"
"openreplay/backend/pkg/messages"
"openreplay/backend/pkg/url"
"os"
"strings"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"openreplay/backend/pkg/license"
)
@ -38,14 +40,28 @@ type connectorImpl struct {
batches map[string]Bulk //driver.Batch
}
// Check env variables. If not present, return default value.
func getEnv(key, fallback string) string {
if value, ok := os.LookupEnv(key); ok {
return value
}
return fallback
}
func NewConnector(url string) Connector {
license.CheckLicense()
// Check username, password, database
userName := getEnv("CH_USERNAME", "default")
password := getEnv("CH_PASSWORD", "")
database := getEnv("CH_DATABASE", "default")
url = strings.TrimPrefix(url, "tcp://")
url = strings.TrimSuffix(url, "/default")
url = strings.TrimSuffix(url, "/"+database)
conn, err := clickhouse.Open(&clickhouse.Options{
Addr: []string{url},
Auth: clickhouse.Auth{
Database: "default",
Database: database,
Username: userName,
Password: password,
},
MaxOpenConns: 20,
MaxIdleConns: 15,

4
ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql
View file

@ -129,7 +129,7 @@ CREATE TABLE IF NOT EXISTS experimental.sessions
user_browser_version LowCardinality(Nullable(String)),
user_device Nullable(String),
user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2),
user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122,'BU'=123, 'VD'=124, 'YD'=125, 'DD'=126, 'BU'=127),
user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122,'BU'=123, 'VD'=124, 'YD'=125, 'DD'=126),
platform Enum8('web'=1,'ios'=2,'android'=3) DEFAULT 'web',
datetime DateTime,
duration UInt32,
@ -392,4 +392,4 @@ WHERE datetime >= now() - INTERVAL 7 DAY
-- FROM experimental.events
-- INNER JOIN experimental.sessions USING (session_id)
-- WHERE event_type = 'ERROR'
-- AND source = 'js_exception';
-- AND source = 'js_exception';

8
scripts/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml
View file

@ -29,6 +29,10 @@ spec:
containers:
- name: {{ .Chart.Name }}-backup
env:
- name: CLICKHOUSE_USER
value: "{{ .Values.username }}"
- name: CLICKHOUSE_PASSWORD
value: "{{ .Values.password }}"
- name: API_LISTEN
value: "0.0.0.0:7171"
{{- range $key, $value := .Values.backupEnv }}
@ -49,6 +53,10 @@ spec:
mountPath: /var/lib/clickhouse
- name: {{ .Chart.Name }}
env:
- name: CLICKHOUSE_USER
value: "{{ .Values.username }}"
- name: CLICKHOUSE_PASSWORD
value: "{{ .Values.password }}"
{{- range $key, $value := .Values.env }}
- name: "{{ $key }}"
value: "{{ $value }}"

3
scripts/helmcharts/databases/charts/clickhouse/values.yaml
View file

@ -15,6 +15,9 @@ backupImage:
pullPolicy: IfNotPresent
tag: "latest"
username: default
password: ""
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""

2
scripts/helmcharts/databases/values.yaml
View file

@ -137,7 +137,7 @@ kafka:
# Enterprise dbs
clickhouse:
image:
tag: "22.2.2.1"
tag: "22.12-alpine"
enabled: false
postgreql:

4
scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml
View file

@ -55,6 +55,10 @@ spec:
value: "{{ .Values.global.clickhouse.chHost }}"
- name: ch_port
value: "{{ .Values.global.clickhouse.service.webPort }}"
- name: ch_username
value: '{{ .Values.global.clickhouse.username }}'
- name: ch_password
value: '{{ .Values.global.clickhouse.passwordk }}'
- name: pg_user
value: '{{ .Values.global.postgresql.postgresqlUser }}'
- name: pg_password

4
scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml
View file

@ -43,6 +43,10 @@ spec:
{{- .Values.healthCheck | toYaml | nindent 10}}
{{- end}}
env:
- name: ch_username
value: "{{ .Values.global.clickhouse.username }}"
- name: ch_password
value: "{{ .Values.global.clickhouse.password }}"
- name: ch_host
value: "{{ .Values.global.clickhouse.chHost }}"
- name: ch_port

4
scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml
View file

@ -43,6 +43,10 @@ spec:
{{- .Values.healthCheck | toYaml | nindent 10}}
{{- end}}
env:
- name: CH_USERNAME
value: '{{ .Values.global.clickhouse.userame }}'
- name: CH_PASSWORD
value: '{{ .Values.global.clickhouse.password }}'
- name: CLICKHOUSE_STRING
value: '{{ .Values.global.clickhouse.chHost }}:{{.Values.global.clickhouse.service.webPort}}/{{.Values.env.ch_db}}'
- name: LICENSE_KEY

4
scripts/helmcharts/openreplay/files/clickhouse.sh
View file

@ -11,7 +11,7 @@ function migrate() {
echo "Migrating clickhouse version $version"
# For now, we can ignore the clickhouse db inject errors.
# TODO: Better error handling in script
clickhouse-client -h ${CH_HOST} --port ${CH_PORT} --multiquery < ${clickhousedir}/${version}/${version}.sql || true
clickhouse-client -h ${CH_HOST} --port ${CH_PORT} --user ${CH_USERNAME} --password ${CH_PASSWORD} --multiquery < ${clickhousedir}/${version}/${version}.sql || true
done
}
@ -19,7 +19,7 @@ function init() {
echo "Initializing clickhouse"
for file in `ls ${clickhousedir}/create/*.sql`; do
echo "Injecting $file"
clickhouse-client -h ${CH_HOST} --port ${CH_PORT} --multiquery < $file || true
clickhouse-client -h ${CH_HOST} --user ${CH_USERNAME} --password ${CH_PASSWORD} --port ${CH_PORT} --multiquery < $file || true
done
}

6
scripts/helmcharts/openreplay/templates/job.yaml
View file

@ -219,7 +219,7 @@ spec:
mountPath: /opt/openreplay
{{- end }}
- name: clickhouse
image: yandex/clickhouse-client:21.9.4.35
image: clickhouse/clickhouse-server:22.12-alpine
env:
- name: FORCE_MIGRATION
value: "{{ .Values.forceMigration }}"
@ -231,6 +231,10 @@ spec:
value: "{{.Values.global.clickhouse.chHost}}"
- name: CH_PORT
value: "{{.Values.global.clickhouse.service.webPort}}"
- name: CH_USERNAME
value: "{{.Values.global.clickhouse.username}}"
- name: CH_PASSWORD
value: "{{.Values.global.clickhouse.password}}"
command:
- /bin/bash
- /opt/migrations/dbops.sh

2
scripts/helmcharts/vars.yaml
View file

@ -21,6 +21,8 @@ clickhouse: &clickhouse
# For enterpriseEdition
enabled: false
chHost: clickhouse-openreplay-clickhouse.db.svc.cluster.local
username: default
password: "changeMePassword"
service:
webPort: 9000