From f64464de5847bc8aec2d1f2a261d91194d4b6505 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 12 Jul 2022 11:38:43 +0200 Subject: [PATCH 001/553] chore(helm): ingress-nginx update image to latest Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/values.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/helmcharts/openreplay/values.yaml b/scripts/helmcharts/openreplay/values.yaml index 57f3a1432..92a932aad 100644 --- a/scripts/helmcharts/openreplay/values.yaml +++ b/scripts/helmcharts/openreplay/values.yaml @@ -94,3 +94,12 @@ nginx-ingress: ingress-nginx: enabled: true + controller: + name: controller + image: + registry: k8s.gcr.io + image: ingress-nginx/controller + ## for backwards compatibility consider setting the full image url via the repository value below + ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + ## repository: + tag: "v1.2.1" From 6d45a41c18794b673b9f39be8e48b2d0d9532d6a Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 12 Jul 2022 12:20:13 +0200 Subject: [PATCH 002/553] feat(backend): set default size of first part of session mob file to 1mb --- backend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 3c3bebc3b..f7e4b3711 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -48,7 +48,7 @@ ENV TZ=UTC \ ASSETS_SIZE_LIMIT=6291456 \ ASSETS_HEADERS="{ \"Cookie\": \"ABv=3;\" }" \ FS_CLEAN_HRS=72 \ - FILE_SPLIT_SIZE=500000 \ + FILE_SPLIT_SIZE=1000000 \ LOG_QUEUE_STATS_INTERVAL_SEC=60 \ DB_BATCH_QUEUE_LIMIT=20 \ DB_BATCH_SIZE_LIMIT=10000000 \ From bae347c4ffc70f902c8020ac3ccc936b8928ce55 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 12 Jul 2022 13:52:26 +0200 Subject: [PATCH 003/553] feat(backend): added extra information for db metrics --- backend/pkg/db/postgres/connector.go | 121 ++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 23 deletions(-) diff --git a/backend/pkg/db/postgres/connector.go b/backend/pkg/db/postgres/connector.go index 4a85029fd..e772b479d 100644 --- a/backend/pkg/db/postgres/connector.go +++ b/backend/pkg/db/postgres/connector.go @@ -24,15 +24,16 @@ type batchItem struct { } type Conn struct { - c *pgxpool.Pool // TODO: conditional usage of Pool/Conn (use interface?) - batches map[uint64]*pgx.Batch - batchSizes map[uint64]int - rawBatches map[uint64][]*batchItem - batchQueueLimit int - batchSizeLimit int - batchSizeBytes syncfloat64.Histogram - batchSizeLines syncfloat64.Histogram - sqlRequestTime syncfloat64.Histogram + c *pgxpool.Pool // TODO: conditional usage of Pool/Conn (use interface?) + batches map[uint64]*pgx.Batch + batchSizes map[uint64]int + rawBatches map[uint64][]*batchItem + batchQueueLimit int + batchSizeLimit int + batchSizeBytes syncfloat64.Histogram + batchSizeLines syncfloat64.Histogram + sqlRequestTime syncfloat64.Histogram + sqlRequestCounter syncfloat64.Counter } func NewConn(url string, queueLimit, sizeLimit int, metrics *monitoring.Metrics) *Conn { @@ -75,6 +76,10 @@ func (conn *Conn) initMetrics(metrics *monitoring.Metrics) { if err != nil { log.Printf("can't create sqlRequestTime metric: %s", err) } + conn.sqlRequestCounter, err = metrics.RegisterCounter("sql_request_number") + if err != nil { + log.Printf("can't create sqlRequestNumber metric: %s", err) + } } func (conn *Conn) batchQueue(sessionID uint64, sql string, args ...interface{}) { @@ -99,6 +104,10 @@ func (conn *Conn) CommitBatches() { // Record batch size in bytes and number of lines conn.batchSizeBytes.Record(context.Background(), float64(conn.batchSizes[sessID])) conn.batchSizeLines.Record(context.Background(), float64(b.Len())) + + start := time.Now() + isFailed := false + // Send batch to db and execute br := conn.c.SendBatch(getTimeoutContext(), b) l := b.Len() @@ -108,9 +117,14 @@ func (conn *Conn) CommitBatches() { failedSql := conn.rawBatches[sessID][i] query := strings.ReplaceAll(failedSql.query, "\n", " ") log.Println("failed sql req:", query, failedSql.arguments) + isFailed = true } } br.Close() // returns err + conn.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), + attribute.String("method", "batch"), attribute.Bool("failed", isFailed)) + conn.sqlRequestCounter.Add(context.Background(), 1, + attribute.String("method", "batch"), attribute.Bool("failed", isFailed)) } conn.batches = make(map[uint64]*pgx.Batch) conn.batchSizes = make(map[uint64]int) @@ -134,6 +148,10 @@ func (conn *Conn) commitBatch(sessionID uint64) { // Record batch size in bytes and number of lines conn.batchSizeBytes.Record(context.Background(), float64(conn.batchSizes[sessionID])) conn.batchSizeLines.Record(context.Background(), float64(b.Len())) + + start := time.Now() + isFailed := false + // Send batch to db and execute br := conn.c.SendBatch(getTimeoutContext(), b) l := b.Len() @@ -143,10 +161,16 @@ func (conn *Conn) commitBatch(sessionID uint64) { failedSql := conn.rawBatches[sessionID][i] query := strings.ReplaceAll(failedSql.query, "\n", " ") log.Println("failed sql req:", query, failedSql.arguments) + isFailed = true } } br.Close() + conn.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), + attribute.String("method", "batch"), attribute.Bool("failed", isFailed)) + conn.sqlRequestCounter.Add(context.Background(), 1, + attribute.String("method", "batch"), attribute.Bool("failed", isFailed)) + // Clean batch info delete(conn.batches, sessionID) delete(conn.batchSizes, sessionID) @@ -156,61 +180,112 @@ func (conn *Conn) commitBatch(sessionID uint64) { func (conn *Conn) query(sql string, args ...interface{}) (pgx.Rows, error) { start := time.Now() res, err := conn.c.Query(getTimeoutContext(), sql, args...) - conn.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), attribute.String("method", methodName(sql))) + method, table := methodName(sql) + conn.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), + attribute.String("method", method), attribute.String("table", table)) + conn.sqlRequestCounter.Add(context.Background(), 1, + attribute.String("method", method), attribute.String("table", table)) return res, err } func (conn *Conn) queryRow(sql string, args ...interface{}) pgx.Row { start := time.Now() res := conn.c.QueryRow(getTimeoutContext(), sql, args...) - conn.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), attribute.String("method", methodName(sql))) + method, table := methodName(sql) + conn.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), + attribute.String("method", method), attribute.String("table", table)) + conn.sqlRequestCounter.Add(context.Background(), 1, + attribute.String("method", method), attribute.String("table", table)) return res } func (conn *Conn) exec(sql string, args ...interface{}) error { start := time.Now() _, err := conn.c.Exec(getTimeoutContext(), sql, args...) - conn.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), attribute.String("method", methodName(sql))) + method, table := methodName(sql) + conn.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), + attribute.String("method", method), attribute.String("table", table)) + conn.sqlRequestCounter.Add(context.Background(), 1, + attribute.String("method", method), attribute.String("table", table)) return err } type _Tx struct { pgx.Tx - sqlRequestTime syncfloat64.Histogram + sqlRequestTime syncfloat64.Histogram + sqlRequestCounter syncfloat64.Counter } func (conn *Conn) begin() (_Tx, error) { start := time.Now() tx, err := conn.c.Begin(context.Background()) - conn.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), attribute.String("method", "begin")) - return _Tx{tx, conn.sqlRequestTime}, err + conn.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), + attribute.String("method", "begin")) + conn.sqlRequestCounter.Add(context.Background(), 1, + attribute.String("method", "begin")) + return _Tx{tx, conn.sqlRequestTime, conn.sqlRequestCounter}, err } func (tx _Tx) exec(sql string, args ...interface{}) error { start := time.Now() _, err := tx.Exec(context.Background(), sql, args...) - tx.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), attribute.String("method", methodName(sql))) + method, table := methodName(sql) + tx.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), + attribute.String("method", method), attribute.String("table", table)) + tx.sqlRequestCounter.Add(context.Background(), 1, + attribute.String("method", method), attribute.String("table", table)) return err } func (tx _Tx) rollback() error { start := time.Now() err := tx.Rollback(context.Background()) - tx.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), attribute.String("method", "rollback")) + tx.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), + attribute.String("method", "rollback")) + tx.sqlRequestCounter.Add(context.Background(), 1, + attribute.String("method", "rollback")) return err } func (tx _Tx) commit() error { start := time.Now() err := tx.Commit(context.Background()) - tx.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), attribute.String("method", "commit")) + tx.sqlRequestTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()), + attribute.String("method", "commit")) + tx.sqlRequestCounter.Add(context.Background(), 1, + attribute.String("method", "commit")) return err } -func methodName(sql string) string { - method := "unknown" - if parts := strings.Split(sql, ""); len(parts) > 0 { - method = parts[0] +func methodName(sql string) (string, string) { + cmd, table := "unknown", "unknown" + + // Prepare sql request for parsing + sql = strings.TrimSpace(sql) + sql = strings.ReplaceAll(sql, "\n", " ") + sql = strings.ReplaceAll(sql, "\t", "") + sql = strings.ToLower(sql) + + // Get sql command name + parts := strings.Split(sql, " ") + if parts[0] == "" { + return cmd, table + } else { + cmd = strings.TrimSpace(parts[0]) } - return strings.ToLower(method) + + // Get table name + switch cmd { + case "select": + for i, p := range parts { + if strings.TrimSpace(p) == "from" { + table = strings.TrimSpace(parts[i+1]) + } + } + case "update": + table = strings.TrimSpace(parts[1]) + case "insert": + table = strings.TrimSpace(parts[2]) + } + return cmd, table } From 130775a968efc528056cbba8b84192fb4d25ee63 Mon Sep 17 00:00:00 2001 From: Eric Chan Date: Wed, 13 Jul 2022 11:41:43 -0400 Subject: [PATCH 004/553] avoid accessing localStorage and sessionStorage before override --- tracker/tracker/src/main/app/index.ts | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index be03a968c..81b99aae6 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -23,8 +23,8 @@ export interface StartOptions { } interface OnStartInfo { - sessionID: string, - sessionToken: string, + sessionID: string, + sessionToken: string, userUUID: string, } const CANCELED = "canceled" as const @@ -60,8 +60,8 @@ type AppOptions = { __is_snippet: boolean; __debug_report_edp: string | null; __debug__?: LoggerOptions; - localStorage: Storage; - sessionStorage: Storage; + localStorage: Storage | null; + sessionStorage: Storage | null; // @deprecated onStart?: StartCallback; @@ -117,8 +117,8 @@ export default class App { verbose: false, __is_snippet: false, __debug_report_edp: null, - localStorage: window.localStorage, - sessionStorage: window.sessionStorage, + localStorage: null, + sessionStorage: null, }, options, ); @@ -140,8 +140,8 @@ export default class App { Object.entries(metadata).forEach(([key, value]) => this.send(new Metadata(key, value))) } }) - this.localStorage = this.options.localStorage; - this.sessionStorage = this.options.sessionStorage; + this.localStorage = this.options.localStorage ?? window.localStorage; + this.sessionStorage = this.options.sessionStorage ?? window.sessionStorage; if (sessionToken != null) { this.sessionStorage.setItem(this.options.session_token_key, sessionToken); @@ -175,7 +175,7 @@ export default class App { this.attachEventListener(document.body, 'mouseleave', alertWorker, false, false); // TODO: stop session after inactivity timeout (make configurable) this.attachEventListener(document, 'visibilitychange', alertWorker, false); - } catch (e) { + } catch (e) { this._debug("worker_start", e); } } @@ -197,9 +197,9 @@ export default class App { send(message: Message, urgent = false): void { if (this.activityState === ActivityState.NotActive) { return } this.messages.push(message); - // TODO: commit on start if there were `urgent` sends; + // TODO: commit on start if there were `urgent` sends; // Clearify where urgent can be used for; - // Clearify workflow for each type of message in case it was sent before start + // Clearify workflow for each type of message in case it was sent before start // (like Fetch before start; maybe add an option "preCapture: boolean" or sth alike) if (this.activityState === ActivityState.Active && urgent) { this.commit(); @@ -339,8 +339,8 @@ export default class App { if (!this.worker) { return Promise.resolve(UnsuccessfulStart("No worker found: perhaps, CSP is not set.")) } - if (this.activityState !== ActivityState.NotActive) { - return Promise.resolve(UnsuccessfulStart("OpenReplay: trying to call `start()` on the instance that has been started already.")) + if (this.activityState !== ActivityState.NotActive) { + return Promise.resolve(UnsuccessfulStart("OpenReplay: trying to call `start()` on the instance that has been started already.")) } this.activityState = ActivityState.Starting; @@ -364,7 +364,7 @@ export default class App { this.worker.postMessage(startWorkerMsg) this.session.update({ // TODO: transparent "session" module logic AND explicit internal api for plugins. - // "updating" with old metadata in order to trigger session's UpdateCallbacks. + // "updating" with old metadata in order to trigger session's UpdateCallbacks. // (for the case of internal .start() calls, like on "restart" webworker signal or assistent connection in tracker-assist ) metadata: startOpts.metadata || this.session.getInfo().metadata, userID: startOpts.userID, @@ -391,7 +391,7 @@ export default class App { if (r.status === 200) { return r.json() } else { - return r.text().then(text => text === CANCELED + return r.text().then(text => text === CANCELED ? Promise.reject(CANCELED) : Promise.reject(`Server error: ${r.status}. ${text}`) ); @@ -418,7 +418,7 @@ export default class App { this.worker.postMessage(startWorkerMsg) this.activityState = ActivityState.Active - + const onStartInfo = { sessionToken: token, userUUID, sessionID }; this.startCallbacks.forEach((cb) => cb(onStartInfo)); // TODO: start as early as possible (before receiving the token) @@ -432,7 +432,7 @@ export default class App { } return SuccessfulStart(onStartInfo) }) - .catch(reason => { + .catch(reason => { this.sessionStorage.removeItem(this.options.session_token_key) this.stop() if (reason === CANCELED) { return UnsuccessfulStart(CANCELED) } From 285d4cf0e58f2926a574d232a5d10a36bd4f4d8f Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 13 Jul 2022 20:25:33 +0200 Subject: [PATCH 005/553] fix(helm): timings for cron jobs Signed-off-by: rjshrjndrn --- .../openreplay/charts/utilities/templates/report-cron.yaml | 2 +- .../charts/utilities/templates/sessions-cleaner-cron.yaml | 2 +- .../openreplay/charts/utilities/templates/telemetry-cron.yaml | 2 +- scripts/helmcharts/openreplay/charts/utilities/values.yaml | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/report-cron.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/report-cron.yaml index 4a321c20e..966a47247 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/report-cron.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/templates/report-cron.yaml @@ -5,7 +5,7 @@ metadata: name: report-cron namespace: {{ .Release.Namespace }} spec: - schedule: "{{ .Values.cron }}" + schedule: "{{ .Values.report.cron }}" failedJobsHistoryLimit: 1 successfulJobsHistoryLimit: 1 jobTemplate: diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/sessions-cleaner-cron.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/sessions-cleaner-cron.yaml index 7a916b4c1..9313e88af 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/sessions-cleaner-cron.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/templates/sessions-cleaner-cron.yaml @@ -5,7 +5,7 @@ metadata: name: sessions-cleaner-cron namespace: {{ .Release.Namespace }} spec: - schedule: "{{ .Values.cron }}" + schedule: "{{ .Values.sessionsCleaner.cron }}" failedJobsHistoryLimit: 1 successfulJobsHistoryLimit: 1 jobTemplate: diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/telemetry-cron.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/telemetry-cron.yaml index ca7dc03c4..95fd26188 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/telemetry-cron.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/templates/telemetry-cron.yaml @@ -5,7 +5,7 @@ metadata: name: telemetry-cron namespace: {{ .Release.Namespace }} spec: - schedule: "{{ .Values.cron }}" + schedule: "{{ .Values.telemetry.cron }}" failedJobsHistoryLimit: 1 successfulJobsHistoryLimit: 1 jobTemplate: diff --git a/scripts/helmcharts/openreplay/charts/utilities/values.yaml b/scripts/helmcharts/openreplay/charts/utilities/values.yaml index 8df1560d0..e973cbeec 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/values.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/values.yaml @@ -35,6 +35,7 @@ report: sessionsCleaner: # https://crontab.guru/#5_1_*_*_* # Midnight 1.05 + cron: "5 1 * * *" image: repository: "{{ .Values.global.openReplayContainerRegistry }}/crons" pullPolicy: IfNotPresent From 9e78bf742e0dc3cd6c606e3c9aae6fc34e2f47cf Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 13 Jul 2022 20:37:27 +0200 Subject: [PATCH 006/553] chore(cron): pull image always Signed-off-by: rjshrjndrn --- .../helmcharts/openreplay/charts/utilities/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/utilities/values.yaml b/scripts/helmcharts/openreplay/charts/utilities/values.yaml index e973cbeec..7bfa67523 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/values.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/values.yaml @@ -6,7 +6,7 @@ replicaCount: 1 image: repository: "{{ .Values.global.openReplayContainerRegistry }}/utilities" - pullPolicy: IfNotPresent + pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. tag: "" @@ -16,7 +16,7 @@ telemetry: cron: "5 12 * * *" image: repository: "{{ .Values.global.openReplayContainerRegistry }}/crons" - pullPolicy: IfNotPresent + pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. tag: "" env: @@ -27,7 +27,7 @@ report: cron: "0 5 * * 1" image: repository: "{{ .Values.global.openReplayContainerRegistry }}/crons" - pullPolicy: IfNotPresent + pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. tag: "" env: @@ -38,7 +38,7 @@ sessionsCleaner: cron: "5 1 * * *" image: repository: "{{ .Values.global.openReplayContainerRegistry }}/crons" - pullPolicy: IfNotPresent + pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. tag: "" env: From ae53f2b139b125a26dabc172cc52d5d800f988b1 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 19 Jul 2022 13:48:55 +0200 Subject: [PATCH 007/553] feat(chalice): ch sessions search --- ee/api/.gitignore | 1 - ee/api/chalicelib/core/sessions.py | 2105 ++++++++++++ ee/api/clean.sh | 1 - ee/api/routers/core_dynamic.py | 10 + .../db/init_dbs/clickhouse/1.8.0/1.8.0.sql | 258 ++ .../clickhouse/1.8.0/__clickhouse.csv | 68 + .../db/init_dbs/clickhouse/1.8.0/fill.sql | 2878 +++++++++++++++++ .../db/init_dbs/clickhouse/1.8.0/queries.sql | 983 ++++++ 8 files changed, 6302 insertions(+), 2 deletions(-) create mode 100644 ee/api/chalicelib/core/sessions.py create mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql create mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/__clickhouse.csv create mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql create mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/queries.sql diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 12a468ef1..a0ca015ed 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -205,7 +205,6 @@ Pipfile /chalicelib/core/log_tool_sumologic.py /chalicelib/core/metadata.py /chalicelib/core/mobile.py -/chalicelib/core/sessions.py /chalicelib/core/sessions_assignments.py /chalicelib/core/sessions_metas.py /chalicelib/core/sessions_mobs.py diff --git a/ee/api/chalicelib/core/sessions.py b/ee/api/chalicelib/core/sessions.py new file mode 100644 index 000000000..f2725188d --- /dev/null +++ b/ee/api/chalicelib/core/sessions.py @@ -0,0 +1,2105 @@ +from typing import List + +import schemas +import schemas_ee +from chalicelib.core import events, metadata, events_ios, \ + sessions_mobs, issues, projects, errors, resources, assist, performance_event +from chalicelib.utils import pg_client, helper, metrics_helper, ch_client + +SESSION_PROJECTION_COLS = """\ +s.project_id, +s.session_id::text AS session_id, +s.user_uuid, +s.user_id, +s.user_os, +s.user_browser, +s.user_device, +s.user_device_type, +s.user_country, +s.start_ts, +s.duration, +s.events_count, +s.pages_count, +s.errors_count, +s.user_anonymous_id, +s.platform, +s.issue_score, +to_jsonb(s.issue_types) AS issue_types, +favorite_sessions.session_id NOTNULL AS favorite, +COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + """ + +SESSION_PROJECTION_COLS_CH = """\ +s.project_id, +s.session_id AS session_id, +s.user_uuid AS user_uuid, +s.user_id AS user_id, +s.user_os AS user_os, +s.user_browser AS user_browser, +s.user_device AS user_device, +s.user_device_type AS user_device_type, +s.user_country AS user_country, +toUnixTimestamp(s.datetime)*1000 AS start_ts, +s.duration AS duration, +s.events_count AS events_count, +s.pages_count AS pages_count, +s.errors_count AS errors_count, +s.user_anonymous_id AS user_anonymous_id, +s.platform AS platform, +0 AS issue_score, +-- , +-- to_jsonb(s.issue_types) AS issue_types, +isNotNull(favorite_sessions.session_id) AS favorite, +-- COALESCE((SELECT TRUE +-- FROM public.user_viewed_sessions AS fs +-- WHERE s.session_id = fs.session_id +-- AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + """ + +SESSION_PROJECTION_COLS_CH_MAP = """\ +'project_id', toString(%(project_id)s), +'session_id', toString(s.session_id), +'user_uuid', toString(s.user_uuid), +'user_id', toString(s.user_id), +'user_os', toString(s.user_os), +'user_browser', toString(s.user_browser), +'user_device', toString(s.user_device), +'user_device_type', toString(s.user_device_type), +'user_country', toString(s.user_country), +'start_ts', toString(toUnixTimestamp(s.datetime)*1000), +'duration', toString(s.duration), +'events_count', toString(s.events_count), +'pages_count', toString(s.pages_count), +'errors_count', toString(s.errors_count), +'user_anonymous_id', toString(s.user_anonymous_id), +'platform', toString(s.platform), +'issue_score', '0', +'favorite', toString(isNotNull(favorite_sessions.session_id)) +""" + + +def __group_metadata(session, project_metadata): + meta = {} + for m in project_metadata.keys(): + if project_metadata[m] is not None and session.get(m) is not None: + meta[project_metadata[m]] = session[m] + session.pop(m) + return meta + + +def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_viewed=False, group_metadata=False, + live=True): + with pg_client.PostgresClient() as cur: + extra_query = [] + if include_fav_viewed: + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite""") + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS viewed""") + query = cur.mogrify( + f"""\ + SELECT + s.*, + s.session_id::text AS session_id, + (SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key + {"," if len(extra_query) > 0 else ""}{",".join(extra_query)} + {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata._get_column_names()]) + ") AS project_metadata") if group_metadata else ''} + FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""} + WHERE s.project_id = %(project_id)s + AND s.session_id = %(session_id)s;""", + {"project_id": project_id, "session_id": session_id, "userId": user_id} + ) + # print("===============") + # print(query) + cur.execute(query=query) + + data = cur.fetchone() + if data is not None: + data = helper.dict_to_camel_case(data) + if full_data: + if data["platform"] == 'ios': + data['events'] = events_ios.get_by_sessionId(project_id=project_id, session_id=session_id) + for e in data['events']: + if e["type"].endswith("_IOS"): + e["type"] = e["type"][:-len("_IOS")] + data['crashes'] = events_ios.get_crashes_by_session_id(session_id=session_id) + data['userEvents'] = events_ios.get_customs_by_sessionId(project_id=project_id, + session_id=session_id) + data['mobsUrl'] = sessions_mobs.get_ios(sessionId=session_id) + else: + data['events'] = events.get_by_sessionId2_pg(project_id=project_id, session_id=session_id, + group_clickrage=True) + all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) + data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] + # to keep only the first stack + data['errors'] = [errors.format_first_stack_frame(e) for e in all_errors if + e['source'] == "js_exception"][ + :500] # limit the number of errors to reduce the response-body size + data['userEvents'] = events.get_customs_by_sessionId2_pg(project_id=project_id, + session_id=session_id) + data['mobsUrl'] = sessions_mobs.get_web(sessionId=session_id) + data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, + start_ts=data["startTs"], + duration=data["duration"]) + + data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) + data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) + data['live'] = live and assist.is_live(project_id=project_id, + session_id=session_id, + project_key=data["projectKey"]) + data["inDB"] = True + return data + elif live: + return assist.get_live_session_by_id(project_id=project_id, session_id=session_id) + else: + return None + + +def __get_sql_operator(op: schemas.SearchEventOperator): + return { + schemas.SearchEventOperator._is: "=", + schemas.SearchEventOperator._is_any: "IN", + schemas.SearchEventOperator._on: "=", + schemas.SearchEventOperator._on_any: "IN", + schemas.SearchEventOperator._is_not: "!=", + schemas.SearchEventOperator._not_on: "!=", + schemas.SearchEventOperator._contains: "ILIKE", + schemas.SearchEventOperator._not_contains: "NOT ILIKE", + schemas.SearchEventOperator._starts_with: "ILIKE", + schemas.SearchEventOperator._ends_with: "ILIKE", + }.get(op, "=") + + +def __is_negation_operator(op: schemas.SearchEventOperator): + return op in [schemas.SearchEventOperator._is_not, + schemas.SearchEventOperator._not_on, + schemas.SearchEventOperator._not_contains] + + +def __reverse_sql_operator(op): + return "=" if op == "!=" else "!=" if op == "=" else "ILIKE" if op == "NOT ILIKE" else "NOT ILIKE" + + +def __get_sql_operator_multiple(op: schemas.SearchEventOperator): + return " IN " if op not in [schemas.SearchEventOperator._is_not, schemas.SearchEventOperator._not_on, + schemas.SearchEventOperator._not_contains] else " NOT IN " + + +def __get_sql_value_multiple(values): + if isinstance(values, tuple): + return values + return tuple(values) if isinstance(values, list) else (values,) + + +def _multiple_conditions(condition, values, value_key="value", is_not=False): + query = [] + for i in range(len(values)): + k = f"{value_key}_{i}" + query.append(condition.replace(value_key, k)) + return "(" + (" AND " if is_not else " OR ").join(query) + ")" + + +def _multiple_values(values, value_key="value"): + query_values = {} + if values is not None and isinstance(values, list): + for i in range(len(values)): + k = f"{value_key}_{i}" + query_values[k] = values[i] + return query_values + + +def _isAny_opreator(op: schemas.SearchEventOperator): + return op in [schemas.SearchEventOperator._on_any, schemas.SearchEventOperator._is_any] + + +def _isUndefined_operator(op: schemas.SearchEventOperator): + return op in [schemas.SearchEventOperator._is_undefined] + + +def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, + error_status=schemas.ErrorStatus.all, count_only=False, issue=None): + full_args, query_part = search_query_parts(data=data, error_status=error_status, errors_only=errors_only, + favorite_only=data.bookmarked, issue=issue, project_id=project_id, + user_id=user_id) + if data.limit is not None and data.page is not None: + full_args["sessions_limit_s"] = (data.page - 1) * data.limit + full_args["sessions_limit_e"] = data.page * data.limit + else: + full_args["sessions_limit_s"] = 1 + full_args["sessions_limit_e"] = 200 + + meta_keys = [] + with pg_client.PostgresClient() as cur: + if errors_only: + main_query = cur.mogrify(f"""SELECT DISTINCT er.error_id, ser.status, ser.parent_error_id, ser.payload, + COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite, + COALESCE((SELECT TRUE + FROM public.user_viewed_errors AS ve + WHERE er.error_id = ve.error_id + AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + {query_part};""", full_args) + + elif count_only: + main_query = cur.mogrify(f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, + COUNT(DISTINCT s.user_uuid) AS count_users + {query_part};""", full_args) + elif data.group_by_user: + g_sort = "count(full_sessions)" + if data.order is None: + data.order = schemas.SortOrderType.desc + else: + data.order = data.order.upper() + if data.sort is not None and data.sort != 'sessionsCount': + sort = helper.key_to_snake_case(data.sort) + g_sort = f"{'MIN' if data.order == schemas.SortOrderType.desc else 'MAX'}({sort})" + else: + sort = 'start_ts' + + meta_keys = metadata.get(project_id=project_id) + main_query = cur.mogrify(f"""SELECT COUNT(*) AS count, + COALESCE(JSONB_AGG(users_sessions) + FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions + FROM (SELECT user_id, + count(full_sessions) AS user_sessions_count, + jsonb_agg(full_sessions) FILTER (WHERE rn <= 1) AS last_session, + MIN(full_sessions.start_ts) AS first_session_ts, + ROW_NUMBER() OVER (ORDER BY {g_sort} {data.order}) AS rn + FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY {sort} {data.order}) AS rn + FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} + {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} + {query_part} + ) AS filtred_sessions + ) AS full_sessions + GROUP BY user_id + ) AS users_sessions;""", + full_args) + else: + if data.order is None: + data.order = schemas.SortOrderType.desc + sort = 'session_id' + if data.sort is not None and data.sort != "session_id": + # sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) + sort = helper.key_to_snake_case(data.sort) + + meta_keys = metadata.get(project_id=project_id) + main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, + COALESCE(JSONB_AGG(full_sessions) + FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions + FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY {sort} {data.order}, issue_score DESC) AS rn + FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} + {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} + {query_part} + ORDER BY s.session_id desc) AS filtred_sessions + ORDER BY {sort} {data.order}, issue_score DESC) AS full_sessions;""", + full_args) + # print("--------------------") + # print(main_query) + # print("--------------------") + try: + cur.execute(main_query) + except Exception as err: + print("--------- SESSIONS SEARCH QUERY EXCEPTION -----------") + print(main_query.decode('UTF-8')) + print("--------- PAYLOAD -----------") + print(data.json()) + print("--------------------") + raise err + if errors_only: + return helper.list_to_camel_case(cur.fetchall()) + + sessions = cur.fetchone() + if count_only: + return helper.dict_to_camel_case(sessions) + + total = sessions["count"] + sessions = sessions["sessions"] + + if data.group_by_user: + for i, s in enumerate(sessions): + sessions[i] = {**s.pop("last_session")[0], **s} + sessions[i].pop("rn") + sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ + if sessions[i][f'metadata_{k["index"]}'] is not None} + else: + for i, s in enumerate(sessions): + sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ + if sessions[i][f'metadata_{k["index"]}'] is not None} + # if not data.group_by_user and data.sort is not None and data.sort != "session_id": + # sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data.sort)], + # reverse=data.order.upper() == "DESC") + return { + 'total': total, + 'sessions': helper.list_to_camel_case(sessions) + } + + +def search2_ch(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, + error_status=schemas.ErrorStatus.all, count_only=False, issue=None): + full_args, query_part = search_query_parts_ch(data=data, error_status=error_status, errors_only=errors_only, + favorite_only=data.bookmarked, issue=issue, project_id=project_id, + user_id=user_id) + if data.sort == "startTs": + data.sort = "datetime" + if data.limit is not None and data.page is not None: + full_args["sessions_limit_s"] = (data.page - 1) * data.limit + full_args["sessions_limit_e"] = data.page * data.limit + full_args["sessions_limit"] = data.limit + else: + full_args["sessions_limit_s"] = 1 + full_args["sessions_limit_e"] = 200 + full_args["sessions_limit"] = 200 + + meta_keys = [] + with ch_client.ClickHouseClient() as cur: + if errors_only: + main_query = cur.mogrify(f"""SELECT DISTINCT er.error_id, ser.status, ser.parent_error_id, ser.payload, + COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite, + COALESCE((SELECT TRUE + FROM public.user_viewed_errors AS ve + WHERE er.error_id = ve.error_id + AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + {query_part};""", full_args) + + elif count_only: + main_query = cur.mogrify(f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, + COUNT(DISTINCT s.user_uuid) AS count_users + {query_part};""", full_args) + elif data.group_by_user: + g_sort = "count(full_sessions)" + if data.order is None: + data.order = schemas.SortOrderType.desc + else: + data.order = data.order.upper() + if data.sort is not None and data.sort != 'sessionsCount': + sort = helper.key_to_snake_case(data.sort) + g_sort = f"{'MIN' if data.order == schemas.SortOrderType.desc else 'MAX'}({sort})" + else: + sort = 'start_ts' + + meta_keys = metadata.get(project_id=project_id) + main_query = cur.mogrify(f"""SELECT COUNT(*) AS count, + COALESCE(JSONB_AGG(users_sessions) + FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions + FROM (SELECT user_id, + count(full_sessions) AS user_sessions_count, + jsonb_agg(full_sessions) FILTER (WHERE rn <= 1) AS last_session, + MIN(full_sessions.start_ts) AS first_session_ts, + ROW_NUMBER() OVER (ORDER BY {g_sort} {data.order}) AS rn + FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY {sort} {data.order}) AS rn + FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} + {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} + {query_part} + ) AS filtred_sessions + ) AS full_sessions + GROUP BY user_id + ) AS users_sessions;""", + full_args) + else: + if data.order is None: + data.order = schemas.SortOrderType.desc + sort = 'session_id' + if data.sort is not None and data.sort != "session_id": + # sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) + sort = helper.key_to_snake_case(data.sort) + + meta_keys = metadata.get(project_id=project_id) + main_query = cur.format(f"""SELECT any(total) AS count, groupArray(%(sessions_limit)s)(details) AS sessions + FROM (SELECT COUNT() OVER () AS total, + rowNumberInAllBlocks() AS rn, + map({SESSION_PROJECTION_COLS_CH_MAP}) AS details + {query_part} +-- ORDER BY {sort} {data.order} + ) AS raw + WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s;""", full_args) + print("--------------------") + print(main_query) + print("--------------------") + try: + sessions = cur.execute(main_query) + except Exception as err: + print("--------- SESSIONS SEARCH QUERY EXCEPTION -----------") + print(main_query) + print("--------- PAYLOAD -----------") + print(data.json()) + print("--------------------") + raise err + if errors_only: + return helper.list_to_camel_case(cur.fetchall()) + + if len(sessions) > 0: + sessions = sessions[0] + # if count_only: + # return helper.dict_to_camel_case(sessions) + # for s in sessions: + # print(s) + # s["session_id"] = str(s["session_id"]) + total = sessions["count"] + sessions = sessions["sessions"] + + if data.group_by_user: + for i, s in enumerate(sessions): + sessions[i] = {**s.pop("last_session")[0], **s} + sessions[i].pop("rn") + sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ + if sessions[i][f'metadata_{k["index"]}'] is not None} + else: + for i in range(len(sessions)): + sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ + if sessions[i].get(f'metadata_{k["index"]}') is not None} + sessions[i] = schemas_ee.SessionModel.parse_obj(helper.dict_to_camel_case(sessions[i])) + + # if not data.group_by_user and data.sort is not None and data.sort != "session_id": + # sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data.sort)], + # reverse=data.order.upper() == "DESC") + return { + 'total': total, + 'sessions': sessions + } + + +def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, density: int, + view_type: schemas.MetricTimeseriesViewType, metric_type: schemas.MetricType, + metric_of: schemas.TableMetricOfType, metric_value: List): + step_size = int(metrics_helper.__get_step_size(endTimestamp=data.endDate, startTimestamp=data.startDate, + density=density, factor=1, decimal=True)) + extra_event = None + if metric_of == schemas.TableMetricOfType.visited_url: + extra_event = "events.pages" + elif metric_of == schemas.TableMetricOfType.issues and len(metric_value) > 0: + data.filters.append(schemas.SessionSearchFilterSchema(value=metric_value, type=schemas.FilterType.issue, + operator=schemas.SearchEventOperator._is)) + full_args, query_part = search_query_parts(data=data, error_status=None, errors_only=False, + favorite_only=False, issue=None, project_id=project_id, + user_id=None, extra_event=extra_event) + full_args["step_size"] = step_size + sessions = [] + with pg_client.PostgresClient() as cur: + if metric_type == schemas.MetricType.timeseries: + if view_type == schemas.MetricTimeseriesViewType.line_chart: + main_query = cur.mogrify(f"""WITH full_sessions AS (SELECT DISTINCT ON(s.session_id) s.session_id, s.start_ts + {query_part}) + SELECT generated_timestamp AS timestamp, + COUNT(s) AS count + FROM generate_series(%(startDate)s, %(endDate)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT 1 AS s + FROM full_sessions + WHERE start_ts >= generated_timestamp + AND start_ts <= generated_timestamp + %(step_size)s) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""", full_args) + else: + main_query = cur.mogrify(f"""SELECT count(DISTINCT s.session_id) AS count + {query_part};""", full_args) + + # print("--------------------") + # print(main_query) + # print("--------------------") + cur.execute(main_query) + if view_type == schemas.MetricTimeseriesViewType.line_chart: + sessions = cur.fetchall() + else: + sessions = cur.fetchone()["count"] + elif metric_type == schemas.MetricType.table: + if isinstance(metric_of, schemas.TableMetricOfType): + main_col = "user_id" + extra_col = "" + extra_where = "" + pre_query = "" + if metric_of == schemas.TableMetricOfType.user_country: + main_col = "user_country" + elif metric_of == schemas.TableMetricOfType.user_device: + main_col = "user_device" + elif metric_of == schemas.TableMetricOfType.user_browser: + main_col = "user_browser" + elif metric_of == schemas.TableMetricOfType.issues: + main_col = "issue" + extra_col = f", UNNEST(s.issue_types) AS {main_col}" + if len(metric_value) > 0: + extra_where = [] + for i in range(len(metric_value)): + arg_name = f"selected_issue_{i}" + extra_where.append(f"{main_col} = %({arg_name})s") + full_args[arg_name] = metric_value[i] + extra_where = f"WHERE ({' OR '.join(extra_where)})" + elif metric_of == schemas.TableMetricOfType.visited_url: + main_col = "path" + extra_col = ", path" + main_query = cur.mogrify(f"""{pre_query} + SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS values + FROM (SELECT {main_col} AS name, + count(full_sessions) AS session_count, + ROW_NUMBER() OVER (ORDER BY count(full_sessions) DESC) AS rn + FROM (SELECT * + FROM (SELECT DISTINCT ON(s.session_id) s.session_id, s.user_uuid, + s.user_id, s.user_os, + s.user_browser, s.user_device, + s.user_device_type, s.user_country, s.issue_types{extra_col} + {query_part} + ORDER BY s.session_id desc) AS filtred_sessions + ) AS full_sessions + {extra_where} + GROUP BY {main_col} + ORDER BY session_count DESC) AS users_sessions;""", + full_args) + # print("--------------------") + # print(main_query) + # print("--------------------") + cur.execute(main_query) + sessions = cur.fetchone() + for s in sessions["values"]: + s.pop("rn") + sessions["values"] = helper.list_to_camel_case(sessions["values"]) + + return sessions + + +def __is_valid_event(is_any: bool, event: schemas._SessionSearchEventSchema): + return not (not is_any and len(event.value) == 0 and event.type not in [schemas.EventType.request_details, + schemas.EventType.graphql_details] \ + or event.type in [schemas.PerformanceEventType.location_dom_complete, + schemas.PerformanceEventType.location_largest_contentful_paint_time, + schemas.PerformanceEventType.location_ttfb, + schemas.PerformanceEventType.location_avg_cpu_load, + schemas.PerformanceEventType.location_avg_memory_usage + ] and (event.source is None or len(event.source) == 0) \ + or event.type in [schemas.EventType.request_details, schemas.EventType.graphql_details] and ( + event.filters is None or len(event.filters) == 0)) + + +def search_query_parts(data, error_status, errors_only, favorite_only, issue, project_id, user_id, extra_event=None): + ss_constraints = [] + full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, + "projectId": project_id, "userId": user_id} + extra_constraints = [ + "s.project_id = %(project_id)s", + "s.duration IS NOT NULL" + ] + extra_from = "" + events_query_part = "" + if len(data.filters) > 0: + meta_keys = None + for i, f in enumerate(data.filters): + if not isinstance(f.value, list): + f.value = [f.value] + filter_type = f.type + f.value = helper.values_for_operator(value=f.value, op=f.operator) + f_k = f"f_value{i}" + full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} + op = __get_sql_operator(f.operator) \ + if filter_type not in [schemas.FilterType.events_count] else f.operator + is_any = _isAny_opreator(f.operator) + is_undefined = _isUndefined_operator(f.operator) + if not is_any and not is_undefined and len(f.value) == 0: + continue + is_not = False + if __is_negation_operator(f.operator): + is_not = True + if filter_type == schemas.FilterType.user_browser: + if is_any: + extra_constraints.append('s.user_browser IS NOT NULL') + ss_constraints.append('ms.user_browser IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: + if is_any: + extra_constraints.append('s.user_os IS NOT NULL') + ss_constraints.append('ms.user_os IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: + if is_any: + extra_constraints.append('s.user_device IS NOT NULL') + ss_constraints.append('ms.user_device IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: + if is_any: + extra_constraints.append('s.user_country IS NOT NULL') + ss_constraints.append('ms.user_country IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.utm_source]: + if is_any: + extra_constraints.append('s.utm_source IS NOT NULL') + ss_constraints.append('ms.utm_source IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.utm_source IS NULL') + ss_constraints.append('ms.utm_source IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_medium]: + if is_any: + extra_constraints.append('s.utm_medium IS NOT NULL') + ss_constraints.append('ms.utm_medium IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.utm_medium IS NULL') + ss_constraints.append('ms.utm_medium IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_campaign]: + if is_any: + extra_constraints.append('s.utm_campaign IS NOT NULL') + ss_constraints.append('ms.utm_campaign IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.utm_campaign IS NULL') + ss_constraints.append('ms.utm_campaign IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, + value_key=f_k)) + + elif filter_type == schemas.FilterType.duration: + if len(f.value) > 0 and f.value[0] is not None: + extra_constraints.append("s.duration >= %(minDuration)s") + ss_constraints.append("ms.duration >= %(minDuration)s") + full_args["minDuration"] = f.value[0] + if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: + extra_constraints.append("s.duration <= %(maxDuration)s") + ss_constraints.append("ms.duration <= %(maxDuration)s") + full_args["maxDuration"] = f.value[1] + elif filter_type == schemas.FilterType.referrer: + extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" + if is_any: + extra_constraints.append('p.base_referrer IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f"p.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + elif filter_type == events.event_type.METADATA.ui_type: + # get metadata list only if you need it + if meta_keys is None: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + if f.source in meta_keys.keys(): + if is_any: + extra_constraints.append(f"s.{metadata.index_to_colname(meta_keys[f.source])} IS NOT NULL") + ss_constraints.append(f"ms.{metadata.index_to_colname(meta_keys[f.source])} IS NOT NULL") + elif is_undefined: + extra_constraints.append(f"s.{metadata.index_to_colname(meta_keys[f.source])} IS NULL") + ss_constraints.append(f"ms.{metadata.index_to_colname(meta_keys[f.source])} IS NULL") + else: + extra_constraints.append( + _multiple_conditions( + f"s.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text", + f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions( + f"ms.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text", + f.value, is_not=is_not, value_key=f_k)) + elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: + if is_any: + extra_constraints.append('s.user_id IS NOT NULL') + ss_constraints.append('ms.user_id IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.user_id IS NULL') + ss_constraints.append('ms.user_id IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f"s.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + elif filter_type in [schemas.FilterType.user_anonymous_id, + schemas.FilterType.user_anonymous_id_ios]: + if is_any: + extra_constraints.append('s.user_anonymous_id IS NOT NULL') + ss_constraints.append('ms.user_anonymous_id IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.user_anonymous_id IS NULL') + ss_constraints.append('ms.user_anonymous_id IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: + if is_any: + extra_constraints.append('s.rev_id IS NOT NULL') + ss_constraints.append('ms.rev_id IS NOT NULL') + elif is_undefined: + extra_constraints.append('s.rev_id IS NULL') + ss_constraints.append('ms.rev_id IS NULL') + else: + extra_constraints.append( + _multiple_conditions(f"s.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) + elif filter_type == schemas.FilterType.platform: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.issue: + if is_any: + extra_constraints.append("array_length(s.issue_types, 1) > 0") + ss_constraints.append("array_length(ms.issue_types, 1) > 0") + else: + extra_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.events_count: + extra_constraints.append( + _multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + # --------------------------------------------------------------------------- + if len(data.events) > 0: + valid_events_count = 0 + for event in data.events: + is_any = _isAny_opreator(event.operator) + if not isinstance(event.value, list): + event.value = [event.value] + if __is_valid_event(is_any=is_any, event=event): + valid_events_count += 1 + events_query_from = [] + event_index = 0 + or_events = data.events_order == schemas.SearchEventOrder._or + # events_joiner = " FULL JOIN " if or_events else " INNER JOIN LATERAL " + events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " + for i, event in enumerate(data.events): + event_type = event.type + is_any = _isAny_opreator(event.operator) + if not isinstance(event.value, list): + event.value = [event.value] + if not __is_valid_event(is_any=is_any, event=event): + continue + op = __get_sql_operator(event.operator) + is_not = False + if __is_negation_operator(event.operator): + is_not = True + op = __reverse_sql_operator(op) + if event_index == 0 or or_events: + event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" + event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", + "main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s", + "ms.start_ts <= %(endDate)s", "ms.duration IS NOT NULL"] + if favorite_only and not errors_only: + event_from += "INNER JOIN public.user_favorite_sessions AS fs USING(session_id)" + event_where.append("fs.user_id = %(userId)s") + else: + event_from = "%s" + event_where = ["main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", + "main.session_id=event_0.session_id"] + if data.events_order == schemas.SearchEventOrder._then: + event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") + e_k = f"e_value{i}" + s_k = e_k + "_source" + if event.type != schemas.PerformanceEventType.time_between_events: + event.value = helper.values_for_operator(value=event.value, op=event.operator) + full_args = {**full_args, + **_multiple_values(event.value, value_key=e_k), + **_multiple_values(event.source, value_key=s_k)} + + if event_type == events.event_type.CLICK.ui_type: + event_from = event_from % f"{events.event_type.CLICK.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, + value_key=e_k)) + + elif event_type == events.event_type.INPUT.ui_type: + event_from = event_from % f"{events.event_type.INPUT.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, + value_key=e_k)) + if event.source is not None and len(event.source) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, + value_key=f"custom{i}")) + full_args = {**full_args, **_multiple_values(event.source, value_key=f"custom{i}")} + + elif event_type == events.event_type.LOCATION.ui_type: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.CUSTOM.ui_type: + event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.REQUEST.ui_type: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.GRAPHQL.ui_type: + event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.STATEACTION.ui_type: + event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.ERROR.ui_type: + event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" + event.source = tuple(event.source) + if not is_any and event.value not in [None, "*", ""]: + event_where.append( + _multiple_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) + if event.source[0] not in [None, "*", ""]: + event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.value, value_key=s_k)) + + + # ----- IOS + elif event_type == events.event_type.CLICK_IOS.ui_type: + event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + + elif event_type == events.event_type.INPUT_IOS.ui_type: + event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + if event.source is not None and len(event.source) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, + value_key="custom{i}")) + full_args = {**full_args, **_multiple_values(event.source, f"custom{i}")} + elif event_type == events.event_type.VIEW_IOS.ui_type: + event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.CUSTOM_IOS.ui_type: + event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.REQUEST_IOS.ui_type: + event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.ERROR_IOS.ui_type: + event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" + if not is_any and event.value not in [None, "*", ""]: + event_where.append( + _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.fetch_failed: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + event.value, value_key=e_k)) + col = performance_event.get_col(event_type) + colname = col["column"] + event_where.append(f"main.{colname} = FALSE") + # elif event_type == schemas.PerformanceEventType.fetch_duration: + # event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + # if not is_any: + # event_where.append( + # _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + # event.value, value_key=e_k)) + # col = performance_event.get_col(event_type) + # colname = col["column"] + # tname = "main" + # e_k += "_custom" + # full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + # _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + # event.source, value_key=e_k)) + elif event_type in [schemas.PerformanceEventType.location_dom_complete, + schemas.PerformanceEventType.location_largest_contentful_paint_time, + schemas.PerformanceEventType.location_ttfb, + schemas.PerformanceEventType.location_avg_cpu_load, + schemas.PerformanceEventType.location_avg_memory_usage + ]: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + col = performance_event.get_col(event_type) + colname = col["column"] + tname = "main" + if col.get("extraJoin") is not None: + tname = "ej" + event_from += f" INNER JOIN {col['extraJoin']} AS {tname} USING(session_id)" + event_where += [f"{tname}.timestamp >= main.timestamp", f"{tname}.timestamp >= %(startDate)s", + f"{tname}.timestamp <= %(endDate)s"] + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + + event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.time_between_events: + event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " + if not isinstance(event.value[0].value, list): + event.value[0].value = [event.value[0].value] + if not isinstance(event.value[1].value, list): + event.value[1].value = [event.value[1].value] + event.value[0].value = helper.values_for_operator(value=event.value[0].value, + op=event.value[0].operator) + event.value[1].value = helper.values_for_operator(value=event.value[1].value, + op=event.value[0].operator) + e_k1 = e_k + "_e1" + e_k2 = e_k + "_e2" + full_args = {**full_args, + **_multiple_values(event.value[0].value, value_key=e_k1), + **_multiple_values(event.value[1].value, value_key=e_k2)} + s_op = __get_sql_operator(event.value[0].operator) + event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] + if event_index > 0 and not or_events: + event_where.append("main2.session_id=event_0.session_id") + is_any = _isAny_opreator(event.value[0].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", + event.value[0].value, value_key=e_k1)) + s_op = __get_sql_operator(event.value[1].operator) + is_any = _isAny_opreator(event.value[1].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", + event.value[1].value, value_key=e_k2)) + + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + event_where.append( + _multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) + + elif event_type == schemas.EventType.request_details: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + apply = False + for j, f in enumerate(event.filters): + is_any = _isAny_opreator(f.operator) + if is_any or len(f.value) == 0: + continue + f.value = helper.values_for_operator(value=f.value, op=f.operator) + op = __get_sql_operator(f.operator) + e_k_f = e_k + f"_fetch{j}" + full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + if f.type == schemas.FetchFilterType._url: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s::text", + f.value, value_key=e_k_f)) + apply = True + elif f.type == schemas.FetchFilterType._status_code: + event_where.append( + _multiple_conditions(f"main.status_code {f.operator} %({e_k_f})s::integer", f.value, + value_key=e_k_f)) + apply = True + elif f.type == schemas.FetchFilterType._method: + event_where.append( + _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + apply = True + elif f.type == schemas.FetchFilterType._duration: + event_where.append( + _multiple_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value, + value_key=e_k_f)) + apply = True + elif f.type == schemas.FetchFilterType._request_body: + event_where.append( + _multiple_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value, value_key=e_k_f)) + apply = True + elif f.type == schemas.FetchFilterType._response_body: + event_where.append( + _multiple_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value, + value_key=e_k_f)) + apply = True + else: + print(f"undefined FETCH filter: {f.type}") + if not apply: + continue + elif event_type == schemas.EventType.graphql_details: + event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " + for j, f in enumerate(event.filters): + is_any = _isAny_opreator(f.operator) + if is_any or len(f.value) == 0: + continue + f.value = helper.values_for_operator(value=f.value, op=f.operator) + op = __get_sql_operator(f.operator) + e_k_f = e_k + f"_graphql{j}" + full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + if f.type == schemas.GraphqlFilterType._name: + event_where.append( + _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value, + value_key=e_k_f)) + elif f.type == schemas.GraphqlFilterType._method: + event_where.append( + _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + elif f.type == schemas.GraphqlFilterType._request_body: + event_where.append( + _multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + elif f.type == schemas.GraphqlFilterType._response_body: + event_where.append( + _multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + else: + print(f"undefined GRAPHQL filter: {f.type}") + else: + continue + if event_index == 0 or or_events: + event_where += ss_constraints + if is_not: + if event_index == 0 or or_events: + events_query_from.append(f"""\ + (SELECT + session_id, + 0 AS timestamp + FROM sessions + WHERE EXISTS(SELECT session_id + FROM {event_from} + WHERE {" AND ".join(event_where)} + AND sessions.session_id=ms.session_id) IS FALSE + AND project_id = %(projectId)s + AND start_ts >= %(startDate)s + AND start_ts <= %(endDate)s + AND duration IS NOT NULL + ) {"" if or_events else (f"AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + else: + events_query_from.append(f"""\ + (SELECT + event_0.session_id, + event_{event_index - 1}.timestamp AS timestamp + WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE + ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ + """) + else: + events_query_from.append(f"""\ + (SELECT main.session_id, {"MIN" if event_index < (valid_events_count - 1) else "MAX"}(main.timestamp) AS timestamp + FROM {event_from} + WHERE {" AND ".join(event_where)} + GROUP BY 1 + ) {"" if or_events else (f"AS event_{event_index} " + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + event_index += 1 + if event_index > 0: + if or_events: + events_query_part = f"""SELECT + session_id, + MIN(timestamp) AS first_event_ts, + MAX(timestamp) AS last_event_ts + FROM ({events_joiner.join(events_query_from)}) AS u + GROUP BY 1""" + else: + events_query_part = f"""SELECT + event_0.session_id, + MIN(event_0.timestamp) AS first_event_ts, + MAX(event_{event_index - 1}.timestamp) AS last_event_ts + FROM {events_joiner.join(events_query_from)} + GROUP BY 1""" + else: + data.events = [] + # --------------------------------------------------------------------------- + if data.startDate is not None: + extra_constraints.append("s.start_ts >= %(startDate)s") + if data.endDate is not None: + extra_constraints.append("s.start_ts <= %(endDate)s") + # if data.platform is not None: + # if data.platform == schemas.PlatformType.mobile: + # extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") + # elif data.platform == schemas.PlatformType.desktop: + # extra_constraints.append( + # b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") + + if errors_only: + extra_from += f" INNER JOIN {events.event_type.ERROR.table} AS er USING (session_id) INNER JOIN public.errors AS ser USING (error_id)" + extra_constraints.append("ser.source = 'js_exception'") + extra_constraints.append("ser.project_id = %(project_id)s") + if error_status != schemas.ErrorStatus.all: + extra_constraints.append("ser.status = %(error_status)s") + full_args["error_status"] = error_status + if favorite_only: + extra_from += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" + extra_constraints.append("ufe.user_id = %(userId)s") + # extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] + if favorite_only and not errors_only and user_id is not None: + extra_from += """INNER JOIN (SELECT user_id, session_id + FROM public.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + USING (session_id)""" + elif not favorite_only and not errors_only and user_id is not None: + extra_from += """LEFT JOIN (SELECT user_id, session_id + FROM public.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + USING (session_id)""" + extra_join = "" + if issue is not None: + extra_join = """ + INNER JOIN LATERAL(SELECT TRUE FROM events_common.issues INNER JOIN public.issues AS p_issues USING (issue_id) + WHERE issues.session_id=f.session_id + AND p_issues.type=%(issue_type)s + AND p_issues.context_string=%(issue_contextString)s + AND timestamp >= f.first_event_ts + AND timestamp <= f.last_event_ts) AS issues ON(TRUE) + """ + full_args["issue_contextString"] = issue["contextString"] + full_args["issue_type"] = issue["type"] + if extra_event: + extra_join += f"""INNER JOIN {extra_event} AS ev USING(session_id)""" + extra_constraints.append("ev.timestamp>=%(startDate)s") + extra_constraints.append("ev.timestamp<=%(endDate)s") + query_part = f"""\ + FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else "public.sessions AS s"} + {extra_join} + {"INNER JOIN public.sessions AS s USING(session_id)" if len(events_query_part) > 0 else ""} + {extra_from} + WHERE + {" AND ".join(extra_constraints)}""" + return full_args, query_part + + +def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, project_id, user_id, extra_event=None): + print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>") + ss_constraints = [] + full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, + "projectId": project_id, "userId": user_id} + extra_constraints = [ + "s.project_id = %(project_id)s", + "isNotNull(s.duration)" + ] + if favorite_only: + extra_constraints.append("""s.session_id IN (SELECT session_id + FROM final.user_favorite_sessions + WHERE user_id = %(userId)s)""") + extra_from = "" + events_query_part = "" + __events_where_basic = ["project_id = %(projectId)s", + "datetime >= toDateTime(%(startDate)s/1000)", + "datetime <= toDateTime(%(endDate)s/1000)"] + events_conditions_where = ["main.project_id = %(projectId)s", + "main.datetime >= toDateTime(%(startDate)s/1000)", + "main.datetime <= toDateTime(%(endDate)s/1000)"] + if len(data.filters) > 0: + meta_keys = None + # to reduce include a sub-query of sessions inside events query, in order to reduce the selected data + include_in_events = False + for i, f in enumerate(data.filters): + if not isinstance(f.value, list): + f.value = [f.value] + filter_type = f.type + f.value = helper.values_for_operator(value=f.value, op=f.operator) + f_k = f"f_value{i}" + full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} + op = __get_sql_operator(f.operator) \ + if filter_type not in [schemas.FilterType.events_count] else f.operator + is_any = _isAny_opreator(f.operator) + is_undefined = _isUndefined_operator(f.operator) + if not is_any and not is_undefined and len(f.value) == 0: + continue + is_not = False + if __is_negation_operator(f.operator): + is_not = True + if filter_type == schemas.FilterType.user_browser: + if is_any: + extra_constraints.append('isNotNull(s.user_browser)') + ss_constraints.append('isNotNull(ms.user_browser)') + else: + extra_constraints.append( + _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: + if is_any: + extra_constraints.append('isNotNull(s.user_os)') + ss_constraints.append('isNotNull(ms.user_os)') + else: + extra_constraints.append( + _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: + if is_any: + extra_constraints.append('isNotNull(s.user_device)') + ss_constraints.append('isNotNull(ms.user_device)') + else: + extra_constraints.append( + _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: + if is_any: + extra_constraints.append('isNotNull(s.user_country)') + ss_constraints.append('isNotNull(ms.user_country)') + else: + extra_constraints.append( + _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.utm_source]: + if is_any: + extra_constraints.append('isNotNull(s.utm_source)') + ss_constraints.append('isNotNull(ms.utm_source)') + elif is_undefined: + extra_constraints.append('isNull(s.utm_source)') + ss_constraints.append('isNull(ms.utm_source)') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_source {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_source {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_medium]: + if is_any: + extra_constraints.append('isNotNull(s.utm_medium)') + ss_constraints.append('isNotNull(ms.utm_medium)') + elif is_undefined: + extra_constraints.append('isNull(s.utm_medium)') + ss_constraints.append('isNull(ms.utm_medium') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_medium {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_medium {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_campaign]: + if is_any: + extra_constraints.append('isNotNull(s.utm_campaign)') + ss_constraints.append('isNotNull(ms.utm_campaign)') + elif is_undefined: + extra_constraints.append('isNull(s.utm_campaign)') + ss_constraints.append('isNull(ms.utm_campaign)') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_campaign {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_campaign {op} toString(%({f_k})s)', f.value, is_not=is_not, + value_key=f_k)) + + elif filter_type == schemas.FilterType.duration: + if len(f.value) > 0 and f.value[0] is not None: + extra_constraints.append("s.duration >= %(minDuration)s") + ss_constraints.append("ms.duration >= %(minDuration)s") + full_args["minDuration"] = f.value[0] + if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: + extra_constraints.append("s.duration <= %(maxDuration)s") + ss_constraints.append("ms.duration <= %(maxDuration)s") + full_args["maxDuration"] = f.value[1] + elif filter_type == schemas.FilterType.referrer: + # extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" + if is_any: + referrer_constraint = 'isNotNull(r.base_referrer)' + else: + referrer_constraint = _multiple_conditions(f"r.base_referrer {op} %({f_k})s", f.value, + is_not=is_not, value_key=f_k) + referrer_constraint = f"""(SELECT DISTINCT session_id + FROM final.events AS r + WHERE {" AND ".join([f"r.{b}" for b in __events_where_basic])} + AND event_type='PAGE' + AND {referrer_constraint})""" + # events_conditions_where.append(f"""main.session_id IN {referrer_constraint}""") + # extra_constraints.append(f"""s.session_id IN {referrer_constraint}""") + extra_from += f"\nINNER JOIN {referrer_constraint} AS referred ON(referred.session_id=s.session_id)" + elif filter_type == events.event_type.METADATA.ui_type: + # get metadata list only if you need it + if meta_keys is None: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + if f.source in meta_keys.keys(): + if is_any: + extra_constraints.append(f"isNotNull(s.{metadata.index_to_colname(meta_keys[f.source])})") + ss_constraints.append(f"isNotNull(ms.{metadata.index_to_colname(meta_keys[f.source])})") + elif is_undefined: + extra_constraints.append(f"isNull(s.{metadata.index_to_colname(meta_keys[f.source])})") + ss_constraints.append(f"isNull(ms.{metadata.index_to_colname(meta_keys[f.source])})") + else: + extra_constraints.append( + _multiple_conditions( + f"s.{metadata.index_to_colname(meta_keys[f.source])} {op} toString(%({f_k})s)", + f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions( + f"ms.{metadata.index_to_colname(meta_keys[f.source])} {op} toString(%({f_k})s)", + f.value, is_not=is_not, value_key=f_k)) + elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: + if is_any: + extra_constraints.append('isNotNull(s.user_id)') + ss_constraints.append('isNotNull(ms.user_id)') + elif is_undefined: + extra_constraints.append('isNull(s.user_id)') + ss_constraints.append('isNull(ms.user_id)') + else: + extra_constraints.append( + _multiple_conditions(f"s.user_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.user_anonymous_id, + schemas.FilterType.user_anonymous_id_ios]: + if is_any: + extra_constraints.append('isNotNull(s.user_anonymous_id)') + ss_constraints.append('isNotNull(ms.user_anonymous_id)') + elif is_undefined: + extra_constraints.append('isNull(s.user_anonymous_id)') + ss_constraints.append('isNull(ms.user_anonymous_id)') + else: + extra_constraints.append( + _multiple_conditions(f"s.user_anonymous_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_anonymous_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: + if is_any: + extra_constraints.append('isNotNull(s.rev_id)') + ss_constraints.append('isNotNull(ms.rev_id)') + elif is_undefined: + extra_constraints.append('isNull(s.rev_id)') + ss_constraints.append('isNull(ms.rev_id)') + else: + extra_constraints.append( + _multiple_conditions(f"s.rev_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.rev_id {op} toString(%({f_k})s)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.platform: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.issue: + if is_any: + extra_constraints.append("array_length(s.issue_types, 1) > 0") + ss_constraints.append("array_length(ms.issue_types, 1) > 0") + else: + extra_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.events_count: + extra_constraints.append( + _multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + else: + continue + include_in_events = True + + if include_in_events: + events_conditions_where.append(f"""main.session_id IN (SELECT s.session_id + FROM final.sessions AS s + WHERE {" AND ".join(extra_constraints)})""") + # --------------------------------------------------------------------------- + if len(data.events) > 0: + valid_events_count = 0 + for event in data.events: + is_any = _isAny_opreator(event.operator) + if not isinstance(event.value, list): + event.value = [event.value] + if __is_valid_event(is_any=is_any, event=event): + valid_events_count += 1 + events_query_from = [] + events_conditions = [] + event_index = 0 + or_events = data.events_order == schemas.SearchEventOrder._or + # events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " + for i, event in enumerate(data.events): + event_type = event.type + print(f">>>>>>>>>>>>>{event_type}") + is_any = _isAny_opreator(event.operator) + if not isinstance(event.value, list): + event.value = [event.value] + if not __is_valid_event(is_any=is_any, event=event): + continue + op = __get_sql_operator(event.operator) + is_not = False + if __is_negation_operator(event.operator): + is_not = True + op = __reverse_sql_operator(op) + # if event_index == 0 or or_events: + # event_from = "%s INNER JOIN final.sessions AS ms USING (session_id)" + event_from = "%s" + event_where = ["main.project_id = %(projectId)s", + "main.datetime >= toDateTime(%(startDate)s/1000)", + "main.datetime <= toDateTime(%(endDate)s/1000)"] + if favorite_only and not errors_only: + event_from += "INNER JOIN final.user_favorite_sessions AS fs USING(session_id)" + event_where.append("fs.user_id = %(userId)s") + # else: + # event_from = "%s" + # event_where = ["main.datetime >= toDateTime(%(startDate)s/1000)", + # "main.datetime <= toDateTime(%(endDate)s/1000)", + # "main.session_id=event_0.session_id"] + # if data.events_order == schemas.SearchEventOrder._then: + # event_where.append(f"event_{event_index - 1}.datetime <= main.datetime") + e_k = f"e_value{i}" + s_k = e_k + "_source" + if event.type != schemas.PerformanceEventType.time_between_events: + event.value = helper.values_for_operator(value=event.value, op=event.operator) + full_args = {**full_args, + **_multiple_values(event.value, value_key=e_k), + **_multiple_values(event.source, value_key=s_k)} + + if event_type == events.event_type.CLICK.ui_type: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='CLICK'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, + value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + + elif event_type == events.event_type.INPUT.ui_type: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='INPUT'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, + value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + if event.source is not None and len(event.source) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, + value_key=f"custom{i}")) + full_args = {**full_args, **_multiple_values(event.source, value_key=f"custom{i}")} + + elif event_type == events.event_type.LOCATION.ui_type: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='PAGE'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + elif event_type == events.event_type.CUSTOM.ui_type: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='CUSTOM'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, + value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + elif event_type == events.event_type.REQUEST.ui_type: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='REQUEST'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, + value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + elif event_type == events.event_type.GRAPHQL.ui_type: + event_from = event_from % f"final.events AS main" + event_where.append(f"main.event_type='GRAPHQL'") + events_conditions.append({"type": event_where[-1]}) + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, + value_key=e_k)) + events_conditions[-1]["condition"] = event_where[-1] + elif event_type == events.event_type.STATEACTION.ui_type: + event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.ERROR.ui_type: + event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" + event.source = tuple(event.source) + if not is_any and event.value not in [None, "*", ""]: + event_where.append( + _multiple_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) + if event.source[0] not in [None, "*", ""]: + event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.value, value_key=s_k)) + + elif event_type == schemas.PerformanceEventType.fetch_failed: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + event.value, value_key=e_k)) + col = performance_event.get_col(event_type) + colname = col["column"] + event_where.append(f"main.{colname} = FALSE") + # elif event_type == schemas.PerformanceEventType.fetch_duration: + # event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + # if not is_any: + # event_where.append( + # _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + # event.value, value_key=e_k)) + # col = performance_event.get_col(event_type) + # colname = col["column"] + # tname = "main" + # e_k += "_custom" + # full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + # _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + # event.source, value_key=e_k)) + elif event_type in [schemas.PerformanceEventType.location_dom_complete, + schemas.PerformanceEventType.location_largest_contentful_paint_time, + schemas.PerformanceEventType.location_ttfb, + schemas.PerformanceEventType.location_avg_cpu_load, + schemas.PerformanceEventType.location_avg_memory_usage + ]: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + col = performance_event.get_col(event_type) + colname = col["column"] + tname = "main" + if col.get("extraJoin") is not None: + tname = "ej" + event_from += f" INNER JOIN {col['extraJoin']} AS {tname} USING(session_id)" + event_where += [f"{tname}.timestamp >= main.timestamp", f"{tname}.timestamp >= %(startDate)s", + f"{tname}.timestamp <= %(endDate)s"] + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + + event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.time_between_events: + event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " + if not isinstance(event.value[0].value, list): + event.value[0].value = [event.value[0].value] + if not isinstance(event.value[1].value, list): + event.value[1].value = [event.value[1].value] + event.value[0].value = helper.values_for_operator(value=event.value[0].value, + op=event.value[0].operator) + event.value[1].value = helper.values_for_operator(value=event.value[1].value, + op=event.value[0].operator) + e_k1 = e_k + "_e1" + e_k2 = e_k + "_e2" + full_args = {**full_args, + **_multiple_values(event.value[0].value, value_key=e_k1), + **_multiple_values(event.value[1].value, value_key=e_k2)} + s_op = __get_sql_operator(event.value[0].operator) + event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] + if event_index > 0 and not or_events: + event_where.append("main2.session_id=event_0.session_id") + is_any = _isAny_opreator(event.value[0].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", + event.value[0].value, value_key=e_k1)) + s_op = __get_sql_operator(event.value[1].operator) + is_any = _isAny_opreator(event.value[1].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", + event.value[1].value, value_key=e_k2)) + + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} + event_where.append( + _multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) + + elif event_type == schemas.EventType.request_details: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='REQUEST'") + events_conditions.append({"type": event_where[-1]}) + apply = False + events_conditions[-1]["condition"] = [] + for j, f in enumerate(event.filters): + is_any = _isAny_opreator(f.operator) + if is_any or len(f.value) == 0: + continue + f.value = helper.values_for_operator(value=f.value, op=f.operator) + op = __get_sql_operator(f.operator) + e_k_f = e_k + f"_fetch{j}" + full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + if f.type == schemas.FetchFilterType._url: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s", f.value, + value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + elif f.type == schemas.FetchFilterType._status_code: + event_where.append( + _multiple_conditions(f"main.status {f.operator} %({e_k_f})s", f.value, + value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + elif f.type == schemas.FetchFilterType._method: + event_where.append( + _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + elif f.type == schemas.FetchFilterType._duration: + event_where.append( + _multiple_conditions(f"main.duration {f.operator} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + elif f.type == schemas.FetchFilterType._request_body: + event_where.append( + _multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + elif f.type == schemas.FetchFilterType._response_body: + event_where.append( + _multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + apply = True + else: + print(f"undefined FETCH filter: {f.type}") + if not apply: + continue + else: + events_conditions[-1]["condition"] = " AND ".join(events_conditions[-1]["condition"]) + + elif event_type == schemas.EventType.graphql_details: + event_from = event_from % f"final.events AS main " + event_where.append(f"main.event_type='REQUEST'") + events_conditions.append({"type": event_where[-1]}) + events_conditions[-1]["condition"] = [] + for j, f in enumerate(event.filters): + is_any = _isAny_opreator(f.operator) + if is_any or len(f.value) == 0: + continue + f.value = helper.values_for_operator(value=f.value, op=f.operator) + op = __get_sql_operator(f.operator) + e_k_f = e_k + f"_graphql{j}" + full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + if f.type == schemas.GraphqlFilterType._name: + event_where.append( + _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value, + value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + elif f.type == schemas.GraphqlFilterType._method: + event_where.append( + _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + elif f.type == schemas.GraphqlFilterType._request_body: + event_where.append( + _multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + elif f.type == schemas.GraphqlFilterType._response_body: + event_where.append( + _multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + events_conditions[-1]["condition"].append(event_where[-1]) + else: + print(f"undefined GRAPHQL filter: {f.type}") + events_conditions[-1]["condition"] = " AND ".join(events_conditions[-1]["condition"]) + else: + continue + if event_index == 0 or or_events: + event_where += ss_constraints + if is_not: + if event_index == 0 or or_events: + events_query_from.append(f"""\ + (SELECT + session_id, + 0 AS timestamp + FROM sessions + WHERE EXISTS(SELECT session_id + FROM {event_from} + WHERE {" AND ".join(event_where)} + AND sessions.session_id=ms.session_id) IS FALSE + AND project_id = %(projectId)s + AND start_ts >= %(startDate)s + AND start_ts <= %(endDate)s + AND duration IS NOT NULL + ) {"" if or_events else (f"AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + else: + events_query_from.append(f"""\ + (SELECT + event_0.session_id, + event_{event_index - 1}.timestamp AS timestamp + WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE + ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ + """) + else: + if data.events_order == schemas.SearchEventOrder._then: + pass + else: + events_query_from.append(f"""\ + (SELECT main.session_id, {"MIN" if event_index < (valid_events_count - 1) else "MAX"}(main.datetime) AS datetime + FROM {event_from} + WHERE {" AND ".join(event_where)} + GROUP BY session_id + ) {"" if or_events else (f"AS event_{event_index} " + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + event_index += 1 + + if event_index < 2: + data.events_order = schemas.SearchEventOrder._or + if favorite_only and user_id is not None: + events_conditions_where.append("""main.session_id IN (SELECT session_id + FROM final.user_favorite_sessions + WHERE user_id = %(userId)s)""") + for e in events_conditions: + print(e) + print("---") + if data.events_order in [schemas.SearchEventOrder._then, schemas.SearchEventOrder._and]: + events_conditions_where.append(f"({' OR '.join([c['type'] for c in events_conditions])})") + events_conditions_where.append(f"({' OR '.join([c['condition'] for c in events_conditions])})") + events_conditions = [c['type'] + ' AND ' + c['condition'] for c in events_conditions] + if data.events_order == schemas.SearchEventOrder._then: + having = f"""HAVING sequenceMatch('{''.join([f'(?{i + 1})' for i in range(len(events_conditions))])}')\ + (main.datetime,{','.join(events_conditions)})""" + else: + having = f"""HAVING {" AND ".join([f"countIf({c})>0" for c in events_conditions])}""" + + events_query_part = f"""SELECT main.session_id, + MIN(main.datetime) AS first_event_ts, + MAX(main.datetime) AS last_event_ts + FROM final.events AS main + WHERE {" AND ".join(events_conditions_where)} + GROUP BY session_id + {having}""" + else: + events_conditions_where.append(f"({' OR '.join([c['type'] for c in events_conditions])})") + events_conditions = [c['type'] + ' AND ' + c['condition'] for c in events_conditions] + events_conditions_where.append(f"({' OR '.join(events_conditions)})") + events_query_part = f"""SELECT main.session_id, + MIN(main.datetime) AS first_event_ts, + MAX(main.datetime) AS last_event_ts + FROM final.events AS main + WHERE {" AND ".join(events_conditions_where)} + GROUP BY session_id""" + else: + data.events = [] + # --------------------------------------------------------------------------- + if data.startDate is not None: + extra_constraints.append("s.datetime >= toDateTime(%(startDate)s/1000)") + if data.endDate is not None: + extra_constraints.append("s.datetime <= toDateTime(%(endDate)s/1000)") + # if data.platform is not None: + # if data.platform == schemas.PlatformType.mobile: + # extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") + # elif data.platform == schemas.PlatformType.desktop: + # extra_constraints.append( + # b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") + + if errors_only: + extra_from += f" INNER JOIN {events.event_type.ERROR.table} AS er USING (session_id) INNER JOIN public.errors AS ser USING (error_id)" + extra_constraints.append("ser.source = 'js_exception'") + extra_constraints.append("ser.project_id = %(project_id)s") + if error_status != schemas.ErrorStatus.all: + extra_constraints.append("ser.status = %(error_status)s") + full_args["error_status"] = error_status + if favorite_only: + extra_from += " INNER JOIN final.user_favorite_errors AS ufe USING (error_id)" + extra_constraints.append("ufe.user_id = %(userId)s") + # extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] + if favorite_only and not errors_only and user_id is not None: + extra_from += """INNER JOIN (SELECT 1 AS session_id) AS favorite_sessions + ON (TRUE)""" + elif not favorite_only and not errors_only and user_id is not None: + extra_from += """LEFT JOIN (SELECT session_id + FROM final.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + ON (s.session_id=favorite_sessions.session_id)""" + extra_join = "" + if issue is not None: + extra_join = """ + INNER JOIN LATERAL(SELECT TRUE FROM events_common.issues INNER JOIN public.issues AS p_issues USING (issue_id) + WHERE issues.session_id=f.session_id + AND p_issues.type=%(issue_type)s + AND p_issues.context_string=%(issue_contextString)s + AND timestamp >= f.first_event_ts + AND timestamp <= f.last_event_ts) AS issues ON(TRUE) + """ + full_args["issue_contextString"] = issue["contextString"] + full_args["issue_type"] = issue["type"] + if extra_event: + extra_join += f"""INNER JOIN {extra_event} AS ev USING(session_id)""" + extra_constraints.append("ev.timestamp>=%(startDate)s") + extra_constraints.append("ev.timestamp<=%(endDate)s") + if len(events_query_part) > 0: + extra_join += f"""INNER JOIN (SELECT * + FROM final.sessions AS s + WHERE {" AND ".join(extra_constraints)}) AS s ON(s.session_id=f.session_id)""" + else: + extra_join += f"""(SELECT * + FROM final.sessions AS s + WHERE {" AND ".join(extra_constraints)}) AS s""" + query_part = f"""\ + FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else ""} + {extra_join} + {extra_from} + """ + return full_args, query_part + + +def search_by_metadata(tenant_id, user_id, m_key, m_value, project_id=None): + if project_id is None: + all_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) + else: + all_projects = [ + projects.get_project(tenant_id=tenant_id, project_id=int(project_id), include_last_session=False, + include_gdpr=False)] + + all_projects = {int(p["projectId"]): p["name"] for p in all_projects} + project_ids = list(all_projects.keys()) + + available_keys = metadata.get_keys_by_projects(project_ids) + for i in available_keys: + available_keys[i]["user_id"] = schemas.FilterType.user_id + available_keys[i]["user_anonymous_id"] = schemas.FilterType.user_anonymous_id + results = {} + for i in project_ids: + if m_key not in available_keys[i].values(): + available_keys.pop(i) + results[i] = {"total": 0, "sessions": [], "missingMetadata": True} + project_ids = list(available_keys.keys()) + if len(project_ids) > 0: + with pg_client.PostgresClient() as cur: + sub_queries = [] + for i in project_ids: + col_name = list(available_keys[i].keys())[list(available_keys[i].values()).index(m_key)] + sub_queries.append(cur.mogrify( + f"(SELECT COALESCE(COUNT(s.*)) AS count FROM public.sessions AS s WHERE s.project_id = %(id)s AND s.{col_name} = %(value)s) AS \"{i}\"", + {"id": i, "value": m_value}).decode('UTF-8')) + query = f"""SELECT {", ".join(sub_queries)};""" + cur.execute(query=query) + + rows = cur.fetchone() + + sub_queries = [] + for i in rows.keys(): + results[i] = {"total": rows[i], "sessions": [], "missingMetadata": False, "name": all_projects[int(i)]} + if rows[i] > 0: + col_name = list(available_keys[int(i)].keys())[list(available_keys[int(i)].values()).index(m_key)] + sub_queries.append( + cur.mogrify( + f"""( + SELECT * + FROM ( + SELECT DISTINCT ON(favorite_sessions.session_id, s.session_id) {SESSION_PROJECTION_COLS} + FROM public.sessions AS s LEFT JOIN (SELECT session_id + FROM public.user_favorite_sessions + WHERE user_favorite_sessions.user_id = %(userId)s + ) AS favorite_sessions USING (session_id) + WHERE s.project_id = %(id)s AND s.duration IS NOT NULL AND s.{col_name} = %(value)s + ) AS full_sessions + ORDER BY favorite DESC, issue_score DESC + LIMIT 10 + )""", + {"id": i, "value": m_value, "userId": user_id}).decode('UTF-8')) + if len(sub_queries) > 0: + cur.execute("\nUNION\n".join(sub_queries)) + rows = cur.fetchall() + for i in rows: + results[str(i["project_id"])]["sessions"].append(helper.dict_to_camel_case(i)) + return results + + +def search_by_issue(user_id, issue, project_id, start_date, end_date): + constraints = ["s.project_id = %(projectId)s", + "p_issues.context_string = %(issueContextString)s", + "p_issues.type = %(issueType)s"] + if start_date is not None: + constraints.append("start_ts >= %(startDate)s") + if end_date is not None: + constraints.append("start_ts <= %(endDate)s") + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""SELECT DISTINCT ON(favorite_sessions.session_id, s.session_id) {SESSION_PROJECTION_COLS} + FROM public.sessions AS s + INNER JOIN events_common.issues USING (session_id) + INNER JOIN public.issues AS p_issues USING (issue_id) + LEFT JOIN (SELECT user_id, session_id + FROM public.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + USING (session_id) + WHERE {" AND ".join(constraints)} + ORDER BY s.session_id DESC;""", + { + "issueContextString": issue["contextString"], + "issueType": issue["type"], "userId": user_id, + "projectId": project_id, + "startDate": start_date, + "endDate": end_date + })) + + rows = cur.fetchall() + return helper.list_to_camel_case(rows) + + +def get_user_sessions(project_id, user_id, start_date, end_date): + with pg_client.PostgresClient() as cur: + constraints = ["s.project_id = %(projectId)s", "s.user_id = %(userId)s"] + if start_date is not None: + constraints.append("s.start_ts >= %(startDate)s") + if end_date is not None: + constraints.append("s.start_ts <= %(endDate)s") + + query_part = f"""\ + FROM public.sessions AS s + WHERE {" AND ".join(constraints)}""" + + cur.execute(cur.mogrify(f"""\ + SELECT s.project_id, + s.session_id::text AS session_id, + s.user_uuid, + s.user_id, + s.user_os, + s.user_browser, + s.user_device, + s.user_country, + s.start_ts, + s.duration, + s.events_count, + s.pages_count, + s.errors_count + {query_part} + ORDER BY s.session_id + LIMIT 50;""", { + "projectId": project_id, + "userId": user_id, + "startDate": start_date, + "endDate": end_date + })) + + sessions = cur.fetchall() + return helper.list_to_camel_case(sessions) + + +def get_session_user(project_id, user_id): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """\ + SELECT + user_id, + count(*) as session_count, + max(start_ts) as last_seen, + min(start_ts) as first_seen + FROM + "public".sessions + WHERE + project_id = %(project_id)s + AND user_id = %(userId)s + AND duration is not null + GROUP BY user_id; + """, + {"project_id": project_id, "userId": user_id} + ) + cur.execute(query=query) + data = cur.fetchone() + return helper.dict_to_camel_case(data) + + +def get_session_ids_by_user_ids(project_id, user_ids): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """\ + SELECT session_id FROM public.sessions + WHERE + project_id = %(project_id)s AND user_id IN %(userId)s;""", + {"project_id": project_id, "userId": tuple(user_ids)} + ) + ids = cur.execute(query=query) + return ids + + +def delete_sessions_by_session_ids(session_ids): + with pg_client.PostgresClient(unlimited_query=True) as cur: + query = cur.mogrify( + """\ + DELETE FROM public.sessions + WHERE + session_id IN %(session_ids)s;""", + {"session_ids": tuple(session_ids)} + ) + cur.execute(query=query) + + return True + + +def delete_sessions_by_user_ids(project_id, user_ids): + with pg_client.PostgresClient(unlimited_query=True) as cur: + query = cur.mogrify( + """\ + DELETE FROM public.sessions + WHERE + project_id = %(project_id)s AND user_id IN %(userId)s;""", + {"project_id": project_id, "userId": tuple(user_ids)} + ) + cur.execute(query=query) + + return True + + +def count_all(): + with pg_client.PostgresClient(unlimited_query=True) as cur: + row = cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions") + return row.get("count", 0) diff --git a/ee/api/clean.sh b/ee/api/clean.sh index fa1ab8cb5..e7785b22e 100755 --- a/ee/api/clean.sh +++ b/ee/api/clean.sh @@ -29,7 +29,6 @@ rm -rf ./chalicelib/core/log_tool_stackdriver.py rm -rf ./chalicelib/core/log_tool_sumologic.py rm -rf ./chalicelib/core/metadata.py rm -rf ./chalicelib/core/mobile.py -rm -rf ./chalicelib/core/sessions.py rm -rf ./chalicelib/core/sessions_assignments.py rm -rf ./chalicelib/core/sessions_metas.py rm -rf ./chalicelib/core/sessions_mobs.py diff --git a/ee/api/routers/core_dynamic.py b/ee/api/routers/core_dynamic.py index 3c5c21905..e5ba6c6c4 100644 --- a/ee/api/routers/core_dynamic.py +++ b/ee/api/routers/core_dynamic.py @@ -176,3 +176,13 @@ def get_general_stats(): def get_projects(context: schemas.CurrentContext = Depends(OR_context)): return {"data": projects.get_projects(tenant_id=context.tenant_id, recording_state=True, gdpr=True, recorded=True, stack_integrations=True, user_id=context.user_id)} + + +@app.post('/{projectId}/sessions/search2', tags=["sessions"]) +def sessions_search2(projectId: int, data: schemas.FlatSessionsSearchPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + if config("LEGACY_SEARCH", cast=bool, default=False): + data = sessions.search2_pg(data=data, project_id=projectId, user_id=context.user_id) + else: + data = sessions.search2_ch(data=data, project_id=projectId, user_id=context.user_id) + return {'data': data} diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql new file mode 100644 index 000000000..f6ba9d751 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql @@ -0,0 +1,258 @@ +ALTER TABLE sessions + DROP COLUMN pages_count; + + +CREATE TABLE projects_metadata +( + project_id UInt32, + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(_timestamp) + ORDER BY (project_id) + SETTINGS index_granularity = 512; + +CREATE TABLE IF NOT EXISTS events_s +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(UInt8), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16), + _timestamp DateTime DEFAULT now() +) ENGINE = MergeTree + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, event_type, session_id) + TTL datetime + INTERVAL 1 MONTH; + +CREATE TABLE IF NOT EXISTS sessions +( + session_id UInt64, + project_id UInt32, + tracker_version LowCardinality(String), + rev_id LowCardinality(Nullable(String)), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + duration UInt32, + pages_count UInt16, + events_count UInt16, + errors_count UInt16, + utm_source Nullable(String), + utm_medium Nullable(String), + utm_campaign Nullable(String), + user_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 1 MONTH + SETTINGS index_granularity = 512; + +CREATE TABLE IF NOT EXISTS autocomplete +( + project_id UInt32 NOT NULL, + type LowCardinality(String) NOT NULL, + value String NOT NULL, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(_timestamp) + ORDER BY (project_id, type) + TTL _timestamp + INTERVAL 1 MONTH; + + +CREATE MATERIALIZED VIEW sessions_l7d_mv + ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 7 DAY + SETTINGS index_granularity = 512 + POPULATE +AS +SELECT * +FROM massive_split.sessions_s +WHERE datetime >= now() - INTERVAL 7 DAY + AND isNotNull(duration) + AND duration > 0; + +CREATE MATERIALIZED VIEW events_l7d_mv + ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 7 DAY + POPULATE +AS +SELECT * +FROM massive_split.events_s +WHERE datetime >= now() - INTERVAL 7 DAY; + + +CREATE MATERIALIZED VIEW sessions_info_l1m_mv + ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 1 MONTH + SETTINGS index_granularity = 512 + POPULATE +AS +SELECT project_id, + session_id, + datetime, + now() AS _timestamp, + toJSONString(map('project_id', toString(project_id), + 'session_id', toString(session_id), + 'user_uuid', toString(user_uuid), + 'user_id', user_id, + 'user_os', user_os, + 'user_browser', user_browser, + 'user_device', user_device, + --'user_device_type', user_device_type, +--'user_country', user_country, + 'start_ts', toString(datetime), + 'duration', toString(duration), + 'events_count', toString(events_count), + 'pages_count', toString(pages_count), + 'errors_count', toString(errors_count), + -- 'user_anonymous_id', user_anonymous_id, +-- 'platform', platform, +-- 'issue_score', issue_score, +-- issue_types, +-- favorite, +-- viewed, + 'metadata', CAST((arrayFilter(x->isNotNull(x), + arrayMap( + x->if(isNotNull(x[1]) AND isNotNull(x[2]), toString(x[1]), + NULL), + [ + [projects_meta.metadata_1,sessions.metadata_1], + [projects_meta.metadata_2,sessions.metadata_2], + [projects_meta.metadata_3,sessions.metadata_3], + [projects_meta.metadata_4,sessions.metadata_4], + [projects_meta.metadata_5,sessions.metadata_5], + [projects_meta.metadata_6,sessions.metadata_6], + [projects_meta.metadata_7,sessions.metadata_7], + [projects_meta.metadata_8,sessions.metadata_8], + [projects_meta.metadata_9,sessions.metadata_9], + [projects_meta.metadata_10,sessions.metadata_10] + ])), + arrayFilter(x->isNotNull(x), + arrayMap( + x->if(isNotNull(x[1]) AND isNotNull(x[2]), toString(x[2]), + NULL), + [ + [projects_meta.metadata_1,sessions.metadata_1], + [projects_meta.metadata_2,sessions.metadata_2], + [projects_meta.metadata_3,sessions.metadata_3], + [projects_meta.metadata_4,sessions.metadata_4], + [projects_meta.metadata_5,sessions.metadata_5], + [projects_meta.metadata_6,sessions.metadata_6], + [projects_meta.metadata_7,sessions.metadata_7], + [projects_meta.metadata_8,sessions.metadata_8], + [projects_meta.metadata_9,sessions.metadata_9], + [projects_meta.metadata_10,sessions.metadata_10] + ]))), 'Map(String,String)') + )) AS info +FROM massive_split.sessions + INNER JOIN projects_metadata USING (project_id) +WHERE datetime >= now() - INTERVAL 1 MONTH + AND isNotNull(duration) + AND duration > 0; + +CREATE MATERIALIZED VIEW sessions_info_l7d_mv + ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 7 DAY + SETTINGS index_granularity = 512 + POPULATE +AS +SELECT * +FROM sessions_info_l1m_mv +WHERE datetime >= now() - INTERVAL 7 DAY; diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/__clickhouse.csv b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/__clickhouse.csv new file mode 100644 index 000000000..7c2f7cab1 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/__clickhouse.csv @@ -0,0 +1,68 @@ +MACHINE ,QUERY ,TABLE ,TIME(max_threads=auto(2)),TIME (max_threads=4),TIME (max_threads=4) 1W,TIME (max_threads=4) 1W_mv,TIME 1d (max_threads=4) 1W_mv,TIME 1d (max_threads=4) 1d_mv,REMARK +r5.large (16Gb) ,Q1 ,massive2.events7 ,timeout ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,Q1 ,massive2.events7 ,12s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),Q1 ,massive2.events7 ,7s ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,Q1 ,massive2.events7 ,24s ,19s ,17s ,- ,- ,- , +r5.xlarge (32Gb) ,Q1.1 ,massive2.events7 ,16s ,14s ,12s ,- ,- ,- ,WHERE user_id +r5.xlarge (32Gb) ,Q1 ,massive_split.events_s,23s ,18s ,16s ,3s ,1s ,0.8s , +r5.xlarge (32Gb) ,Q1.2 ,massive_split.events_s,timeout ,27s ,23s ,2s ,0.8s ,0.7s ,events INNER JOIN meta WHERE user_id +r5.xlarge (32Gb) ,Q1.2.1 ,massive_split.events_s,timeout ,timeout ,27s ,4s ,1.6s ,0.8s ,events INNER JOIN meta WHERE ALL AND user_id +r5.xlarge (32Gb) ,Q1.3 ,massive_split.events_s,15s ,13s ,11s ,2s ,0.7s ,0.6s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,Q1.4 ,massive_split.events_s,memory ,memory ,memory ,4s ,0.7s ,0.6s ,SUBQUERY WHERE user_id INNER JOIN events +r5.xlarge (32Gb) ,Q1.5 ,massive_split.events_s,11s ,10s ,8s ,1s ,0.6s ,0.6s ,WHERE session_id IN SUBQUERY WHERE user_id +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== +r5.large (16Gb) ,Q2 ,massive2.events7 ,timeout ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,Q2 ,massive2.events7 ,12s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),Q2 ,massive2.events7 ,7s ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,Q2 ,massive2.events7 ,24s ,20s ,17s ,- ,- ,- , +r5.xlarge (32Gb) ,Q2.1 ,massive2.events7 ,21s ,14s ,12s ,- ,- ,- ,WHERE user_id +r5.xlarge (32Gb) ,Q2 ,massive_split.events_s,22s ,19s ,16s ,3s ,0.7s ,0.6s , +r5.xlarge (32Gb) ,Q2.2 ,massive_split.events_s,timeout ,27s ,23s ,2s ,0.6s ,0.6s ,events INNER JOIN meta WHERE user_id +r5.xlarge (32Gb) ,Q2.2.1 ,massive_split.events_s,timeout ,timeout ,27s ,4s ,1.3s ,0.7s ,events INNER JOIN meta WHERE ALL AND user_id +r5.xlarge (32Gb) ,Q2.3 ,massive_split.events_s,15s ,13s ,11s ,2s ,0.6s ,0.6s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,Q2.4 ,massive_split.events_s,memory ,memory ,memory ,4s ,0.6s ,0.6s ,SUBQUERY WHERE user_id INNER JOIN events +r5.xlarge (32Gb) ,Q2.5 ,massive_split.events_s,11s ,10s ,8s ,1s ,0.6s ,0.6s ,WHERE session_id IN SUBQUERY WHERE user_id +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== +r5.large (16Gb) ,Q3 ,massive2.events7 ,timeout ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,Q3 ,massive2.events7 ,11s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),Q3 ,massive2.events7 ,6s ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,Q3 ,massive2.events7 ,22s ,19s ,15s ,- ,- ,- , +r5.xlarge (32Gb) ,Q3.1 ,massive2.events7 ,18s ,15s ,12s ,- ,- ,- ,WHERE user_id +r5.xlarge (32Gb) ,Q3 ,massive_split.events_s,21s ,18s ,15s ,3s ,0.7s ,0.7s , +r5.xlarge (32Gb) ,Q3.2 ,massive_split.events_s,24s ,20s ,17s ,2s ,0.7s ,0.6s ,events INNER JOIN meta WHERE user_id +r5.xlarge (32Gb) ,Q3.2.1 ,massive_split.events_s,26s ,22s ,19s ,4s ,1.4s ,0.7s ,events INNER JOIN meta WHERE ALL AND user_id +r5.xlarge (32Gb) ,Q3.3 ,massive_split.events_s,15s ,13s ,11s ,2s ,0.6s ,0.6s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,Q3.4 ,massive_split.events_s,memory ,memory ,memroy ,3s ,0.7s ,0.7s ,SUBQUERY WHERE user_id INNER JOIN events +r5.xlarge (32Gb) ,Q3.5 ,massive_split.events_s,13s ,11s ,9s ,1s ,0.7s ,0.6s ,WHERE session_id IN SUBQUERY WHERE user_id +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== +r5.large (16Gb) ,Q4 ,massive2.events7 ,timeout ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,Q4 ,massive2.events7 ,11s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),Q4 ,massive2.events7 ,6s ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,Q4 ,massive2.events7 ,22s ,18s ,15s ,- ,- ,- , +r5.xlarge (32Gb) ,Q4.1 ,massive2.events7 ,18s ,15s ,13s ,- ,- ,- ,WHERE user_id +r5.xlarge (32Gb) ,Q4 ,massive_split.events_s,21s ,18s ,15s ,3s ,0.7s ,0.6s , +r5.xlarge (32Gb) ,Q4.2 ,massive_split.events_s,24s ,20s ,17s ,2s ,0.7s ,0.6s ,events INNER JOIN meta WHERE user_id +r5.xlarge (32Gb) ,Q4.2.1 ,massive_split.events_s,27s ,22s ,19s ,4s ,1.4s ,0.7s ,events INNER JOIN meta WHERE ALL AND user_id +r5.xlarge (32Gb) ,Q4.3 ,massive_split.events_s,15s ,13s ,11s ,2s ,0.6s ,0.7s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,Q4.4 ,massive_split.events_s,memory ,memory ,memroy ,3s ,1.2s ,0.7s ,SUBQUERY WHERE user_id INNER JOIN events +r5.xlarge (32Gb) ,Q4.5 ,massive_split.events_s,14s ,12s ,9s ,2s ,0.6s ,0.6s ,WHERE session_id IN SUBQUERY WHERE user_id +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== +r5.large (16Gb) ,QU1 ,massive2.events7 ,timeout ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,QU1 ,massive2.events7 ,18s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),QU1 ,massive2.events7 ,10s ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,QU1 ,massive2.events7 ,timeout ,28s ,24s ,- ,- ,- , +r5.xlarge (32Gb) ,QU1.1 ,massive2.events7 ,17s ,14s ,12s ,- ,- ,- ,WHERE user_id +r5.xlarge (32Gb) ,QU1.2 ,massive_split.events_s,timeout ,timeout ,timeout ,6s ,1.5s ,0.7s ,events INNER JOIN meta +r5.xlarge (32Gb) ,QU1.3 ,massive_split.events_s,timeout ,timeout ,23s ,3s ,1.4s ,0.7s ,events INNER JOIN meta WHERE user_id +r5.xlarge (32Gb) ,QU1.4 ,massive_split.events_s,15s ,13s ,11s ,2s ,0.7s ,0.6s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,QU1.4-A,massive_split.events_s,timeout ,timeout ,timeout ,6s ,0.7s ,0.7s ,events INNER JOIN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,QU1.5 ,massive_split.events_s,memory ,memory ,memory ,memory ,memory ,2s ,SUBQUERY WHERE user_id INNER JOIN events +r5.xlarge (32Gb) ,QU1.6 ,massive_split.events_s,11s ,10s ,8s ,1s ,0.7s ,0.7s ,events INNER JOIN SUBQUERY WHERE user_id WHERE session_id IN SUBQUERY WHERE user_id +r5.xlarge (32Gb) ,QU1.6-A,massive_split.events_s,timeout ,timeout ,timeout ,6s ,0.7s ,0.7s ,events INNER JOIN SUBQUERY WHERE session_id IN SUBQUERY +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== +r5.large (16Gb) ,QM4 ,massive2.events7 ,- ,- ,- ,- ,- ,- , +r5.2xlarge (64Gb) ,QM4 ,massive2.events7 ,3s ,- ,- ,- ,- ,- , +r5.3xlarge (128Gb),QM4 ,massive2.events7 ,- ,- ,- ,- ,- ,- , +r5.xlarge (32Gb) ,QM4 ,massive2.events7 ,7s ,5s ,4s ,- ,- ,- , +r5.xlarge (32Gb) ,QM4 ,massive_split.events_s,6s ,5s ,4s ,1s ,0.7s ,0.6s , +======== ,===== ,================= ,=========== ,============ ,== ,== ,== ,== ,== \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql new file mode 100644 index 000000000..e22b73848 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/fill.sql @@ -0,0 +1,2878 @@ +-- CREATE TABLE IF NOT EXISTS single_t.events +-- ( +-- session_id UInt64, +-- project_id UInt32, +-- event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), +-- tracker_version LowCardinality(String), +-- rev_id Nullable(String), +-- user_uuid UUID, +-- user_os LowCardinality(String), +-- user_os_version LowCardinality(Nullable(String)), +-- user_browser LowCardinality(String), +-- user_browser_version LowCardinality(Nullable(String)), +-- user_device Nullable(String), +-- user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), +-- user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), +-- datetime DateTime, +-- label Nullable(String), +-- hesitation_time Nullable(UInt32), +-- name Nullable(String), +-- payload Nullable(String), +-- level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), +-- source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), +-- message Nullable(String), +-- error_id Nullable(String), +-- duration Nullable(UInt16), +-- context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), +-- container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), +-- container_id Nullable(String), +-- container_name Nullable(String), +-- container_src Nullable(String), +-- url Nullable(String), +-- url_host Nullable(String) MATERIALIZED lower(domain(url)), +-- url_path Nullable(String) MATERIALIZED lower(pathFull(url)), +-- request_start Nullable(UInt16), +-- response_start Nullable(UInt16), +-- response_end Nullable(UInt16), +-- dom_content_loaded_event_start Nullable(UInt16), +-- dom_content_loaded_event_end Nullable(UInt16), +-- load_event_start Nullable(UInt16), +-- load_event_end Nullable(UInt16), +-- first_paint Nullable(UInt16), +-- first_contentful_paint Nullable(UInt16), +-- speed_index Nullable(UInt16), +-- visually_complete Nullable(UInt16), +-- time_to_interactive Nullable(UInt16), +-- ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), +-- minus(response_start, request_start), Null), +-- ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), +-- minus(response_end, request_start), Null), +-- response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), +-- minus(response_end, response_start), Null), +-- dom_building_time Nullable(UInt16) MATERIALIZED if( +-- greaterOrEquals(dom_content_loaded_event_start, response_end), +-- minus(dom_content_loaded_event_start, response_end), Null), +-- dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( +-- greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), +-- minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), +-- load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), +-- minus(load_event_end, load_event_start), Null), +-- min_fps Nullable(UInt8), +-- avg_fps Nullable(UInt8), +-- max_fps Nullable(UInt8), +-- min_cpu Nullable(UInt8), +-- avg_cpu Nullable(UInt8), +-- max_cpu Nullable(UInt8), +-- min_total_js_heap_size Nullable(UInt64), +-- avg_total_js_heap_size Nullable(UInt64), +-- max_total_js_heap_size Nullable(UInt64), +-- min_used_js_heap_size Nullable(UInt64), +-- avg_used_js_heap_size Nullable(UInt64), +-- max_used_js_heap_size Nullable(UInt64), +-- type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), +-- header_size Nullable(UInt16), +-- encoded_body_size Nullable(UInt32), +-- decoded_body_size Nullable(UInt32), +-- compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), +-- success Nullable(UInt8), +-- method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), +-- status Nullable(UInt16) +-- ) ENGINE = MergeTree +-- PARTITION BY toDate(datetime) +-- ORDER BY (project_id, datetime); +-- -- TTL datetime + INTERVAL 1 MONTH; +-- DROP TABLE single_t.events; +-- +-- INSERT INTO eng_t.events4(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, label, hesitation_time, name, payload, level, source, message, +-- error_id, duration, context, container_type, container_id, container_name, container_src, +-- url, request_start, response_start, response_end, dom_content_loaded_event_start, +-- dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, +-- first_contentful_paint, speed_index, visually_complete, time_to_interactive, min_fps, +-- avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, +-- max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, +-- type, header_size, encoded_body_size, decoded_body_size, success, method, status) +-- SELECT session_id, +-- project_id, +-- event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- label, +-- hesitation_time, +-- name, +-- payload, +-- level, +-- source, +-- message, +-- error_id, +-- duration, +-- context, +-- container_type, +-- container_id, +-- container_name, +-- container_src, +-- url, +-- request_start, +-- response_start, +-- response_end, +-- dom_content_loaded_event_start, +-- dom_content_loaded_event_end, +-- load_event_start, +-- load_event_end, +-- first_paint, +-- first_contentful_paint, +-- speed_index, +-- visually_complete, +-- time_to_interactive, +-- min_fps, +-- avg_fps, +-- max_fps, +-- min_cpu, +-- avg_cpu, +-- max_cpu, +-- min_total_js_heap_size, +-- avg_total_js_heap_size, +-- max_total_js_heap_size, +-- min_used_js_heap_size, +-- avg_used_js_heap_size, +-- max_used_js_heap_size, +-- type, +-- header_size, +-- encoded_body_size, +-- decoded_body_size, +-- success, +-- method, +-- status +-- FROM ( +-- SELECT session_id, +-- project_id, +-- 'CLICK' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- label, +-- hesitation_time, +-- null AS name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- null AS message, +-- null AS error_id, +-- null AS duration, +-- null AS context, +-- null AS container_type, +-- null AS container_id, +-- null AS container_name, +-- null AS container_src, +-- null AS url, +-- null AS request_start, +-- null AS response_start, +-- null AS response_end, +-- null AS dom_content_loaded_event_start, +-- null AS dom_content_loaded_event_end, +-- null AS load_event_start, +-- null AS load_event_end, +-- null AS first_paint, +-- null AS first_contentful_paint, +-- null AS speed_index, +-- null AS visually_complete, +-- null AS time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- null AS type, +-- null AS header_size, +-- null AS encoded_body_size, +-- null AS decoded_body_size, +-- null AS success, +-- null AS method, +-- null AS status +-- FROM clicks +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'ERROR' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- null AS label, +-- null AS hesitation_time, +-- name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- message, +-- error_id, +-- null AS duration, +-- null AS context, +-- null AS container_type, +-- null AS container_id, +-- null AS container_name, +-- null AS container_src, +-- null AS url, +-- null AS request_start, +-- null AS response_start, +-- null AS response_end, +-- null AS dom_content_loaded_event_start, +-- null AS dom_content_loaded_event_end, +-- null AS load_event_start, +-- null AS load_event_end, +-- null AS first_paint, +-- null AS first_contentful_paint, +-- null AS speed_index, +-- null AS visually_complete, +-- null AS time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- null AS type, +-- null AS header_size, +-- null AS encoded_body_size, +-- null AS decoded_body_size, +-- null AS success, +-- null AS method, +-- null AS status +-- FROM errors +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'INPUT' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- label, +-- null AS hesitation_time, +-- null AS name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- null AS message, +-- null AS error_id, +-- null AS duration, +-- null AS context, +-- null AS container_type, +-- null AS container_id, +-- null AS container_name, +-- null AS container_src, +-- null AS url, +-- null AS request_start, +-- null AS response_start, +-- null AS response_end, +-- null AS dom_content_loaded_event_start, +-- null AS dom_content_loaded_event_end, +-- null AS load_event_start, +-- null AS load_event_end, +-- null AS first_paint, +-- null AS first_contentful_paint, +-- null AS speed_index, +-- null AS visually_complete, +-- null AS time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- null AS type, +-- null AS header_size, +-- null AS encoded_body_size, +-- null AS decoded_body_size, +-- null AS success, +-- null AS method, +-- null AS status +-- FROM inputs +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'LONGTASK' event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- null AS label, +-- null AS hesitation_time, +-- null AS name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- null AS message, +-- null AS error_id, +-- duration, +-- context, +-- container_type, +-- container_id, +-- container_name, +-- container_src, +-- null AS url, +-- null AS request_start, +-- null AS response_start, +-- null AS response_end, +-- null AS dom_content_loaded_event_start, +-- null AS dom_content_loaded_event_end, +-- null AS load_event_start, +-- null AS load_event_end, +-- null AS first_paint, +-- null AS first_contentful_paint, +-- null AS speed_index, +-- null AS visually_complete, +-- null AS time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- null AS type, +-- null AS header_size, +-- null AS encoded_body_size, +-- null AS decoded_body_size, +-- null AS success, +-- null AS method, +-- null AS status +-- FROM longtasks +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'PAGE' event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- null AS label, +-- null AS hesitation_time, +-- null AS name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- null AS message, +-- null AS error_id, +-- null AS duration, +-- null AS context, +-- null AS container_type, +-- null AS container_id, +-- null AS container_name, +-- null AS container_src, +-- url, +-- request_start, +-- response_start, +-- response_end, +-- dom_content_loaded_event_start, +-- dom_content_loaded_event_end, +-- load_event_start, +-- load_event_end, +-- first_paint, +-- first_contentful_paint, +-- speed_index, +-- visually_complete, +-- time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- null AS type, +-- null AS header_size, +-- null AS encoded_body_size, +-- null AS decoded_body_size, +-- null AS success, +-- null AS method, +-- null AS status +-- FROM pages +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'PERFORMANCE' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- null AS label label, +-- null AS label hesitation_time, +-- null AS label name, +-- null AS label payload, +-- null AS label level, +-- null AS label source, +-- null AS label message, +-- null AS label error_id, +-- null AS label duration, +-- null AS label context, +-- null AS label container_type, +-- null AS label container_id, +-- null AS label container_name, +-- null AS label container_src, +-- null AS labelurl, +-- null AS label request_start, +-- null AS label response_start, +-- null AS label response_end, +-- null AS label dom_content_loaded_event_start, +-- null AS label dom_content_loaded_event_end, +-- null AS label load_event_startnull, +-- null AS label load_event_end, +-- null AS label first_paint, +-- null AS label first_contentful_paint, +-- null AS label speed_index, +-- null AS label visually_complete, +-- null AS label time_to_interactive, +-- min_fps, +-- avg_fps, +-- max_fps, +-- min_cpu, +-- avg_cpu, +-- max_cpu, +-- min_total_js_heap_size, +-- avg_total_js_heap_size, +-- max_total_js_heap_size, +-- min_used_js_heap_size, +-- avg_used_js_heap_size, +-- max_used_js_heap_size, +-- null AS label type, +-- null AS label header_size, +-- null AS label encoded_body_size, +-- null AS label decoded_body_size, +-- null AS label success, +-- null AS label method, +-- null AS label status +-- FROM performance +-- UNION ALL +-- SELECT session_id, +-- project_id, +-- 'RESOURCE' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- null AS label, +-- null AS hesitation_time, +-- null AS name, +-- null AS payload, +-- null AS level, +-- null AS source, +-- null AS message, +-- null AS error_id, +-- duration, +-- null AS context, +-- null AS container_type, +-- null AS container_id, +-- null AS container_name, +-- null AS container_src, +-- url, +-- null AS request_start, +-- null AS response_start, +-- null AS response_end, +-- null AS dom_content_loaded_event_start, +-- null AS dom_content_loaded_event_end, +-- null AS load_event_start, +-- null AS load_event_end, +-- null AS first_paint, +-- null AS first_contentful_paint, +-- null AS speed_index, +-- null AS visually_complete, +-- null AS time_to_interactive, +-- null AS min_fps, +-- null AS avg_fps, +-- null AS max_fps, +-- null AS min_cpu, +-- null AS avg_cpu, +-- null AS max_cpu, +-- null AS min_total_js_heap_size, +-- null AS avg_total_js_heap_size, +-- null AS max_total_js_heap_size, +-- null AS min_used_js_heap_size, +-- null AS avg_used_js_heap_size, +-- null AS max_used_js_heap_size, +-- type, +-- header_size, +-- encoded_body_size, +-- decoded_body_size, +-- success, +-- method, +-- status +-- FROM resources); +-- +-- +-- INSERT INTO eng_t.events4(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, label, hesitation_time) +-- SELECT session_id, +-- project_id, +-- 'CLICK' AS event_type, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- label, +-- hesitation_time +-- FROM clicks; +-- +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, +-- user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, +-- source, name, message, error_id) +-- +-- SELECT 'ERROR' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- source, +-- name, +-- message, +-- error_id +-- FROM errors; +-- +-- +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, label) +-- +-- SELECT 'INPUT' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- label +-- FROM inputs; +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, duration, context, container_type, container_id, container_name, +-- container_src) +-- SELECT 'LONGTASK' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- duration, +-- context, +-- container_type, +-- container_id, +-- container_name, +-- container_src +-- FROM longtasks; +-- +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, url, request_start, response_start, response_end, +-- dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, +-- load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, +-- time_to_interactive) +-- SELECT 'PAGE' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- url, +-- request_start, +-- response_start, +-- response_end, +-- dom_content_loaded_event_start, +-- dom_content_loaded_event_end, +-- load_event_start, +-- load_event_end, +-- first_paint, +-- first_contentful_paint, +-- speed_index, +-- visually_complete, +-- time_to_interactive +-- FROM pages; +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, +-- min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, +-- min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size) +-- SELECT 'PERFORMANCE' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- min_fps, +-- avg_fps, +-- max_fps, +-- min_cpu, +-- avg_cpu, +-- max_cpu, +-- min_total_js_heap_size, +-- avg_total_js_heap_size, +-- max_total_js_heap_size, +-- min_used_js_heap_size, +-- avg_used_js_heap_size, +-- max_used_js_heap_size +-- FROM performance; +-- +-- INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, +-- user_os_version, user_browser, user_browser_version, user_device, user_device_type, +-- user_country, datetime, url, type, duration, header_size, encoded_body_size, +-- decoded_body_size, success, method, status) +-- SELECT 'RESOURCE' AS event_type, +-- session_id, +-- project_id, +-- tracker_version, +-- rev_id, +-- user_uuid, +-- user_os, +-- user_os_version, +-- user_browser, +-- user_browser_version, +-- user_device, +-- user_device_type, +-- user_country, +-- datetime, +-- url, +-- type, +-- duration, +-- header_size, +-- encoded_body_size, +-- decoded_body_size, +-- success, +-- method, +-- status +-- FROM resources; +-- +-- +-- SELECT table, formatReadableSize(size) as size, rows, days, formatReadableSize(avgDaySize) as avgDaySize +-- FROM ( +-- SELECT table, +-- sum(bytes) AS size, +-- sum(rows) AS rows, +-- min(min_date) AS min_date, +-- max(max_date) AS max_date, +-- (max_date - min_date) AS days, +-- size / (max_date - min_date) AS avgDaySize +-- FROM system.parts +-- WHERE active +-- GROUP BY table +-- ORDER BY rows DESC +-- ); +-- +-- SELECT database, +-- table, +-- formatReadableSize(sum(bytes)) as size, +-- min(min_date) as min_date, +-- max(max_date) as max_date +-- FROM system.parts +-- WHERE active +-- GROUP BY database, table; +-- +-- SELECT count(*) +-- FROM single_t.events; +-- -- 449 484 932 +-- -- 449 484 932 +-- +-- SELECT (SELECT count(*) FROM clicks) + (SELECT count(*) FROM inputs) + (SELECT count(*) FROM longtasks) + +-- (SELECT count(*) FROM errors) + (SELECT count(*) FROM pages) + (SELECT count(*) FROM resources) + +-- (SELECT count(*) FROM performance) AS totl; +-- +-- +-- +-- CREATE TABLE IF NOT EXISTS single_t.events3 +-- ( +-- session_id UInt64, +-- project_id UInt32, +-- event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), +-- tracker_version LowCardinality(String), +-- rev_id Nullable(String), +-- user_uuid UUID, +-- user_os LowCardinality(String), +-- user_os_version LowCardinality(Nullable(String)), +-- user_browser LowCardinality(String), +-- user_browser_version LowCardinality(Nullable(String)), +-- user_device Nullable(String), +-- user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), +-- user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), +-- datetime DateTime, +-- label Nullable(String), +-- hesitation_time Nullable(UInt32), +-- name Nullable(String), +-- payload Nullable(String), +-- level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), +-- source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), +-- message Nullable(String), +-- error_id Nullable(String), +-- duration Nullable(UInt16), +-- context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), +-- container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), +-- container_id Nullable(String), +-- container_name Nullable(String), +-- container_src Nullable(String), +-- url Nullable(String), +-- url_host Nullable(String) MATERIALIZED lower(domain(url)), +-- url_path Nullable(String) MATERIALIZED lower(pathFull(url)), +-- request_start Nullable(UInt16), +-- response_start Nullable(UInt16), +-- response_end Nullable(UInt16), +-- dom_content_loaded_event_start Nullable(UInt16), +-- dom_content_loaded_event_end Nullable(UInt16), +-- load_event_start Nullable(UInt16), +-- load_event_end Nullable(UInt16), +-- first_paint Nullable(UInt16), +-- first_contentful_paint Nullable(UInt16), +-- speed_index Nullable(UInt16), +-- visually_complete Nullable(UInt16), +-- time_to_interactive Nullable(UInt16), +-- ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), +-- minus(response_start, request_start), Null), +-- ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), +-- minus(response_end, request_start), Null), +-- response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), +-- minus(response_end, response_start), Null), +-- dom_building_time Nullable(UInt16) MATERIALIZED if( +-- greaterOrEquals(dom_content_loaded_event_start, response_end), +-- minus(dom_content_loaded_event_start, response_end), Null), +-- dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( +-- greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), +-- minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), +-- load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), +-- minus(load_event_end, load_event_start), Null), +-- min_fps Nullable(UInt8), +-- avg_fps Nullable(UInt8), +-- max_fps Nullable(UInt8), +-- min_cpu Nullable(UInt8), +-- avg_cpu Nullable(UInt8), +-- max_cpu Nullable(UInt8), +-- min_total_js_heap_size Nullable(UInt64), +-- avg_total_js_heap_size Nullable(UInt64), +-- max_total_js_heap_size Nullable(UInt64), +-- min_used_js_heap_size Nullable(UInt64), +-- avg_used_js_heap_size Nullable(UInt64), +-- max_used_js_heap_size Nullable(UInt64), +-- type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), +-- header_size Nullable(UInt16), +-- encoded_body_size Nullable(UInt32), +-- decoded_body_size Nullable(UInt32), +-- compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), +-- success Nullable(UInt8), +-- method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), +-- status Nullable(UInt16) +-- ) ENGINE = MergeTree +-- PARTITION BY toDate(datetime) +-- ORDER BY (project_id, datetime,event_type); +-- +-- -- INSERT INTO eng_t.events42(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label, hesitation_time, name, payload, level, source, message, error_id, duration, context, container_type, container_id, container_name, container_src, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, time_to_interactive, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, type, header_size, encoded_body_size, decoded_body_size, success, method, status) +-- -- SELECT session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label, hesitation_time, name, payload, level, source, message, error_id, duration, context, container_type, container_id, container_name, container_src, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, time_to_interactive, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, type, header_size, encoded_body_size, decoded_body_size, success, method, status FROM single_t.events; + +CREATE TABLE IF NOT EXISTS single_t.events3 +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + tracker_version LowCardinality(String), + rev_id Nullable(String), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(BOOLEAN), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16) +) ENGINE = MergeTree + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime, event_type); + +INSERT INTO eng_t.events4(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, hesitation_time) +SELECT session_id, + project_id, + 'CLICK' AS event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + hesitation_time +FROM clicks +WHERE mod(session_id, 2) = 1; + + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, + user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, + source, name, message, error_id) + +SELECT 'ERROR' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + source, + name, + message, + error_id +FROM errors +WHERE mod(session_id, 2) = 0; +TRUNCATE TABLE eng_t.events4; +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label) + +SELECT 'INPUT' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label +FROM inputs +WHERE mod(session_id, 2) = 0; + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, duration, context, container_type, container_id, container_name, + container_src) +SELECT 'LONGTASK' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + context, + container_type, + container_id, + container_name, + container_src +FROM longtasks +WHERE mod(session_id, 2) = 0; + + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, request_start, response_start, response_end, + dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, + load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, + time_to_interactive) +SELECT 'PAGE' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint, + speed_index, + visually_complete, + time_to_interactive +FROM pages +WHERE mod(session_id, 2) = 0; + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, + min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, + min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size) +SELECT 'PERFORMANCE' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size +FROM performance +WHERE mod(session_id, 2) = 0; + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status) +SELECT 'RESOURCE' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + if(status IS NOT NULL, status = 1, null) AS status +FROM resources +WHERE type != 'fetch' + AND mod(session_id, 2) = 0; + +INSERT INTO eng_t.events4(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status) +SELECT 'REQUEST' AS event_type, + session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + if(status IS NOT NULL, status = 1, null) AS status +FROM resources +WHERE type = 'fetch' + AND mod(session_id, 2) = 0; + +CREATE TABLE IF NOT EXISTS eng_t.events4 +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + tracker_version LowCardinality(String), + rev_id Nullable(String), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(BOOLEAN), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16) +) ENGINE = Join(ALL, INNER, session_id); + +TRUNCATE TABLE eng_t.events4; + + +-- merge metadata with events +CREATE DATABASE full_meerge; + +CREATE TABLE IF NOT EXISTS massive.events6 +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + tracker_version LowCardinality(String), + rev_id Nullable(String), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(UInt8), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16), + user_id Nullable(String), + user_anonymous_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String) +) ENGINE = MergeTree + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime); +-- TTL datetime + INTERVAL 1 MONTH; +INSERT INTO massive.events6(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, hesitation_time, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10) +SELECT session_id + 6651141467121565 * 3 AS session_id, + project_id, + 'CLICK' AS event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + hesitation_time, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.clicks + LEFT JOIN default.sessions_metadata USING (session_id); + + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, + datetime, source, name, message, error_id, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) + +SELECT 'ERROR' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + source, + name, + message, + error_id, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.errors + LEFT JOIN default.sessions_metadata USING (session_id); + + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) + +SELECT 'INPUT' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.inputs + LEFT JOIN default.sessions_metadata USING (session_id); + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, duration, context, container_type, container_id, container_name, + container_src, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) +SELECT 'LONGTASK' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + context, + container_type, + container_id, + container_name, + container_src, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.longtasks + LEFT JOIN default.sessions_metadata USING (session_id); + + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, request_start, response_start, response_end, + dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, + load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, + time_to_interactive, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) +SELECT 'PAGE' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint, + speed_index, + visually_complete, + time_to_interactive, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.pages + LEFT JOIN default.sessions_metadata USING (session_id); + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, + min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, + min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) +SELECT 'PERFORMANCE' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.performance + LEFT JOIN default.sessions_metadata USING (session_id); + +INSERT INTO massive.events6(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) +SELECT 'RESOURCE' AS event_type, + session_id + 6651141467121565 * 3 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + if(status IS NOT NULL, status = 1, null) AS status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.resources + LEFT JOIN default.sessions_metadata USING (session_id) +WHERE type != 'fetch'; + +INSERT INTO massive2.events7(event_type, session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) +SELECT 'REQUEST' AS event_type, + session_id + 6651141467121565 * 4 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + if(status IS NOT NULL, status = 1, null) AS status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM default.resources + LEFT JOIN default.sessions_metadata USING (session_id) +WHERE type = 'fetch' + AND mod(project_id, 2) = 0; + +-- -- TO GENERATE RANDOM USER IDS +-- INSERT INTO sessions_metadata(session_id, user_id, datetime, project_id, user_device_type) +-- SELECT session_id, +-- arrayElement( +-- array('Ze2wc7lvYi', 'NYd7m0Ytg8', 'qgNpvEkXap', 'wvWqM4Ow2G', 'n5Y6DK7ZdP', 'uW4SEYjXxI', 't4EfJiNxk9', +-- 'qWQ8WuIRLS', 'fnRWCwkFyB', '8wf298MFWR', 'G3A3DL0Fdd', 'cQcZHNNiAJ', 'MKcW2adQ38', 'OBzk9EFxVe', +-- '8SBiqoFail', '3Wh9Ur0eOr', 'z6KuuxiPXX', '7j4HaReEsF', 'Ros0kDOVeV', 'PvHi3cBkgV', 'HLjUo6oBlJ', +-- '4Tmi34faA0', 'O9ZATbPjaB', '7ATvuWQCIH', 'kXW4LHnW5X', 'HIHc9TTyTc', 'i5p9jRe7I0', '7dRnUEFoZO', +-- 'u3PDLkI5uG', 'HTYjxmDJCG', '6hKHjcKniO', 'qmPNUWgDIx', 'RfoN9oeYZD', 'HHXpBaYm3k', 'VdpZDfnL9J', +-- 'Qfwa1dPrrF', 'cgdD2GfFVT', 'iRvT6l7qj3', 'QokprB2GMV', 'umqISqbncX', '7bvRdQ4al3', 'VGKZAUIRjy', +-- 'SNTEGLKbCD', 'zfUaVSD8Jn', 'De7zUojKNt', 'lXiotVRkil', 'bQaDX5kESw', 'tngESCaH6I', 'uucUZvTpPd', +-- 'BFJpni8D3I'), mod(session_id, 50)) AS user_id, +-- datetime, +-- project_id, +-- user_device_type +-- FROM sessions +-- WHERE project_id = 2460; + +INSERT INTO massive2.sessions2(session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, + user_browser, user_browser_version, user_device, user_device_type, user_country, + datetime, + duration, events_count, errors_count, utm_source, utm_medium, utm_campaign) +SELECT session_id + 6651141467121565 * 4 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign +FROM default.sessions; + + +CREATE DATABASE massive2; +CREATE TABLE IF NOT EXISTS massive2.events7 +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + tracker_version LowCardinality(String), + rev_id Nullable(String), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(UInt8), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16), + user_id Nullable(String), + user_anonymous_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String) +) ENGINE = MergeTree + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime, event_type); + + + +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 43 +FROM massive.events6 +WHERE event_type = 'REQUEST' + AND mod(project_id, 2) = 0; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 42 +FROM massive.events6 +WHERE event_type = 'REQUEST' + AND mod(project_id, 2) = 1; + +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 41 +FROM massive.events6 +WHERE event_type = 'RESOURCE' + AND mod(project_id, 2) = 0; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, type, duration, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + type, + duration, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 40 +FROM massive.events6 +WHERE event_type = 'RESOURCE' + AND mod(project_id, 2) = 1; + +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, hesitation_time, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + hesitation_time, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'CLICK'; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, + min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, + min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, + metadata_7, metadata_8, metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'PERFORMANCE'; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, url, request_start, response_start, response_end, + dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, + load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, + time_to_interactive, user_id, user_anonymous_id, metadata_1, metadata_2, metadata_3, + metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, + riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + url, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint, + speed_index, + visually_complete, + time_to_interactive, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'PAGE'; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, user_id, user_anonymous_id, metadata_1, metadata_2, + metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, + metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'INPUT'; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, source, name, message, error_id, user_id, user_anonymous_id, + metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, + metadata_8, metadata_9, metadata_10, riteration) +SELECT session_id + 6651141467121565 * 4 AS session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + source, + name, + message, + error_id, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'ERROR'; +INSERT INTO massive2.events7(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, duration, context, container_type, container_id, container_name, + container_src, user_id, user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, + metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, riteration) +SELECT session_id + 6651141467121565 * 4 AS session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + context, + container_type, + container_id, + container_name, + container_src, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 4 +FROM massive.events6 +WHERE event_type = 'LONGTASK'; + +ALTER TABLE massive2.events7 + ADD COLUMN riteration UInt8 DEFAULT 0; +ALTER TABLE massive2.sessions2 + ADD COLUMN riteration UInt8 DEFAULT 0; + + + +INSERT INTO massive2.sessions2(session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, + user_browser, user_browser_version, user_device, user_device_type, user_country, + datetime, + duration, events_count, errors_count, utm_source, utm_medium, utm_campaign, riteration) +SELECT session_id + 6651141467121565 * 4 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign, + 4 +FROM massive.sessions; + +SELECT COUNT(*) +FROM massive2.events7; + +CREATE DATABASE massive30; +CREATE TABLE IF NOT EXISTS massive30.events30 +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + tracker_version LowCardinality(String), + rev_id Nullable(String), + user_uuid UUID, + user_os LowCardinality(String), + user_os_version LowCardinality(Nullable(String)), + user_browser LowCardinality(String), + user_browser_version LowCardinality(Nullable(String)), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(UInt8), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16), + user_id Nullable(String), + user_anonymous_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String) +) ENGINE = MergeTree + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, event_type); + +ALTER TABLE massive30.events30 + ADD COLUMN riteration UInt8; + +INSERT INTO massive30.events30(session_id, project_id, event_type, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, label, hesitation_time, name, payload, level, source, message, + error_id, duration, context, container_type, container_id, container_name, container_src, + url, request_start, response_start, response_end, dom_content_loaded_event_start, + dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, + first_contentful_paint, speed_index, visually_complete, time_to_interactive, min_fps, + avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, + avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, + avg_used_js_heap_size, max_used_js_heap_size, type, header_size, encoded_body_size, + decoded_body_size, success, method, status, user_id, user_anonymous_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, + metadata_9, metadata_10, riteration) +SELECT session_id, + project_id, + event_type, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + label, + hesitation_time, + name, + payload, + level, + source, + message, + error_id, + duration, + context, + container_type, + container_id, + container_name, + container_src, + url, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint, + speed_index, + visually_complete, + time_to_interactive, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + type, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status, + user_id, + user_anonymous_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10, + 9 AS riteration +FROM massive2.events7 +WHERE mod(project_id, 10) = 9; +-- ORDER BY datetime LIMIT 500000; + +DROP TABLE massive30.events30; + + +DESCRIBE TABLE massive2.events7; + + +-- ----------------------------------------------------- +CREATE DATABASE massive_split; +CREATE TABLE IF NOT EXISTS massive_split.events_s +( + session_id UInt64, + project_id UInt32, + event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8), + datetime DateTime, + label Nullable(String), + hesitation_time Nullable(UInt32), + name Nullable(String), + payload Nullable(String), + level Nullable(Enum8('info'=0, 'error'=1)) DEFAULT if(event_type == 'CUSTOM', 'info', null), + source Nullable(Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9)), + message Nullable(String), + error_id Nullable(String), + duration Nullable(UInt16), + context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), + container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), + container_id Nullable(String), + container_name Nullable(String), + container_src Nullable(String), + url Nullable(String), + url_host Nullable(String) MATERIALIZED lower(domain(url)), + url_path Nullable(String) MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), + minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), + minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), + minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_start, response_end), + minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if( + greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), + minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), + minus(load_event_end, load_event_start), Null), + min_fps Nullable(UInt8), + avg_fps Nullable(UInt8), + max_fps Nullable(UInt8), + min_cpu Nullable(UInt8), + avg_cpu Nullable(UInt8), + max_cpu Nullable(UInt8), + min_total_js_heap_size Nullable(UInt64), + avg_total_js_heap_size Nullable(UInt64), + max_total_js_heap_size Nullable(UInt64), + min_used_js_heap_size Nullable(UInt64), + avg_used_js_heap_size Nullable(UInt64), + max_used_js_heap_size Nullable(UInt64), + type Nullable(Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4)), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success Nullable(UInt8), + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16) +) ENGINE = MergeTree + PARTITION BY toYYYYMM(datetime) + ORDER BY (project_id, datetime, event_type, session_id); + +CREATE TABLE IF NOT EXISTS massive_split.sessions_s +( + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + duration UInt32, + pages_count UInt16, + events_count UInt16, + errors_count UInt16, + utm_source Nullable(String), + utm_medium Nullable(String), + utm_campaign Nullable(String), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 1 MONTH + SETTINGS index_granularity = 512; + + +CREATE TABLE IF NOT EXISTS massive_split.metadata_s +( + session_id UInt64, + project_id UInt32, + datetime DateTime, + user_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime, session_id); + +INSERT INTO massive_split.sessions_s(session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, user_device_type, + user_country, datetime, duration, pages_count, events_count, errors_count, + utm_source, utm_medium, utm_campaign, riteration) +SELECT session_id + 6651141467121565 * 4 AS session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + mod(rand(), 100) AS pages_count, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign, + 4 AS riteration +FROM default.sessions; + +ALTER TABLE massive_split.sessions_s + ADD COLUMN riteration UInt8; + +INSERT INTO massive_split.metadata_s(session_id, project_id, datetime, user_id, metadata_1, metadata_2, metadata_3, + metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, + metadata_10) +SELECT session_id, + project_id, + datetime, + user_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM massive2.events7 AS s +LIMIT 1 BY session_id; + +INSERT INTO massive_split.events_s(session_id, project_id, event_type, datetime, label, hesitation_time, name, payload, + level, source, message, error_id, duration, context, container_type, container_id, + container_name, container_src, url, request_start, response_start, response_end, + dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, + load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, + time_to_interactive, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, + min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, + min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, type, + header_size, encoded_body_size, decoded_body_size, success, method, status) +SELECT session_id, + project_id, + event_type, + datetime, + label, + hesitation_time, + name, + payload, + level, + source, + message, + error_id, + duration, + context, + container_type, + container_id, + container_name, + container_src, + url, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint, + speed_index, + visually_complete, + time_to_interactive, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + type, + header_size, + encoded_body_size, + decoded_body_size, + success, + method, + status +FROM massive2.events7; + +SELECT COUNT(*) +FROM massive_split.sessions_s; +SELECT COUNT(*) +FROM massive_split.metadata_s; +SELECT COUNT(*) +FROM massive_split.events_s; +SELECT COUNT(*) +FROM massive2.events7; +-- SELECT COUNT(*) FROM massive2.sessions2; + + +CREATE TABLE IF NOT EXISTS massive_split.sessions_meta +( + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + duration UInt32, + pages_count UInt16, + events_count UInt16, + errors_count UInt16, + utm_source Nullable(String), + utm_medium Nullable(String), + utm_campaign Nullable(String), + user_id Nullable(String), + metadata_1 Nullable(String), + metadata_2 Nullable(String), + metadata_3 Nullable(String), + metadata_4 Nullable(String), + metadata_5 Nullable(String), + metadata_6 Nullable(String), + metadata_7 Nullable(String), + metadata_8 Nullable(String), + metadata_9 Nullable(String), + metadata_10 Nullable(String), + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 1 MONTH + SETTINGS index_granularity = 512; + +INSERT INTO massive_split.sessions_meta(session_id, project_id, tracker_version, rev_id, user_uuid, user_os, + user_os_version, user_browser, user_browser_version, user_device, + user_device_type, user_country, datetime, duration, pages_count, events_count, + errors_count, utm_source, utm_medium, utm_campaign, user_id, metadata_1, + metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, + metadata_8, metadata_9, metadata_10) +SELECT session_id, + project_id, + tracker_version, + rev_id, + user_uuid, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_country, + datetime, + duration, + pages_count, + events_count, + errors_count, + utm_source, + utm_medium, + utm_campaign, + user_id, + metadata_1, + metadata_2, + metadata_3, + metadata_4, + metadata_5, + metadata_6, + metadata_7, + metadata_8, + metadata_9, + metadata_10 +FROM massive_split.sessions_s AS s + LEFT JOIN massive_split.metadata_s AS m ON (s.project_id = m.project_id AND s.session_id = m.session_id); diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/queries.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/queries.sql new file mode 100644 index 000000000..556209c79 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/queries.sql @@ -0,0 +1,983 @@ +-- Q1 +SELECT session_id +-- FROM massive2.events7 +-- FROM events_l7d_mv +FROM events_l24h_mv +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + + +-- Q1.1 +SELECT session_id +FROM massive2.events7 +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q1.2 +SELECT session_id +FROM +-- massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +events_l24h_mv + INNER JOIN metadata_l24h_mv USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q1.2.1 +SELECT session_id +FROM +-- massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- events_l7d_mv AS events_s +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) +events_l24h_mv AS events_s + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) +WHERE events_s.project_id = 2460 + AND events_s.datetime >= '2022-04-02 00:00:00' + AND events_s.datetime <= '2022-04-03 00:00:00' +-- AND events_s.datetime <= '2022-04-10 00:00:00' +-- AND events_s.datetime <= '2022-05-02 00:00:00' + AND metadata_s.project_id = 2460 + AND metadata_s.datetime >= '2022-04-02 00:00:00' + AND metadata_s.datetime <= '2022-04-03 00:00:00' +-- AND metadata_s.datetime <= '2022-04-10 00:00:00' +-- AND metadata_s.datetime <= '2022-05-02 00:00:00' + AND metadata_s.user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q1.3 +SELECT session_id +FROM +-- massive_split.events_s +-- events_l7d_mv +events_l24h_mv + INNER JOIN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q1.4 +SELECT session_id +FROM (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta + -- INNER JOIN massive_split.events_s USING (session_id) +-- INNER JOIN events_l7d_mv USING (session_id) + INNER JOIN events_l24h_mv USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q1.5 +SELECT session_id +-- FROM massive_split.events_s +-- FROM events_l7d_mv +FROM events_l24h_mv +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive2.events7 +-- FROM events_l7d_mv + FROM events_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' + -- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.1 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 + FROM massive2.events7 + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.2 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv +-- INNER JOIN metadata_l7d_mv USING (session_id) + FROM events_l24h_mv + INNER JOIN metadata_l24h_mv USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.2.1 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv AS events_s +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) + FROM events_l24h_mv AS events_s + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) + WHERE events_s.project_id = 2460 + AND events_s.datetime >= '2022-04-02 00:00:00' + AND events_s.datetime <= '2022-04-03 00:00:00' +-- AND events_s.datetime <= '2022-04-10 00:00:00' +-- AND events_s.datetime <= '2022-05-02 00:00:00' + AND metadata_s.project_id = 2460 + AND metadata_s.datetime >= '2022-04-02 00:00:00' + AND metadata_s.datetime <= '2022-04-03 00:00:00' +-- AND metadata_s.datetime <= '2022-04-10 00:00:00' +-- AND metadata_s.datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.3 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- FROM events_l7d_mv + FROM events_l24h_mv + INNER JOIN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' + -- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.4 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 + FROM (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta + -- INNER JOIN massive_split.events_s USING (session_id) +-- INNER JOIN events_l7d_mv USING (session_id) + INNER JOIN events_l24h_mv USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' + -- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q2.5 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- FROM events_l7d_mv + FROM events_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd')) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3 +SELECT session_id +-- FROM massive_split.events_s +-- FROM events_l7d_mv +FROM events_l24h_mv +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.1 +SELECT session_id +FROM massive2.events7 +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.2 +SELECT session_id +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv +-- INNER JOIN metadata_l7d_mv USING (session_id) +FROM events_l24h_mv + INNER JOIN metadata_l24h_mv USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.2.1 +SELECT session_id +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv AS events_s +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) +FROM events_l24h_mv AS events_s + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) +WHERE events_s.project_id = 2460 + AND events_s.datetime >= '2022-04-02 00:00:00' + AND events_s.datetime <= '2022-04-03 00:00:00' +-- AND events_s.datetime <= '2022-04-10 00:00:00' +-- AND events_s.datetime <= '2022-05-02 00:00:00' + AND (events_s.event_type = 'CLICK' OR events_s.event_type = 'REQUEST') + AND metadata_s.project_id = 2460 + AND metadata_s.datetime >= '2022-04-02 00:00:00' + AND metadata_s.datetime <= '2022-04-03 00:00:00' +-- AND metadata_s.datetime <= '2022-04-10 00:00:00' +-- AND metadata_s.datetime <= '2022-05-02 00:00:00' + AND metadata_s.user_id = 'uucUZvTpPd' +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.3 +SELECT session_id +-- FROM massive_split.events_s +-- FROM events_l7d_mv +FROM events_l24h_mv + INNER JOIN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.4 +SELECT session_id +FROM (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta + -- INNER JOIN massive_split.events_s USING (session_id) +-- INNER JOIN events_l7d_mv USING (session_id) + INNER JOIN events_l24h_mv USING (session_id) +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q3.5 +SELECT session_id +-- FROM massive_split.events_s +-- FROM events_l7d_mv +FROM events_l24h_mv +WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- FROM events_l7d_mv + FROM events_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST')) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.1 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 + FROM massive2.events7 + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.2 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv +-- INNER JOIN metadata_l7d_mv USING (session_id) + FROM events_l24h_mv + INNER JOIN metadata_l24h_mv USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND user_id = 'uucUZvTpPd') +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.2.1 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv AS events_s +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) + FROM events_l24h_mv AS events_s + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) + WHERE events_s.project_id = 2460 + AND events_s.datetime >= '2022-04-02 00:00:00' + AND events_s.datetime <= '2022-04-03 00:00:00' +-- AND events_s.datetime <= '2022-04-10 00:00:00' +-- AND events_s.datetime <= '2022-05-02 00:00:00' + AND (events_s.event_type = 'CLICK' OR events_s.event_type = 'REQUEST') + AND metadata_s.user_id = 'uucUZvTpPd' + AND metadata_s.project_id = 2460 + AND metadata_s.datetime >= '2022-04-02 00:00:00' + AND metadata_s.datetime <= '2022-04-03 00:00:00' + -- AND metadata_s.datetime <= '2022-04-10 00:00:00' +-- AND metadata_s.datetime <= '2022-05-02 00:00:00' + ) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.3 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 +-- FROM massive_split.events_s +-- FROM events_l7d_mv + FROM events_l24h_mv + INNER JOIN (SELECT DISTINCT session_id +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST')) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.4 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 + FROM (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta + -- INNER JOIN massive_split.events_s USING (session_id) +-- INNER JOIN events_l7d_mv USING (session_id) + INNER JOIN events_l24h_mv USING (session_id) + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST')) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- Q4.5 +SELECT session_id +FROM (SELECT session_id, + datetime, + event_type = 'CLICK' AND label ILIKE '%invoice%' AS c1, + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%' AS c2 + FROM events_l24h_mv +-- FROM events_l7d_mv +-- FROM massive_split.events_s + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND (event_type = 'CLICK' OR event_type = 'REQUEST') + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd')) +GROUP BY session_id +HAVING windowFunnel(99999)(datetime, c1, c2) = 2 +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, session_id + FROM massive2.events7 AS events + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.1 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, session_id + FROM massive2.events7 AS events + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.2 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv AS events +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) + FROM events_l24h_mv AS events + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.3 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- INNER JOIN massive_split.metadata_s USING (session_id) +-- FROM events_l7d_mv AS events +-- INNER JOIN metadata_l7d_mv AS metadata_s USING (session_id) + FROM events_l24h_mv AS events + INNER JOIN metadata_l24h_mv AS metadata_s USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.4 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- FROM events_l7d_mv AS events + FROM events_l24h_mv AS events + INNER JOIN (SELECT DISTINCT session_id, + user_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.4-A +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- FROM events_l7d_mv AS events + FROM events_l24h_mv AS events + INNER JOIN (SELECT DISTINCT session_id, + user_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' + -- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) AS meta USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.5 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, session_id + FROM (SELECT DISTINCT session_id, + user_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta + -- INNER JOIN massive_split.events_s AS events USING (session_id) +-- INNER JOIN events_l7d_mv AS events USING (session_id) + INNER JOIN events_l24h_mv AS events USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.6 +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- FROM events_l7d_mv AS events + FROM events_l24h_mv AS events + INNER JOIN (SELECT DISTINCT session_id, + user_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') AS meta USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + AND user_id = 'uucUZvTpPd') + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QU1.6-A +SELECT user_id, COUNT(session_id) +FROM (SELECT user_id, + session_id +-- FROM massive_split.events_s AS events +-- FROM events_l7d_mv AS events + FROM events_l24h_mv AS events + INNER JOIN (SELECT DISTINCT session_id, + user_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' + -- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) AS meta USING (session_id) + WHERE events.project_id = 2460 + AND events.datetime >= '2022-04-02 00:00:00' + AND events.datetime <= '2022-04-03 00:00:00' +-- AND events.datetime <= '2022-04-10 00:00:00' +-- AND events.datetime <= '2022-05-02 00:00:00' + AND session_id IN (SELECT DISTINCT session_id +-- FROM massive_split.metadata_s +-- FROM metadata_l7d_mv + FROM metadata_l24h_mv + WHERE project_id = 2460 + AND datetime >= '2022-04-02 00:00:00' + AND datetime <= '2022-04-03 00:00:00' +-- AND datetime <= '2022-04-10 00:00:00' +-- AND datetime <= '2022-05-02 00:00:00' + ) + GROUP BY user_id, session_id + HAVING windowFunnel(99999)(datetime, event_type = 'CLICK' AND label ILIKE '%invoice%', + event_type = 'REQUEST' AND url ILIKE '%letsdeel.com/pay%') = 2 + ) AS filtred_sessions +GROUP BY user_id +LIMIT 10 +SETTINGS +max_threads = 4; + +-- QM4: +SELECT timestamp, + groupArray([toString(t.type), toString(t.count)]) AS types +FROM (SELECT toUnixTimestamp(toStartOfInterval(events7.datetime, INTERVAL 37565 second)) * 1000 AS timestamp, + events7.type, + COUNT(events7.session_id) AS count +-- FROM massive_split.events_s AS events7 +-- FROM events_l7d_mv AS events7 + FROM events_l24h_mv AS events7 + WHERE events7.project_id = toUInt32(2460) + AND toStartOfInterval(events7.datetime, INTERVAL 37565 second) >= '2022-04-02 00:00:00' + AND events7.datetime <= '2022-04-03 00:00:00' +-- AND events7.datetime <= '2022-04-10 00:00:00' +-- AND events7.datetime < '2022-05-02 00:00:00' + AND events7.event_type = 'RESOURCE' + GROUP BY timestamp, events7.type + ORDER BY timestamp) AS t +GROUP BY timestamp + SETTINGS + max_threads = 4; From 6f6b832a18e6dd364def4c7fd60b62b82cc35820 Mon Sep 17 00:00:00 2001 From: sylenien Date: Mon, 11 Jul 2022 12:49:28 +0200 Subject: [PATCH 008/553] fix(ui): fix dashbaord selector --- .../Dashboard/components/DashboardView/DashboardView.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/app/components/Dashboard/components/DashboardView/DashboardView.tsx b/frontend/app/components/Dashboard/components/DashboardView/DashboardView.tsx index d376e97f0..108d961a5 100644 --- a/frontend/app/components/Dashboard/components/DashboardView/DashboardView.tsx +++ b/frontend/app/components/Dashboard/components/DashboardView/DashboardView.tsx @@ -65,6 +65,9 @@ function DashboardView(props: Props) { trimQuery(); } }, []); + useEffect(() => { + dashboardStore.selectDefaultDashboard(); + }, [siteId]) const onAddWidgets = () => { dashboardStore.initDashboard(dashboard); From d1806b08efe54817da2a68391857a942c5268bef Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 11 Jul 2022 17:56:01 +0200 Subject: [PATCH 009/553] fix(ui) - devtools inputfields --- frontend/app/components/Session_/Console/ConsoleContent.js | 2 +- frontend/app/components/Session_/Fetch/Fetch.js | 2 +- frontend/app/components/Session_/LongTasks/LongTasks.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/Session_/Console/ConsoleContent.js b/frontend/app/components/Session_/Console/ConsoleContent.js index 880f660e7..0ffaa6032 100644 --- a/frontend/app/components/Session_/Console/ConsoleContent.js +++ b/frontend/app/components/Session_/Console/ConsoleContent.js @@ -60,7 +60,7 @@ export default class ConsoleContent extends React.PureComponent { activeTab: ALL, } onTabClick = activeTab => this.setState({ activeTab }) - onFilterChange = (e, { value }) => this.setState({ filter: value }) + onFilterChange = ({ target: { value }}) => this.setState({ filter: value }) render() { const { logs, isResult, additionalHeight, time } = this.props; diff --git a/frontend/app/components/Session_/Fetch/Fetch.js b/frontend/app/components/Session_/Fetch/Fetch.js index 46cc44d38..cf05f944d 100644 --- a/frontend/app/components/Session_/Fetch/Fetch.js +++ b/frontend/app/components/Session_/Fetch/Fetch.js @@ -28,7 +28,7 @@ export default class Fetch extends React.PureComponent { hasPreviousError: false, } - onFilterChange = (e, { value }) => { + onFilterChange = ({ target: { value } }) => { const { list } = this.props; const filterRE = getRE(value, 'i'); const filtered = list diff --git a/frontend/app/components/Session_/LongTasks/LongTasks.js b/frontend/app/components/Session_/LongTasks/LongTasks.js index 61f8b41ea..55f204ea4 100644 --- a/frontend/app/components/Session_/LongTasks/LongTasks.js +++ b/frontend/app/components/Session_/LongTasks/LongTasks.js @@ -29,7 +29,7 @@ export default class GraphQL extends React.PureComponent { state = { filter: "", } - onFilterChange = (e, { value }) => this.setState({ filter: value }) + onFilterChange = ({ target: { value } }) => this.setState({ filter: value }) jump = ({ time }) => { jump(time); From de9a11a7dace79d92b75d68dd5536fd56ef7414c Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Tue, 12 Jul 2022 17:40:24 +0200 Subject: [PATCH 010/553] fix random issues (tooltip, assist, button) (#596) * fix(ui) - live player loader * change(ui) - removed unused * change(ui) - button changes * fix(ui) - tooltip changes --- .../AssistActions/AssistActions.tsx | 261 +++++++++--------- .../EventsToggleButton.module.css | 7 - .../EventsToggleButton/EventsToggleButton.tsx | 35 --- .../Session/EventsToggleButton/index.js | 1 - .../app/components/Session/LiveSession.js | 18 +- .../Player/Overlay/ElementsMarker/Marker.tsx | 8 +- .../LiveSessionList/LiveSessionList.tsx | 112 ++++---- frontend/app/components/ui/Button/Button.tsx | 115 ++++---- .../components/ui/Pagination/Pagination.tsx | 13 +- frontend/app/components/ui/Popup/Popup.tsx | 15 +- 10 files changed, 287 insertions(+), 298 deletions(-) delete mode 100644 frontend/app/components/Session/EventsToggleButton/EventsToggleButton.module.css delete mode 100644 frontend/app/components/Session/EventsToggleButton/EventsToggleButton.tsx delete mode 100644 frontend/app/components/Session/EventsToggleButton/index.js diff --git a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx index de843e969..ba1a7cf0b 100644 --- a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx +++ b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx @@ -1,153 +1,166 @@ -import React, { useState, useEffect } from 'react' -import { Popup, Icon, IconButton } from 'UI' -import { connect } from 'react-redux' -import cn from 'classnames' +import React, { useState, useEffect } from 'react'; +import { Popup, Icon, Button, IconButton } from 'UI'; +import { connect } from 'react-redux'; +import cn from 'classnames'; import { toggleChatWindow } from 'Duck/sessions'; import { connectPlayer } from 'Player/store'; import ChatWindow from '../../ChatWindow'; -import { callPeer, requestReleaseRemoteControl, toggleAnnotation } from 'Player' +import { callPeer, requestReleaseRemoteControl, toggleAnnotation } from 'Player'; import { CallingState, ConnectionStatus, RemoteControlStatus } from 'Player/MessageDistributor/managers/AssistManager'; import RequestLocalStream from 'Player/MessageDistributor/managers/LocalStream'; import type { LocalStream } from 'Player/MessageDistributor/managers/LocalStream'; import { toast } from 'react-toastify'; import { confirm } from 'UI'; -import stl from './AassistActions.module.css' +import stl from './AassistActions.module.css'; function onClose(stream) { - stream.getTracks().forEach(t=>t.stop()); + stream.getTracks().forEach((t) => t.stop()); } function onReject() { - toast.info(`Call was rejected.`); + toast.info(`Call was rejected.`); } function onError(e) { - toast.error(typeof e === 'string' ? e : e.message); + toast.error(typeof e === 'string' ? e : e.message); } - interface Props { - userId: String, - toggleChatWindow: (state) => void, - calling: CallingState, - annotating: boolean, - peerConnectionStatus: ConnectionStatus, - remoteControlStatus: RemoteControlStatus, - hasPermission: boolean, - isEnterprise: boolean, + userId: String; + toggleChatWindow: (state) => void; + calling: CallingState; + annotating: boolean; + peerConnectionStatus: ConnectionStatus; + remoteControlStatus: RemoteControlStatus; + hasPermission: boolean; + isEnterprise: boolean; } -function AssistActions({ toggleChatWindow, userId, calling, annotating, peerConnectionStatus, remoteControlStatus, hasPermission, isEnterprise }: Props) { - const [ incomeStream, setIncomeStream ] = useState(null); - const [ localStream, setLocalStream ] = useState(null); - const [ callObject, setCallObject ] = useState<{ end: ()=>void } | null >(null); +function AssistActions({ + toggleChatWindow, + userId, + calling, + annotating, + peerConnectionStatus, + remoteControlStatus, + hasPermission, + isEnterprise, +}: Props) { + const [incomeStream, setIncomeStream] = useState(null); + const [localStream, setLocalStream] = useState(null); + const [callObject, setCallObject] = useState<{ end: () => void } | null>(null); - useEffect(() => { - return callObject?.end() - }, []) + useEffect(() => { + return callObject?.end(); + }, []); - useEffect(() => { - if (peerConnectionStatus == ConnectionStatus.Disconnected) { - toast.info(`Live session was closed.`); - } - }, [peerConnectionStatus]) - - function call() { - RequestLocalStream().then(lStream => { - setLocalStream(lStream); - setCallObject(callPeer( - lStream, - setIncomeStream, - lStream.stop.bind(lStream), - onReject, - onError - )); - }).catch(onError) - } - - const confirmCall = async () => { - if (await confirm({ - header: 'Start Call', - confirmButton: 'Call', - confirmation: `Are you sure you want to call ${userId ? userId : 'User'}?` - })) { - call() - } - } - - const onCall = calling === CallingState.OnCall || calling === CallingState.Reconnecting - const cannotCall = (peerConnectionStatus !== ConnectionStatus.Connected) || (isEnterprise && !hasPermission) - const remoteActive = remoteControlStatus === RemoteControlStatus.Enabled - - return ( -
- {(onCall || remoteActive) && ( - <> -
toggleAnnotation(!annotating) } - role="button" - > - -
-
- - )} -
{ + if (peerConnectionStatus == ConnectionStatus.Disconnected) { + toast.info(`Live session was closed.`); } - onClick={ requestReleaseRemoteControl } - role="button" - > - -
-
- - -
- -
- + }, [peerConnectionStatus]); -
- { onCall && callObject && } -
-
- ) + function call() { + RequestLocalStream() + .then((lStream) => { + setLocalStream(lStream); + setCallObject(callPeer(lStream, setIncomeStream, lStream.stop.bind(lStream), onReject, onError)); + }) + .catch(onError); + } + + const confirmCall = async () => { + if ( + await confirm({ + header: 'Start Call', + confirmButton: 'Call', + confirmation: `Are you sure you want to call ${userId ? userId : 'User'}?`, + }) + ) { + call(); + } + }; + + const onCall = calling === CallingState.OnCall || calling === CallingState.Reconnecting; + const cannotCall = peerConnectionStatus !== ConnectionStatus.Connected || (isEnterprise && !hasPermission); + const remoteActive = remoteControlStatus === RemoteControlStatus.Enabled; + + return ( +
+ {(onCall || remoteActive) && ( + <> +
toggleAnnotation(!annotating)} + role="button" + > + + {/* */} +
+
+ + )} +
+ + {/* */} +
+
+ + +
+ + {/* */} +
+ + +
+ {onCall && callObject && ( + + )} +
+
+ ); } -const con = connect(state => { - const permissions = state.getIn([ 'user', 'account', 'permissions' ]) || [] - return { - hasPermission: permissions.includes('ASSIST_CALL'), - isEnterprise: state.getIn([ 'user', 'account', 'edition' ]) === 'ee', - } -}, { toggleChatWindow }) +const con = connect( + (state) => { + const permissions = state.getIn(['user', 'account', 'permissions']) || []; + return { + hasPermission: permissions.includes('ASSIST_CALL'), + isEnterprise: state.getIn(['user', 'account', 'edition']) === 'ee', + }; + }, + { toggleChatWindow } +); -export default con(connectPlayer(state => ({ - calling: state.calling, - annotating: state.annotating, - remoteControlStatus: state.remoteControl, - peerConnectionStatus: state.peerConnectionStatus, -}))(AssistActions)) +export default con( + connectPlayer((state) => ({ + calling: state.calling, + annotating: state.annotating, + remoteControlStatus: state.remoteControl, + peerConnectionStatus: state.peerConnectionStatus, + }))(AssistActions) +); diff --git a/frontend/app/components/Session/EventsToggleButton/EventsToggleButton.module.css b/frontend/app/components/Session/EventsToggleButton/EventsToggleButton.module.css deleted file mode 100644 index 19eda2568..000000000 --- a/frontend/app/components/Session/EventsToggleButton/EventsToggleButton.module.css +++ /dev/null @@ -1,7 +0,0 @@ -.wrapper { - background-color: rgba(255, 255, 255, 1); - border-top-left-radius: 20px; - border-bottom-left-radius: 20px; - padding: 5px; - box-shadow: -1px 1px 1px rgba(0, 0, 0, 0.5); -} \ No newline at end of file diff --git a/frontend/app/components/Session/EventsToggleButton/EventsToggleButton.tsx b/frontend/app/components/Session/EventsToggleButton/EventsToggleButton.tsx deleted file mode 100644 index 4d4b70895..000000000 --- a/frontend/app/components/Session/EventsToggleButton/EventsToggleButton.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import React from 'react' -import { Icon, Popup } from 'UI' -import { connectPlayer, toggleEvents, scale } from 'Player'; -import cn from 'classnames' -import stl from './EventsToggleButton.module.css' - -function EventsToggleButton({ showEvents, toggleEvents }: any) { - const toggle = () => { - toggleEvents() - scale() - } - return ( - - - - ) -} - -export default connectPlayer(state => ({ - showEvents: !state.showEvents -}), { toggleEvents })(EventsToggleButton) - diff --git a/frontend/app/components/Session/EventsToggleButton/index.js b/frontend/app/components/Session/EventsToggleButton/index.js deleted file mode 100644 index b391f169d..000000000 --- a/frontend/app/components/Session/EventsToggleButton/index.js +++ /dev/null @@ -1 +0,0 @@ -export { default } from './EventsToggleButton' \ No newline at end of file diff --git a/frontend/app/components/Session/LiveSession.js b/frontend/app/components/Session/LiveSession.js index 673e0c701..8ab1adf03 100644 --- a/frontend/app/components/Session/LiveSession.js +++ b/frontend/app/components/Session/LiveSession.js @@ -4,17 +4,17 @@ import { connect } from 'react-redux'; import usePageTitle from 'App/hooks/usePageTitle'; import { fetch as fetchSession } from 'Duck/sessions'; import { fetchList as fetchSlackList } from 'Duck/integrations/slack'; -import { Link, NoContent, Loader } from 'UI'; -import { sessions as sessionsRoute } from 'App/routes'; +// import { Link, NoContent, Loader } from 'UI'; +// import { sessions as sessionsRoute } from 'App/routes'; import withPermissions from 'HOCs/withPermissions' import LivePlayer from './LivePlayer'; -const SESSIONS_ROUTE = sessionsRoute(); +// const SESSIONS_ROUTE = sessionsRoute(); function LiveSession({ sessionId, - loading, - hasErrors, + // loading, + // hasErrors, session, fetchSession, fetchSlackList, @@ -38,9 +38,9 @@ function LiveSession({ },[ sessionId, hasSessionsPath ]); return ( - + // - + // ); } @@ -50,8 +50,8 @@ export default withPermissions(['ASSIST_LIVE'], '', true)(connect((state, props) const hasSessiosPath = state.getIn([ 'sessions', 'sessionPath' ]).pathname.includes('/sessions'); return { sessionId, - loading: state.getIn([ 'sessions', 'loading' ]), - hasErrors: !!state.getIn([ 'sessions', 'errors' ]), + // loading: state.getIn([ 'sessions', 'loading' ]), + // hasErrors: !!state.getIn([ 'sessions', 'errors' ]), session: state.getIn([ 'sessions', 'current' ]), hasSessionsPath: hasSessiosPath && !isAssist, }; diff --git a/frontend/app/components/Session_/Player/Overlay/ElementsMarker/Marker.tsx b/frontend/app/components/Session_/Player/Overlay/ElementsMarker/Marker.tsx index f91f1d963..d5afa65a2 100644 --- a/frontend/app/components/Session_/Player/Overlay/ElementsMarker/Marker.tsx +++ b/frontend/app/components/Session_/Player/Overlay/ElementsMarker/Marker.tsx @@ -4,7 +4,7 @@ import type { MarkedTarget } from 'Player/MessageDistributor/StatedScreen/Stated import cn from 'classnames'; import stl from './Marker.module.css'; import { activeTarget } from 'Player'; -import { Popup } from 'UI'; +import { Tooltip } from 'react-tippy'; interface Props { target: MarkedTarget; @@ -21,17 +21,17 @@ export default function Marker({ target, active }: Props) { return (
activeTarget(target.index)}>
{target.index + 1}
- {target.count} Clicks
)} >
- +
) } \ No newline at end of file diff --git a/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx b/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx index 75a09eb80..4adcc8365 100644 --- a/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx +++ b/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx @@ -116,67 +116,67 @@ function LiveSessionList(props: Props) {
Sort By - 0} - > -
- i.value === filter.sort) || sortOptions[0]} + /> +
+ props.applyFilter({ order: state })} sortOrder={filter.order} /> +
- - - See how to setup the{' '} - - {'Assist'} - {' '} - plugin, if you haven’t done that already. - - } - image={} - show={!loading && list.size === 0} - > -
- {list.map((session) => ( - <> - -
- - ))} - -
- props.updateCurrentPage(page)} - limit={PER_PAGE} - /> +
+ + + See how to setup the{' '} + + {'Assist'} + {' '} + plugin, if you haven’t done that already. + + } + image={} + show={!loading && list.size === 0} + > +
+ {list.map((session) => ( + <> + +
+ + ))}
-
- - + + + +
+ props.updateCurrentPage(page)} + limit={PER_PAGE} + debounceRequest={500} + /> +
+
); } diff --git a/frontend/app/components/ui/Button/Button.tsx b/frontend/app/components/ui/Button/Button.tsx index ec57b2231..d324d4f4a 100644 --- a/frontend/app/components/ui/Button/Button.tsx +++ b/frontend/app/components/ui/Button/Button.tsx @@ -3,67 +3,74 @@ import cn from 'classnames'; import { CircularLoader, Icon } from 'UI'; interface Props { - className?: string; - children: React.ReactNode; - onClick?: () => void; - disabled?: boolean; - type?: 'button' | 'submit' | 'reset'; - loading?: boolean; - icon?: string; - [x: string]: any + className?: string; + children: React.ReactNode; + onClick?: () => void; + disabled?: boolean; + type?: 'button' | 'submit' | 'reset'; + loading?: boolean; + icon?: string; + [x: string]: any; } export default (props: Props) => { - const { - icon = '', - className = '', - variant = "default", - type = "button", - size = '', - disabled = false, - children, - loading = false, - ...rest - } = props; + const { + icon = '', + className = '', + variant = 'default', + type = 'button', + size = '', + disabled = false, + children, + loading = false, + ...rest + } = props; - const classes = ['relative flex items-center h-10 px-3 rounded tracking-wide whitespace-nowrap']; - if (variant === 'default') { - classes.push('bg-white hover:bg-gray-lightest border border-gray-light') - } + const classes = ['relative flex items-center h-10 px-3 rounded tracking-wide whitespace-nowrap']; + if (variant === 'default') { + classes.push('bg-white hover:bg-gray-lightest border border-gray-light'); + } - if (variant === 'primary') { - classes.push('bg-teal color-white hover:bg-teal-dark') - } + if (variant === 'primary') { + classes.push('bg-teal color-white hover:bg-teal-dark'); + } - if (variant === 'text') { - classes.push('bg-transparent color-gray-dark hover:bg-gray-lightest hover:color-gray-dark') - } + if (variant === 'text') { + classes.push('bg-transparent color-gray-dark hover:bg-gray-lightest hover:color-gray-dark'); + } - if (variant === 'text-primary') { - classes.push('bg-transparent color-teal hover:bg-teal-light hover:color-teal-dark') - } + if (variant === 'text-primary') { + classes.push('bg-transparent color-teal hover:bg-teal-light hover:color-teal-dark'); + } - if (variant === 'outline') { - classes.push('bg-white color-teal border border-teal hover:bg-teal-light') - } + if (variant === 'text-red') { + classes.push('bg-transparent color-red hover:bg-teal-light'); + } - if (disabled) { - classes.push('opacity-40 pointer-events-none') - } + if (variant === 'outline') { + classes.push('bg-white color-teal border border-teal hover:bg-teal-light'); + } - const iconColor = variant === 'text' || variant === 'default' ? 'gray-dark' : 'teal'; - // console.log('children', children) + if (disabled) { + classes.push('opacity-40 pointer-events-none'); + } - return ( - - ); -} + let iconColor = variant === 'text' || variant === 'default' ? 'gray-dark' : 'teal'; + if (variant === 'primary') { + iconColor = 'white'; + } + if (variant === 'text-red') { + iconColor = 'red'; + } + + return ( + + ); +}; diff --git a/frontend/app/components/ui/Pagination/Pagination.tsx b/frontend/app/components/ui/Pagination/Pagination.tsx index ff1f8fde5..b36d2b397 100644 --- a/frontend/app/components/ui/Pagination/Pagination.tsx +++ b/frontend/app/components/ui/Pagination/Pagination.tsx @@ -33,9 +33,10 @@ export default function Pagination(props: Props) { return (
} > - +
))} @@ -254,7 +254,7 @@ export default class Timeline extends React.PureComponent {
} > - +
} @@ -277,7 +277,7 @@ export default class Timeline extends React.PureComponent {
} > - +
)) @@ -330,7 +330,7 @@ export default class Timeline extends React.PureComponent { } > - + )) From 269863ea13e03c9e79fc8c559c960dfd8cc52522 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 13 Jul 2022 12:45:00 +0200 Subject: [PATCH 015/553] change(ui) - assist sort by timestamp --- .../LiveSessionList/LiveSessionList.tsx | 61 +++++-------------- .../SortOrderButton/SortOrderButton.tsx | 4 +- frontend/app/duck/liveSearch.js | 2 +- 3 files changed, 17 insertions(+), 50 deletions(-) diff --git a/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx b/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx index 4adcc8365..fd4af677c 100644 --- a/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx +++ b/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx @@ -1,6 +1,6 @@ import React, { useEffect } from 'react'; import { connect } from 'react-redux'; -import { NoContent, Loader, Pagination, Popup } from 'UI'; +import { NoContent, Loader, Pagination } from 'UI'; import { List } from 'immutable'; import SessionItem from 'Shared/SessionItem'; import withPermissions from 'HOCs/withPermissions'; @@ -39,41 +39,11 @@ function LiveSessionList(props: Props) { var timeoutId: any; const { filters } = filter; const hasUserFilter = filters.map((i: any) => i.key).includes(KEYS.USERID); - const sortOptions = metaList + const sortOptions = [{ label: 'Newest', value: 'timestamp' }].concat(metaList .map((i: any) => ({ label: capitalize(i), value: i, - })) - .toJS(); - - // useEffect(() => { - // if (metaListLoading || metaList.size === 0 || !!filter.sort) return; - - // if (sortOptions[0]) { - // props.applyFilter({ sort: sortOptions[0].value }); - // } - // }, [metaListLoading]); - - // useEffect(() => { - // const filteredSessions = filters.size > 0 ? props.list.filter(session => { - // let hasValidFilter = true; - // filters.forEach(filter => { - // if (!hasValidFilter) return; - - // const _values = filter.value.filter(i => i !== '' && i !== null && i !== undefined).map(i => i.toLowerCase()); - // if (filter.key === FilterKey.USERID) { - // const _userId = session.userId ? session.userId.toLowerCase() : ''; - // hasValidFilter = _values.length > 0 ? (_values.includes(_userId) && hasValidFilter) || _values.some(i => _userId.includes(i)) : hasValidFilter; - // } - // if (filter.category === FilterCategory.METADATA) { - // const _source = session.metadata[filter.key] ? session.metadata[filter.key].toLowerCase() : ''; - // hasValidFilter = _values.length > 0 ? (_values.includes(_source) && hasValidFilter) || _values.some(i => _source.includes(i)) : hasValidFilter; - // } - // }) - // return hasValidFilter; - // }) : props.list; - // setSessions(filteredSessions); - // }, [filters, list]); + })).toJS()); useEffect(() => { props.applyFilter({ ...filter }); @@ -116,20 +86,17 @@ function LiveSessionList(props: Props) {
Sort By - 0}> -
- i.value === filter.sort) || sortOptions[0]} + /> +
+ props.applyFilter({ order: state })} sortOrder={filter.order} /> +
diff --git a/frontend/app/components/shared/SortOrderButton/SortOrderButton.tsx b/frontend/app/components/shared/SortOrderButton/SortOrderButton.tsx index 1a10d8030..7d7901783 100644 --- a/frontend/app/components/shared/SortOrderButton/SortOrderButton.tsx +++ b/frontend/app/components/shared/SortOrderButton/SortOrderButton.tsx @@ -14,7 +14,7 @@ export default React.memo(function SortOrderButton(props: Props) {
onChange('asc')} > @@ -23,7 +23,7 @@ export default React.memo(function SortOrderButton(props: Props) {
onChange('desc')} > diff --git a/frontend/app/duck/liveSearch.js b/frontend/app/duck/liveSearch.js index d75916eec..9df03dc81 100644 --- a/frontend/app/duck/liveSearch.js +++ b/frontend/app/duck/liveSearch.js @@ -20,7 +20,7 @@ const FETCH_SESSION_LIST = fetchListType(`${name}/FETCH_SESSION_LIST`); const initialState = Map({ list: List(), - instance: new Filter({ filters: [], sort: '' }), + instance: new Filter({ filters: [], sort: 'timestamp' }), filterSearchList: {}, currentPage: 1, }); From 776ff4f30b61dacc68427b11d40378cfaa72a053 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 13 Jul 2022 17:20:04 +0200 Subject: [PATCH 016/553] fix(ui) - filters input value (#604) * fix(ui) - search filters input field * fix(ui) - search filters input field --- .../FilterAutoCompleteLocal/FilterAutoCompleteLocal.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/shared/Filters/FilterAutoCompleteLocal/FilterAutoCompleteLocal.tsx b/frontend/app/components/shared/Filters/FilterAutoCompleteLocal/FilterAutoCompleteLocal.tsx index 38dabe535..ad73be681 100644 --- a/frontend/app/components/shared/Filters/FilterAutoCompleteLocal/FilterAutoCompleteLocal.tsx +++ b/frontend/app/components/shared/Filters/FilterAutoCompleteLocal/FilterAutoCompleteLocal.tsx @@ -35,12 +35,12 @@ function FilterAutoCompleteLocal(props: Props) { if(allowDecimals) { const value = e.target.value; setQuery(value); - props.onSelect(null, { value }); + props.onSelect(null, value); } else { const value = e.target.value.replace(/[^\d]/, ""); if (+value !== 0) { setQuery(value); - props.onSelect(null, { value }); + props.onSelect(null, value); } } }; From 37e128e767072c2e71f69ccea7f21e8c7265ca83 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 13 Jul 2022 19:28:27 +0200 Subject: [PATCH 017/553] fix(ui) - session clean error state --- frontend/app/components/Session/Session.js | 2 +- frontend/app/player/MessageDistributor/MessageDistributor.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/app/components/Session/Session.js b/frontend/app/components/Session/Session.js index c86d7d3e4..2d9bfa882 100644 --- a/frontend/app/components/Session/Session.js +++ b/frontend/app/components/Session/Session.js @@ -38,7 +38,7 @@ function Session({ subtext={ {'Please check your data retention plan, or try '} - {'another one'} + {'another one'} } > diff --git a/frontend/app/player/MessageDistributor/MessageDistributor.ts b/frontend/app/player/MessageDistributor/MessageDistributor.ts index 936f7409f..d9f2aac2b 100644 --- a/frontend/app/player/MessageDistributor/MessageDistributor.ts +++ b/frontend/app/player/MessageDistributor/MessageDistributor.ts @@ -45,6 +45,7 @@ export interface State extends SuperState, AssistState { domContentLoadedTime?: any, domBuildingTime?: any, loadTime?: any, + error: boolean, } export const INITIAL_STATE: State = { ...SUPER_INITIAL_STATE, @@ -52,6 +53,7 @@ export const INITIAL_STATE: State = { ...ASSIST_INITIAL_STATE, performanceChartData: [], skipIntervals: [], + error: false }; From 3c9a96e7ab1f685785fcc1063dd91b3f464f5934 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 13 Jul 2022 19:32:01 +0200 Subject: [PATCH 018/553] fix(ui) - meta list more popup color --- frontend/app/components/ui/Popup/Popup.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/app/components/ui/Popup/Popup.tsx b/frontend/app/components/ui/Popup/Popup.tsx index c8928fa07..129b9a8a4 100644 --- a/frontend/app/components/ui/Popup/Popup.tsx +++ b/frontend/app/components/ui/Popup/Popup.tsx @@ -21,6 +21,7 @@ export default ({ delay = 1000, disabled = false, arrow = true, + theme = 'dark', ...props }: Props) => ( { props.children } From 625176df97ecaddb38a5608d6d60e7fd8a42bedd Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 13 Jul 2022 20:19:06 +0200 Subject: [PATCH 019/553] change(ui) - assist sort by first meta --- frontend/app/components/Header/Header.js | 2 +- frontend/app/components/Header/SiteDropdown.js | 5 ++++- .../shared/LiveSessionList/LiveSessionList.tsx | 9 +++++++-- frontend/app/duck/liveSearch.js | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/frontend/app/components/Header/Header.js b/frontend/app/components/Header/Header.js index fc1aab477..9e6efa0de 100644 --- a/frontend/app/components/Header/Header.js +++ b/frontend/app/components/Header/Header.js @@ -51,6 +51,7 @@ const Header = (props) => { Promise.all([ userStore.fetchLimits(), notificationStore.fetchNotificationsCount(), + props.fetchMetadata(), ]).then(() => { userStore.updateKey('initialDataFetched', true); }); @@ -60,7 +61,6 @@ const Header = (props) => { useEffect(() => { activeSite = sites.find(s => s.id == siteId); props.initSite(activeSite); - props.fetchMetadata(); }, [siteId]) return ( diff --git a/frontend/app/components/Header/SiteDropdown.js b/frontend/app/components/Header/SiteDropdown.js index 9b3af1f8f..228190111 100644 --- a/frontend/app/components/Header/SiteDropdown.js +++ b/frontend/app/components/Header/SiteDropdown.js @@ -11,6 +11,7 @@ import styles from './siteDropdown.module.css'; import cn from 'classnames'; import NewSiteForm from '../Client/Sites/NewSiteForm'; import { clearSearch } from 'Duck/search'; +import { clearSearch as clearSearchLive } from 'Duck/liveSearch'; import { fetchList as fetchIntegrationVariables } from 'Duck/customField'; import { withStore } from 'App/mstore' import AnimatedSVG, { ICONS } from '../shared/AnimatedSVG/AnimatedSVG'; @@ -27,6 +28,7 @@ import NewProjectButton from './NewProjectButton'; pushNewSite, init, clearSearch, + clearSearchLive, fetchIntegrationVariables, }) export default class SiteDropdown extends React.PureComponent { @@ -45,8 +47,9 @@ export default class SiteDropdown extends React.PureComponent { const { mstore, location } = this.props this.props.setSiteId(siteId); - this.props.clearSearch(location.pathname.includes('/sessions')); this.props.fetchIntegrationVariables(); + this.props.clearSearch(location.pathname.includes('/sessions')); + this.props.clearSearchLive(); mstore.initClient(); } diff --git a/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx b/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx index fd4af677c..f4bb1f45d 100644 --- a/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx +++ b/frontend/app/components/shared/LiveSessionList/LiveSessionList.tsx @@ -46,12 +46,17 @@ function LiveSessionList(props: Props) { })).toJS()); useEffect(() => { - props.applyFilter({ ...filter }); + if (metaListLoading) return; + const _filter = { ...filter }; + if (sortOptions[1]) { + _filter.sort = sortOptions[1].value; + } + props.applyFilter(_filter); timeout(); return () => { clearTimeout(timeoutId); }; - }, []); + }, [metaListLoading]); const onUserClick = (userId: string, userAnonymousId: string) => { if (userId) { diff --git a/frontend/app/duck/liveSearch.js b/frontend/app/duck/liveSearch.js index 9df03dc81..ecb8a720e 100644 --- a/frontend/app/duck/liveSearch.js +++ b/frontend/app/duck/liveSearch.js @@ -96,7 +96,7 @@ export const fetchSessions = (filter) => (dispatch, getState) => { }; export const clearSearch = () => (dispatch, getState) => { - dispatch(edit(new Filter({ filters: [] }))); + dispatch(edit(new Filter({ filters: [], sort: 'timestamp' }))); return dispatch({ type: CLEAR_SEARCH, }); From cab904e0e2d1adec1691fb6ca227f4e76e1954be Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Wed, 13 Jul 2022 22:40:29 +0200 Subject: [PATCH 020/553] v1.7.0 enhanced (#606) * feat(assist): changed Dockerfile * feat(assist): changed Dockerfile * feat(assist): changed Dockerfile * feat(assist): changed lock file * feat(assist): changed Dockerfile * feat(chalice): return role name after update user * feat(chalice): changed sessions search * feat(chalice): changed sessions search * feat(chalice): changed Dockerfile feat(chalice): changed entrypoint feat(alerts): changed Dockerfile feat(alerts): changed entrypoint * feat(assist): handle null uws payload * feat(crons): fixed coroutine * feat(chalice): optimize get projects --- api/Dockerfile | 18 +- api/Dockerfile.alerts | 18 +- api/chalicelib/core/sessions.py | 12 +- api/entrypoint.sh | 2 +- api/entrypoint_alerts.sh | 2 +- ee/api/app_crons.py | 17 +- ee/api/chalicelib/core/projects.py | 12 +- ee/api/chalicelib/core/users.py | 12 +- .../db/init_dbs/postgresql/1.7.0/1.7.0.sql | 1 + .../db/init_dbs/postgresql/init_schema.sql | 1 + ee/utilities/.gitignore | 2 +- ee/utilities/Dockerfile | 15 + ee/utilities/clean.sh | 1 + ee/utilities/package-lock.json | 773 ++++++++++-------- ee/utilities/utils/helper-ee.js | 5 +- 15 files changed, 518 insertions(+), 373 deletions(-) create mode 100644 ee/utilities/Dockerfile diff --git a/api/Dockerfile b/api/Dockerfile index ae149b034..8b2acf1f7 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,23 +1,15 @@ -FROM python:3.10-slim +FROM python:3.10-alpine LABEL Maintainer="Rajesh Rajendran" LABEL Maintainer="KRAIEM Taha Yassine" +RUN apk add --no-cache nodejs npm tini ARG envarg # Add Tini # Startup daemon -ENV TINI_VERSION=v0.19.0 \ - SOURCE_MAP_VERSION=0.7.4 \ +ENV SOURCE_MAP_VERSION=0.7.4 \ APP_NAME=chalice \ ENTERPRISE_BUILD=${envarg} -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini -ADD https://unpkg.com/source-map@${SOURCE_MAP_VERSION}/lib/mappings.wasm /mappings.wasm -RUN chmod +x /tini -# Installing Nodejs -RUN apt update && apt install -y curl && \ - curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \ - apt install -y nodejs && \ - apt remove --purge -y curl && \ - rm -rf /var/lib/apt/lists/* +ADD https://unpkg.com/source-map@${SOURCE_MAP_VERSION}/lib/mappings.wasm /mappings.wasm WORKDIR /work_tmp COPY requirements.txt /work_tmp/requirements.txt @@ -29,5 +21,5 @@ WORKDIR /work COPY . . RUN mv env.default .env && mv /work_tmp/node_modules sourcemap-reader/. -ENTRYPOINT ["/tini", "--"] +ENTRYPOINT ["/sbin/tini", "--"] CMD ./entrypoint.sh diff --git a/api/Dockerfile.alerts b/api/Dockerfile.alerts index bb8f11285..70879cae8 100644 --- a/api/Dockerfile.alerts +++ b/api/Dockerfile.alerts @@ -1,16 +1,12 @@ -FROM python:3.10-slim +FROM python:3.10-alpine LABEL Maintainer="Rajesh Rajendran" LABEL Maintainer="KRAIEM Taha Yassine" -ENV APP_NAME alerts -ENV pg_minconn 2 -ENV pg_maxconn 10 -# Add Tini -# Startup daemon -ENV TINI_VERSION v0.19.0 +RUN apk add --no-cache tini ARG envarg -ENV ENTERPRISE_BUILD ${envarg} -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini -RUN chmod +x /tini +ENV APP_NAME=alerts \ + pg_minconn=2 \ + pg_maxconn=10 \ + ENTERPRISE_BUILD=${envarg} COPY requirements.txt /work_tmp/requirements.txt RUN pip install --no-cache-dir --upgrade -r /work_tmp/requirements.txt @@ -19,5 +15,5 @@ WORKDIR /work COPY . . RUN mv env.default .env && mv app_alerts.py app.py && mv entrypoint_alerts.sh entrypoint.sh -ENTRYPOINT ["/tini", "--"] +ENTRYPOINT ["/sbin/tini", "--"] CMD ./entrypoint.sh \ No newline at end of file diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 3856acaa0..8dc87c90d 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -863,12 +863,12 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} if f.type == schemas.FetchFilterType._url: event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s", f.value, - value_key=e_k_f)) + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s::text", + f.value, value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._status_code: event_where.append( - _multiple_conditions(f"main.status_code {f.operator} %({e_k_f})s", f.value, + _multiple_conditions(f"main.status_code {f.operator} %({e_k_f})s::integer", f.value, value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._method: @@ -877,15 +877,15 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr apply = True elif f.type == schemas.FetchFilterType._duration: event_where.append( - _multiple_conditions(f"main.duration {f.operator} %({e_k_f})s", f.value, value_key=e_k_f)) + _multiple_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value, value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._request_body: event_where.append( - _multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + _multiple_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value, value_key=e_k_f)) apply = True elif f.type == schemas.FetchFilterType._response_body: event_where.append( - _multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + _multiple_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value, value_key=e_k_f)) apply = True else: print(f"undefined FETCH filter: {f.type}") diff --git a/api/entrypoint.sh b/api/entrypoint.sh index a41427181..94b56121a 100755 --- a/api/entrypoint.sh +++ b/api/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh cd sourcemap-reader nohup npm start &> /tmp/sourcemap-reader.log & cd .. diff --git a/api/entrypoint_alerts.sh b/api/entrypoint_alerts.sh index 5f15a78b7..861206589 100755 --- a/api/entrypoint_alerts.sh +++ b/api/entrypoint_alerts.sh @@ -1,3 +1,3 @@ -#!/bin/bash +#!/bin/sh uvicorn app:app --host 0.0.0.0 --reload diff --git a/ee/api/app_crons.py b/ee/api/app_crons.py index 3dfea4fe4..97f4b5406 100644 --- a/ee/api/app_crons.py +++ b/ee/api/app_crons.py @@ -1,19 +1,28 @@ print("============= CRONS =============") import sys +import asyncio from routers.crons import core_dynamic_crons -def process(action): - { +def default_action(action): + async def _func(): + print(f"{action} not found in crons-definitions") + + return _func + + +async def process(action): + await { "TELEMETRY": core_dynamic_crons.telemetry_cron, "JOB": core_dynamic_crons.run_scheduled_jobs, "REPORT": core_dynamic_crons.weekly_report2 - }.get(action.upper(), lambda: print(f"{action} not found in crons-definitions"))() + }.get(action.upper(), default_action(action))() if __name__ == '__main__': if len(sys.argv) < 2 or len(sys.argv[1]) < 1: print("please provide actions as argument") else: - process(sys.argv[1]) + print(f"action: {sys.argv[1]}") + asyncio.run(process(sys.argv[1])) diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index 6a06e8230..e6ef34760 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -52,14 +52,23 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st AND users.tenant_id = %(tenant_id)s AND (roles.all_projects OR roles_projects.project_id = s.project_id) ) AS role_project ON (TRUE)""" + pre_select = "" + if recorded: + pre_select = """WITH recorded_p AS (SELECT DISTINCT projects.project_id + FROM projects INNER JOIN sessions USING (project_id) + WHERE tenant_id =%(tenant_id)s + AND deleted_at IS NULL + AND duration > 0)""" cur.execute( cur.mogrify(f"""\ + {pre_select} SELECT s.project_id, s.name, s.project_key, s.save_request_payloads {',s.gdpr' if gdpr else ''} - {',COALESCE((SELECT TRUE FROM public.sessions WHERE sessions.project_id = s.project_id LIMIT 1), FALSE) AS recorded' if recorded else ''} + {',EXISTS(SELECT 1 FROM recorded_p WHERE recorded_p.project_id = s.project_id) AS recorded' if recorded else ''} {',stack_integrations.count>0 AS stack_integrations' if stack_integrations else ''} FROM public.projects AS s + {'LEFT JOIN recorded_p USING (project_id)' if recorded else ''} {'LEFT JOIN LATERAL (SELECT COUNT(*) AS count FROM public.integrations WHERE s.project_id = integrations.project_id LIMIT 1) AS stack_integrations ON TRUE' if stack_integrations else ''} {role_query if user_id is not None else ""} WHERE s.tenant_id =%(tenant_id)s @@ -76,7 +85,6 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st WHERE sessions.start_ts >= %(startDate)s AND sessions.start_ts <= %(endDate)s GROUP BY project_id;""", {"startDate": TimeUTC.now(delta_days=-3), "endDate": TimeUTC.now(delta_days=1)}) - cur.execute(query=query) status = cur.fetchall() for r in rows: diff --git a/ee/api/chalicelib/core/users.py b/ee/api/chalicelib/core/users.py index 7b5162b27..815d39106 100644 --- a/ee/api/chalicelib/core/users.py +++ b/ee/api/chalicelib/core/users.py @@ -168,7 +168,11 @@ def update(tenant_id, user_id, changes): (CASE WHEN users.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, (CASE WHEN users.role = 'admin' THEN TRUE ELSE FALSE END) AS admin, (CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member, - users.role_id;""", + users.role_id, + (SELECT roles.name + FROM roles + WHERE roles.tenant_id=%(tenant_id)s + AND roles.role_id=users.role_id) AS role_name;""", {"tenant_id": tenant_id, "user_id": user_id, **changes}) ) if len(sub_query_bauth) > 0: @@ -187,7 +191,11 @@ def update(tenant_id, user_id, changes): (CASE WHEN users.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, (CASE WHEN users.role = 'admin' THEN TRUE ELSE FALSE END) AS admin, (CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member, - users.role_id;""", + users.role_id, + (SELECT roles.name + FROM roles + WHERE roles.tenant_id=%(tenant_id)s + AND roles.role_id=users.role_id) AS role_name;""", {"tenant_id": tenant_id, "user_id": user_id, **changes}) ) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.7.0/1.7.0.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.7.0/1.7.0.sql index 3f74ad21d..1ea5c6ab6 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.7.0/1.7.0.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.7.0/1.7.0.sql @@ -56,6 +56,7 @@ ALTER TABLE IF EXISTS events.resources PRIMARY KEY (session_id, message_id, timestamp); COMMIT; +CREATE INDEX CONCURRENTLY IF NOT EXISTS projects_tenant_id_idx ON public.projects (tenant_id); CREATE INDEX CONCURRENTLY IF NOT EXISTS projects_project_id_deleted_at_n_idx ON public.projects (project_id) WHERE deleted_at IS NULL; ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'funnel'; diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 7cac6e10e..2d1c2b95f 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -257,6 +257,7 @@ $$ ); + CREATE INDEX IF NOT EXISTS projects_tenant_id_idx ON public.projects (tenant_id); CREATE INDEX IF NOT EXISTS projects_project_key_idx ON public.projects (project_key); CREATE INDEX IF NOT EXISTS projects_project_id_deleted_at_n_idx ON public.projects (project_id) WHERE deleted_at IS NULL; DROP TRIGGER IF EXISTS on_insert_or_update ON projects; diff --git a/ee/utilities/.gitignore b/ee/utilities/.gitignore index f54e439ba..0eaed6d80 100644 --- a/ee/utilities/.gitignore +++ b/ee/utilities/.gitignore @@ -10,7 +10,7 @@ build.sh servers/peerjs-server.js servers/sourcemaps-handler.js servers/sourcemaps-server.js -/Dockerfile /utils/geoIP.js /utils/HeapSnapshot.js /utils/helper.js +/utils/assistHelper.js diff --git a/ee/utilities/Dockerfile b/ee/utilities/Dockerfile new file mode 100644 index 000000000..f3e605ccc --- /dev/null +++ b/ee/utilities/Dockerfile @@ -0,0 +1,15 @@ +FROM node:18-alpine +LABEL Maintainer="KRAIEM Taha Yassine" +RUN apk add --no-cache tini git libc6-compat && ln -s /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2 +ARG envarg +ENV ENTERPRISE_BUILD=${envarg} \ + MAXMINDDB_FILE=/root/geoip.mmdb + +WORKDIR /work +ADD https://static.openreplay.com/geoip/GeoLite2-Country.mmdb $MAXMINDDB_FILE +COPY package.json . +COPY package-lock.json . +RUN npm install +COPY . . +ENTRYPOINT ["/sbin/tini", "--"] +CMD npm start \ No newline at end of file diff --git a/ee/utilities/clean.sh b/ee/utilities/clean.sh index 3e8ec080b..ec1aaeae4 100755 --- a/ee/utilities/clean.sh +++ b/ee/utilities/clean.sh @@ -1,6 +1,7 @@ rm -rf ./utils/geoIP.js rm -rf ./utils/HeapSnapshot.js rm -rf ./utils/helper.js +rm -rf ./utils/assistHelper.js rm -rf servers/peerjs-server.js rm -rf servers/sourcemaps-handler.js diff --git a/ee/utilities/package-lock.json b/ee/utilities/package-lock.json index 19699560a..8aa6c9196 100644 --- a/ee/utilities/package-lock.json +++ b/ee/utilities/package-lock.json @@ -29,68 +29,67 @@ "maxmind": "^4.2.0" } }, - "node_modules/@node-redis/bloom": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@node-redis/bloom/-/bloom-1.0.1.tgz", - "integrity": "sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw==", + "node_modules/@redis/bloom": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.0.2.tgz", + "integrity": "sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw==", "peerDependencies": { - "@node-redis/client": "^1.0.0" + "@redis/client": "^1.0.0" } }, - "node_modules/@node-redis/client": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@node-redis/client/-/client-1.0.5.tgz", - "integrity": "sha512-ESZ3bd1f+od62h4MaBLKum+klVJfA4wAeLHcVQBkoXa1l0viFesOWnakLQqKg+UyrlJhZmXJWtu0Y9v7iTMrig==", + "node_modules/@redis/client": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.2.0.tgz", + "integrity": "sha512-a8Nlw5fv2EIAFJxTDSSDVUT7yfBGpZO96ybZXzQpgkyLg/dxtQ1uiwTc0EGfzg1mrPjZokeBSEGTbGXekqTNOg==", "dependencies": { "cluster-key-slot": "1.1.0", "generic-pool": "3.8.2", - "redis-parser": "3.0.0", "yallist": "4.0.0" }, "engines": { - "node": ">=12" + "node": ">=14" } }, - "node_modules/@node-redis/graph": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@node-redis/graph/-/graph-1.0.0.tgz", - "integrity": "sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g==", + "node_modules/@redis/graph": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.0.1.tgz", + "integrity": "sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ==", "peerDependencies": { - "@node-redis/client": "^1.0.0" + "@redis/client": "^1.0.0" } }, - "node_modules/@node-redis/json": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@node-redis/json/-/json-1.0.2.tgz", - "integrity": "sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g==", + "node_modules/@redis/json": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.3.tgz", + "integrity": "sha512-4X0Qv0BzD9Zlb0edkUoau5c1bInWSICqXAGrpwEltkncUwcxJIGEcVryZhLgb0p/3PkKaLIWkjhHRtLe9yiA7Q==", "peerDependencies": { - "@node-redis/client": "^1.0.0" + "@redis/client": "^1.0.0" } }, - "node_modules/@node-redis/search": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@node-redis/search/-/search-1.0.5.tgz", - "integrity": "sha512-MCOL8iCKq4v+3HgEQv8zGlSkZyXSXtERgrAJ4TSryIG/eLFy84b57KmNNa/V7M1Q2Wd2hgn2nPCGNcQtk1R1OQ==", + "node_modules/@redis/search": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.0.6.tgz", + "integrity": "sha512-pP+ZQRis5P21SD6fjyCeLcQdps+LuTzp2wdUbzxEmNhleighDDTD5ck8+cYof+WLec4csZX7ks+BuoMw0RaZrA==", "peerDependencies": { - "@node-redis/client": "^1.0.0" + "@redis/client": "^1.0.0" } }, - "node_modules/@node-redis/time-series": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@node-redis/time-series/-/time-series-1.0.2.tgz", - "integrity": "sha512-HGQ8YooJ8Mx7l28tD7XjtB3ImLEjlUxG1wC1PAjxu6hPJqjPshUZxAICzDqDjtIbhDTf48WXXUcx8TQJB1XTKA==", + "node_modules/@redis/time-series": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.3.tgz", + "integrity": "sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA==", "peerDependencies": { - "@node-redis/client": "^1.0.0" + "@redis/client": "^1.0.0" } }, "node_modules/@socket.io/redis-adapter": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/@socket.io/redis-adapter/-/redis-adapter-7.1.0.tgz", - "integrity": "sha512-vbsNJKUQgtVHcOqNL2ac8kSemTVNKHRzYPldqQJt0eFKvlAtAviuAMzBP0WmOp5OoRLQMjhVsVvgMzzMsVsK5g==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@socket.io/redis-adapter/-/redis-adapter-7.2.0.tgz", + "integrity": "sha512-/r6oF6Myz0K9uatB/pfCi0BhKg/KRMh1OokrqcjlNz6aq40WiXdFLRbHJQuwGHq/KvB+D6141K+IynbVxZGvhw==", "dependencies": { "debug": "~4.3.1", "notepack.io": "~2.2.0", - "socket.io-adapter": "~2.3.0", + "socket.io-adapter": "^2.4.0", "uid2": "0.0.3" }, "engines": { @@ -113,9 +112,9 @@ "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" }, "node_modules/@types/node": { - "version": "17.0.42", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.42.tgz", - "integrity": "sha512-Q5BPGyGKcvQgAMbsr7qEGN/kIPN6zZecYYABeTDBizOsau+2NMdSVTar9UQw21A2+JyA2KRNDYaYrPB0Rpk2oQ==" + "version": "18.0.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.0.3.tgz", + "integrity": "sha512-HzNRZtp4eepNitP+BD6k2L6DROIDG4Q0fm4x+dwfsr6LGmROENnok75VGw40628xf+iR24WeMFcHuuBDUAzzsQ==" }, "node_modules/accepts": { "version": "1.3.8", @@ -132,12 +131,12 @@ "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, "node_modules/assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==", "engines": { "node": ">=0.8" } @@ -151,23 +150,26 @@ } }, "node_modules/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", + "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==", "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.4", "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "1.8.1", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", - "on-finished": "~2.3.0", - "qs": "6.9.7", - "raw-body": "2.4.3", - "type-is": "~1.6.18" + "on-finished": "2.4.1", + "qs": "6.10.3", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" } }, "node_modules/body-parser/node_modules/debug": { @@ -181,7 +183,7 @@ "node_modules/body-parser/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "node_modules/bytes": { "version": "3.1.2", @@ -191,6 +193,18 @@ "node": ">= 0.8" } }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/camelcase": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", @@ -252,9 +266,9 @@ } }, "node_modules/cookie": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", - "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", "engines": { "node": ">= 0.6" } @@ -262,12 +276,12 @@ "node_modules/cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, "node_modules/core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" }, "node_modules/cors": { "version": "2.8.5", @@ -298,27 +312,31 @@ } }, "node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", "engines": { "node": ">= 0.8" } @@ -351,51 +369,60 @@ "node": ">=10.0.0" } }, + "node_modules/engine.io/node_modules/cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", "engines": { "node": ">= 0.6" } }, "node_modules/express": { - "version": "4.17.3", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", - "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz", + "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.19.2", + "body-parser": "1.20.0", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.4.2", + "cookie": "0.5.0", "cookie-signature": "1.0.6", "debug": "2.6.9", - "depd": "~1.1.2", + "depd": "2.0.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "~1.1.2", + "finalhandler": "1.2.0", "fresh": "0.5.2", + "http-errors": "2.0.0", "merge-descriptors": "1.0.1", "methods": "~1.1.2", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", "proxy-addr": "~2.0.7", - "qs": "6.9.7", + "qs": "6.10.3", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", - "send": "0.17.2", - "serve-static": "1.14.2", + "send": "0.18.0", + "serve-static": "1.15.0", "setprototypeof": "1.2.0", - "statuses": "~1.5.0", + "statuses": "2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" @@ -415,27 +442,27 @@ "node_modules/express/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "node_modules/extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==", "engines": [ "node >=0.6.0" ] }, "node_modules/finalhandler": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", - "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", "dependencies": { "debug": "2.6.9", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", - "statuses": "~1.5.0", + "statuses": "2.0.1", "unpipe": "~1.0.0" }, "engines": { @@ -453,7 +480,7 @@ "node_modules/finalhandler/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "node_modules/forwarded": { "version": "0.2.0", @@ -466,11 +493,16 @@ "node_modules/fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", "engines": { "node": ">= 0.6" } }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, "node_modules/generic-pool": { "version": "3.8.2", "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", @@ -479,19 +511,54 @@ "node": ">= 4" } }, - "node_modules/http-errors": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", - "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "node_modules/get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", "dependencies": { - "depd": "~1.1.2", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dependencies": { + "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", - "statuses": ">= 1.5.0 < 2", + "statuses": "2.0.1", "toidentifier": "1.0.1" }, "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/iconv-lite": { @@ -549,7 +616,7 @@ "node_modules/lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", - "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=" + "integrity": "sha512-4hNPN5jlm/N/HLMCO43v8BXKq9Z7QdAGc/VGrRD61w8gN9g/6jF9A4L1pbUgBLCffi0w9VsXfTOij5x8iTyFvg==" }, "node_modules/map-obj": { "version": "4.3.0", @@ -578,7 +645,7 @@ "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", "engines": { "node": ">= 0.6" } @@ -586,12 +653,12 @@ "node_modules/merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, "node_modules/methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", "engines": { "node": ">= 0.6" } @@ -661,10 +728,18 @@ "node": ">=0.10.0" } }, + "node_modules/object-inspect": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "dependencies": { "ee-first": "1.1.1" }, @@ -683,7 +758,7 @@ "node_modules/path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "node_modules/proxy-addr": { "version": "2.0.7", @@ -698,9 +773,12 @@ } }, "node_modules/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==", + "version": "6.10.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", + "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "dependencies": { + "side-channel": "^1.0.4" + }, "engines": { "node": ">=0.6" }, @@ -728,12 +806,12 @@ } }, "node_modules/raw-body": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", - "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", "dependencies": { "bytes": "3.1.2", - "http-errors": "1.8.1", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", "unpipe": "1.0.0" }, @@ -742,35 +820,16 @@ } }, "node_modules/redis": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/redis/-/redis-4.0.6.tgz", - "integrity": "sha512-IaPAxgF5dV0jx+A9l6yd6R9/PAChZIoAskDVRzUODeLDNhsMlq7OLLTmu0AwAr0xjrJ1bibW5xdpRwqIQ8Q0Xg==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.2.0.tgz", + "integrity": "sha512-bCR0gKVhIXFg8zCQjXEANzgI01DDixtPZgIUZHBCmwqixnu+MK3Tb2yqGjh+HCLASQVVgApiwhNkv+FoedZOGQ==", "dependencies": { - "@node-redis/bloom": "1.0.1", - "@node-redis/client": "1.0.5", - "@node-redis/graph": "1.0.0", - "@node-redis/json": "1.0.2", - "@node-redis/search": "1.0.5", - "@node-redis/time-series": "1.0.2" - } - }, - "node_modules/redis-errors": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", - "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=", - "engines": { - "node": ">=4" - } - }, - "node_modules/redis-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", - "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", - "dependencies": { - "redis-errors": "^1.0.0" - }, - "engines": { - "node": ">=4" + "@redis/bloom": "1.0.2", + "@redis/client": "1.2.0", + "@redis/graph": "1.0.1", + "@redis/json": "1.0.3", + "@redis/search": "1.0.6", + "@redis/time-series": "1.0.3" } }, "node_modules/safe-buffer": { @@ -798,23 +857,23 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/send": { - "version": "0.17.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", - "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", "dependencies": { "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", + "depd": "2.0.0", + "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "1.8.1", + "http-errors": "2.0.0", "mime": "1.6.0", "ms": "2.1.3", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "range-parser": "~1.2.1", - "statuses": "~1.5.0" + "statuses": "2.0.1" }, "engines": { "node": ">= 0.8.0" @@ -831,7 +890,7 @@ "node_modules/send/node_modules/debug/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "node_modules/send/node_modules/ms": { "version": "2.1.3", @@ -839,14 +898,14 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/serve-static": { - "version": "1.14.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", - "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", "dependencies": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.17.2" + "send": "0.18.0" }, "engines": { "node": ">= 0.8.0" @@ -857,6 +916,19 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/socket.io": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.5.1.tgz", @@ -874,14 +946,14 @@ } }, "node_modules/socket.io-adapter": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.3.3.tgz", - "integrity": "sha512-Qd/iwn3VskrpNO60BeRyCyr8ZWw9CPZyitW4AQwmRZ8zCiyDiL+znRnWX6tDHXnWn1sJrM1+b6Mn6wEDJJ4aYQ==" + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.4.0.tgz", + "integrity": "sha512-W4N+o69rkMEGVuk2D/cvca3uYsvGlMwsySWV447y99gUPghxq42BxqLNMndb+a1mm/5/7NeXVQS7RLa2XyXvYg==" }, "node_modules/socket.io-parser": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.0.4.tgz", - "integrity": "sha512-t+b0SS+IxG7Rxzda2EVvyBZbvFPBCjJoyHuE0P//7OAsN23GItzDRdWa6ALxZI/8R5ygK7jAR6t028/z+7295g==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.0.5.tgz", + "integrity": "sha512-sNjbT9dX63nqUFIOv95tTVm6elyIU4RvB1m8dOeZt+IgWwcWklFDOdmGcfo3zSiRsnR/3pJkjY5lfoGqEe4Eig==", "dependencies": { "@types/component-emitter": "^1.2.10", "component-emitter": "~1.3.0", @@ -891,17 +963,12 @@ "node": ">=10.0.0" } }, - "node_modules/socket.io/node_modules/socket.io-adapter": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.4.0.tgz", - "integrity": "sha512-W4N+o69rkMEGVuk2D/cvca3uYsvGlMwsySWV447y99gUPghxq42BxqLNMndb+a1mm/5/7NeXVQS7RLa2XyXvYg==" - }, "node_modules/statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/tiny-lru": { @@ -964,12 +1031,12 @@ "node_modules/uid2": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/uid2/-/uid2-0.0.3.tgz", - "integrity": "sha1-SDEm4Rd03y9xuLY53NeZw3YWK4I=" + "integrity": "sha512-5gSP1liv10Gjp8cMEnFd6shzkL/D6W1uhXSFNCxDC+YI8+L8wkCYCbJ7n77Ezb4wE/xzMogecE+DtamEe9PZjg==" }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", "engines": { "node": ">= 0.8" } @@ -977,19 +1044,19 @@ "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", "engines": { "node": ">= 0.4.0" } }, "node_modules/uWebSockets.js": { - "version": "20.6.0", - "resolved": "git+ssh://git@github.com/uNetworking/uWebSockets.js.git#a58e810e47a23696410f6073c8c905dc38f75da5" + "version": "20.10.0", + "resolved": "git+ssh://git@github.com/uNetworking/uWebSockets.js.git#806df48c9da86af7b3341f3e443388c7cd15c3de" }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", "engines": { "node": ">= 0.8" } @@ -997,7 +1064,7 @@ "node_modules/verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", "engines": [ "node >=0.6.0" ], @@ -1045,55 +1112,54 @@ "maxmind": "^4.2.0" } }, - "@node-redis/bloom": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@node-redis/bloom/-/bloom-1.0.1.tgz", - "integrity": "sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw==", + "@redis/bloom": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.0.2.tgz", + "integrity": "sha512-EBw7Ag1hPgFzdznK2PBblc1kdlj5B5Cw3XwI9/oG7tSn85/HKy3X9xHy/8tm/eNXJYHLXHJL/pkwBpFMVVefkw==", "requires": {} }, - "@node-redis/client": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@node-redis/client/-/client-1.0.5.tgz", - "integrity": "sha512-ESZ3bd1f+od62h4MaBLKum+klVJfA4wAeLHcVQBkoXa1l0viFesOWnakLQqKg+UyrlJhZmXJWtu0Y9v7iTMrig==", + "@redis/client": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.2.0.tgz", + "integrity": "sha512-a8Nlw5fv2EIAFJxTDSSDVUT7yfBGpZO96ybZXzQpgkyLg/dxtQ1uiwTc0EGfzg1mrPjZokeBSEGTbGXekqTNOg==", "requires": { "cluster-key-slot": "1.1.0", "generic-pool": "3.8.2", - "redis-parser": "3.0.0", "yallist": "4.0.0" } }, - "@node-redis/graph": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@node-redis/graph/-/graph-1.0.0.tgz", - "integrity": "sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g==", + "@redis/graph": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.0.1.tgz", + "integrity": "sha512-oDE4myMCJOCVKYMygEMWuriBgqlS5FqdWerikMoJxzmmTUErnTRRgmIDa2VcgytACZMFqpAOWDzops4DOlnkfQ==", "requires": {} }, - "@node-redis/json": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@node-redis/json/-/json-1.0.2.tgz", - "integrity": "sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g==", + "@redis/json": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.3.tgz", + "integrity": "sha512-4X0Qv0BzD9Zlb0edkUoau5c1bInWSICqXAGrpwEltkncUwcxJIGEcVryZhLgb0p/3PkKaLIWkjhHRtLe9yiA7Q==", "requires": {} }, - "@node-redis/search": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@node-redis/search/-/search-1.0.5.tgz", - "integrity": "sha512-MCOL8iCKq4v+3HgEQv8zGlSkZyXSXtERgrAJ4TSryIG/eLFy84b57KmNNa/V7M1Q2Wd2hgn2nPCGNcQtk1R1OQ==", + "@redis/search": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.0.6.tgz", + "integrity": "sha512-pP+ZQRis5P21SD6fjyCeLcQdps+LuTzp2wdUbzxEmNhleighDDTD5ck8+cYof+WLec4csZX7ks+BuoMw0RaZrA==", "requires": {} }, - "@node-redis/time-series": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@node-redis/time-series/-/time-series-1.0.2.tgz", - "integrity": "sha512-HGQ8YooJ8Mx7l28tD7XjtB3ImLEjlUxG1wC1PAjxu6hPJqjPshUZxAICzDqDjtIbhDTf48WXXUcx8TQJB1XTKA==", + "@redis/time-series": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.3.tgz", + "integrity": "sha512-OFp0q4SGrTH0Mruf6oFsHGea58u8vS/iI5+NpYdicaM+7BgqBZH8FFvNZ8rYYLrUO/QRqMq72NpXmxLVNcdmjA==", "requires": {} }, "@socket.io/redis-adapter": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/@socket.io/redis-adapter/-/redis-adapter-7.1.0.tgz", - "integrity": "sha512-vbsNJKUQgtVHcOqNL2ac8kSemTVNKHRzYPldqQJt0eFKvlAtAviuAMzBP0WmOp5OoRLQMjhVsVvgMzzMsVsK5g==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@socket.io/redis-adapter/-/redis-adapter-7.2.0.tgz", + "integrity": "sha512-/r6oF6Myz0K9uatB/pfCi0BhKg/KRMh1OokrqcjlNz6aq40WiXdFLRbHJQuwGHq/KvB+D6141K+IynbVxZGvhw==", "requires": { "debug": "~4.3.1", "notepack.io": "~2.2.0", - "socket.io-adapter": "~2.3.0", + "socket.io-adapter": "^2.4.0", "uid2": "0.0.3" } }, @@ -1113,9 +1179,9 @@ "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" }, "@types/node": { - "version": "17.0.42", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.42.tgz", - "integrity": "sha512-Q5BPGyGKcvQgAMbsr7qEGN/kIPN6zZecYYABeTDBizOsau+2NMdSVTar9UQw21A2+JyA2KRNDYaYrPB0Rpk2oQ==" + "version": "18.0.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.0.3.tgz", + "integrity": "sha512-HzNRZtp4eepNitP+BD6k2L6DROIDG4Q0fm4x+dwfsr6LGmROENnok75VGw40628xf+iR24WeMFcHuuBDUAzzsQ==" }, "accepts": { "version": "1.3.8", @@ -1129,12 +1195,12 @@ "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==" }, "base64id": { "version": "2.0.0", @@ -1142,20 +1208,22 @@ "integrity": "sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==" }, "body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", + "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==", "requires": { "bytes": "3.1.2", "content-type": "~1.0.4", "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "1.8.1", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", - "on-finished": "~2.3.0", - "qs": "6.9.7", - "raw-body": "2.4.3", - "type-is": "~1.6.18" + "on-finished": "2.4.1", + "qs": "6.10.3", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" }, "dependencies": { "debug": { @@ -1169,7 +1237,7 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" } } }, @@ -1178,6 +1246,15 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "camelcase": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", @@ -1218,19 +1295,19 @@ "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" }, "cookie": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", - "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==" + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==" }, "cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" }, "cors": { "version": "2.8.5", @@ -1250,24 +1327,24 @@ } }, "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" }, "destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" }, "engine.io": { "version": "6.2.0", @@ -1284,6 +1361,13 @@ "debug": "~4.3.1", "engine.io-parser": "~5.0.3", "ws": "~8.2.3" + }, + "dependencies": { + "cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==" + } } }, "engine.io-parser": { @@ -1294,45 +1378,46 @@ "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, "express": { - "version": "4.17.3", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", - "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz", + "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==", "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.19.2", + "body-parser": "1.20.0", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.4.2", + "cookie": "0.5.0", "cookie-signature": "1.0.6", "debug": "2.6.9", - "depd": "~1.1.2", + "depd": "2.0.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "~1.1.2", + "finalhandler": "1.2.0", "fresh": "0.5.2", + "http-errors": "2.0.0", "merge-descriptors": "1.0.1", "methods": "~1.1.2", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", "proxy-addr": "~2.0.7", - "qs": "6.9.7", + "qs": "6.10.3", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", - "send": "0.17.2", - "serve-static": "1.14.2", + "send": "0.18.0", + "serve-static": "1.15.0", "setprototypeof": "1.2.0", - "statuses": "~1.5.0", + "statuses": "2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" @@ -1349,26 +1434,26 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" } } }, "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==" }, "finalhandler": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", - "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", "requires": { "debug": "2.6.9", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", - "statuses": "~1.5.0", + "statuses": "2.0.1", "unpipe": "~1.0.0" }, "dependencies": { @@ -1383,7 +1468,7 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" } } }, @@ -1395,22 +1480,50 @@ "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "generic-pool": { "version": "3.8.2", "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", "integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==" }, - "http-errors": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", - "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", "requires": { - "depd": "~1.1.2", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, + "http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "requires": { + "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", - "statuses": ">= 1.5.0 < 2", + "statuses": "2.0.1", "toidentifier": "1.0.1" } }, @@ -1460,7 +1573,7 @@ "lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", - "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=" + "integrity": "sha512-4hNPN5jlm/N/HLMCO43v8BXKq9Z7QdAGc/VGrRD61w8gN9g/6jF9A4L1pbUgBLCffi0w9VsXfTOij5x8iTyFvg==" }, "map-obj": { "version": "4.3.0", @@ -1479,17 +1592,17 @@ "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" }, "mime": { "version": "1.6.0", @@ -1534,10 +1647,15 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" }, + "object-inspect": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==" + }, "on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "requires": { "ee-first": "1.1.1" } @@ -1550,7 +1668,7 @@ "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "proxy-addr": { "version": "2.0.7", @@ -1562,9 +1680,12 @@ } }, "qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==" + "version": "6.10.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", + "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "requires": { + "side-channel": "^1.0.4" + } }, "quick-lru": { "version": "5.1.1", @@ -1577,40 +1698,27 @@ "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, "raw-body": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", - "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", "requires": { "bytes": "3.1.2", - "http-errors": "1.8.1", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", "unpipe": "1.0.0" } }, "redis": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/redis/-/redis-4.0.6.tgz", - "integrity": "sha512-IaPAxgF5dV0jx+A9l6yd6R9/PAChZIoAskDVRzUODeLDNhsMlq7OLLTmu0AwAr0xjrJ1bibW5xdpRwqIQ8Q0Xg==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.2.0.tgz", + "integrity": "sha512-bCR0gKVhIXFg8zCQjXEANzgI01DDixtPZgIUZHBCmwqixnu+MK3Tb2yqGjh+HCLASQVVgApiwhNkv+FoedZOGQ==", "requires": { - "@node-redis/bloom": "1.0.1", - "@node-redis/client": "1.0.5", - "@node-redis/graph": "1.0.0", - "@node-redis/json": "1.0.2", - "@node-redis/search": "1.0.5", - "@node-redis/time-series": "1.0.2" - } - }, - "redis-errors": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", - "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" - }, - "redis-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", - "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", - "requires": { - "redis-errors": "^1.0.0" + "@redis/bloom": "1.0.2", + "@redis/client": "1.2.0", + "@redis/graph": "1.0.1", + "@redis/json": "1.0.3", + "@redis/search": "1.0.6", + "@redis/time-series": "1.0.3" } }, "safe-buffer": { @@ -1624,23 +1732,23 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "send": { - "version": "0.17.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", - "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", "requires": { "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", + "depd": "2.0.0", + "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "1.8.1", + "http-errors": "2.0.0", "mime": "1.6.0", "ms": "2.1.3", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "range-parser": "~1.2.1", - "statuses": "~1.5.0" + "statuses": "2.0.1" }, "dependencies": { "debug": { @@ -1654,7 +1762,7 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" } } }, @@ -1666,14 +1774,14 @@ } }, "serve-static": { - "version": "1.14.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", - "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", "requires": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.17.2" + "send": "0.18.0" } }, "setprototypeof": { @@ -1681,6 +1789,16 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "socket.io": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.5.1.tgz", @@ -1692,24 +1810,17 @@ "engine.io": "~6.2.0", "socket.io-adapter": "~2.4.0", "socket.io-parser": "~4.0.4" - }, - "dependencies": { - "socket.io-adapter": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.4.0.tgz", - "integrity": "sha512-W4N+o69rkMEGVuk2D/cvca3uYsvGlMwsySWV447y99gUPghxq42BxqLNMndb+a1mm/5/7NeXVQS7RLa2XyXvYg==" - } } }, "socket.io-adapter": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.3.3.tgz", - "integrity": "sha512-Qd/iwn3VskrpNO60BeRyCyr8ZWw9CPZyitW4AQwmRZ8zCiyDiL+znRnWX6tDHXnWn1sJrM1+b6Mn6wEDJJ4aYQ==" + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.4.0.tgz", + "integrity": "sha512-W4N+o69rkMEGVuk2D/cvca3uYsvGlMwsySWV447y99gUPghxq42BxqLNMndb+a1mm/5/7NeXVQS7RLa2XyXvYg==" }, "socket.io-parser": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.0.4.tgz", - "integrity": "sha512-t+b0SS+IxG7Rxzda2EVvyBZbvFPBCjJoyHuE0P//7OAsN23GItzDRdWa6ALxZI/8R5ygK7jAR6t028/z+7295g==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.0.5.tgz", + "integrity": "sha512-sNjbT9dX63nqUFIOv95tTVm6elyIU4RvB1m8dOeZt+IgWwcWklFDOdmGcfo3zSiRsnR/3pJkjY5lfoGqEe4Eig==", "requires": { "@types/component-emitter": "^1.2.10", "component-emitter": "~1.3.0", @@ -1717,9 +1828,9 @@ } }, "statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" }, "tiny-lru": { "version": "8.0.2", @@ -1753,31 +1864,31 @@ "uid2": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/uid2/-/uid2-0.0.3.tgz", - "integrity": "sha1-SDEm4Rd03y9xuLY53NeZw3YWK4I=" + "integrity": "sha512-5gSP1liv10Gjp8cMEnFd6shzkL/D6W1uhXSFNCxDC+YI8+L8wkCYCbJ7n77Ezb4wE/xzMogecE+DtamEe9PZjg==" }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, "uWebSockets.js": { - "version": "git+ssh://git@github.com/uNetworking/uWebSockets.js.git#a58e810e47a23696410f6073c8c905dc38f75da5", + "version": "git+ssh://git@github.com/uNetworking/uWebSockets.js.git#806df48c9da86af7b3341f3e443388c7cd15c3de", "from": "uWebSockets.js@github:uNetworking/uWebSockets.js#v20.10.0" }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", "requires": { "assert-plus": "^1.0.0", "core-util-is": "1.0.2", diff --git a/ee/utilities/utils/helper-ee.js b/ee/utilities/utils/helper-ee.js index dc821b94a..50b414b7a 100644 --- a/ee/utilities/utils/helper-ee.js +++ b/ee/utilities/utils/helper-ee.js @@ -70,7 +70,10 @@ const extractPayloadFromRequest = async function (req, res) { filters.filter.userID = [req.getQuery("userId")]; } if (!filters.query.value) { - let body = await getBodyFromUWSResponse(res); + let body = {}; + if (req.getMethod() !== 'get') { + body = await getBodyFromUWSResponse(res); + } filters = { ...filters, "sort": { From 8e4e5a0f70e12e5ad997c85878055922b1b194e7 Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Thu, 14 Jul 2022 12:11:47 +0200 Subject: [PATCH 021/553] v1.7.0 enhanced (#607) feat(alerts): changed Dockerfile --- api/Dockerfile | 2 +- api/Dockerfile.alerts | 2 +- api/Dockerfile.alerts.dockerignore | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 api/Dockerfile.alerts.dockerignore diff --git a/api/Dockerfile b/api/Dockerfile index 8b2acf1f7..7e99cc457 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.10-alpine LABEL Maintainer="Rajesh Rajendran" LABEL Maintainer="KRAIEM Taha Yassine" -RUN apk add --no-cache nodejs npm tini +RUN apk add --no-cache build-base nodejs npm tini ARG envarg # Add Tini # Startup daemon diff --git a/api/Dockerfile.alerts b/api/Dockerfile.alerts index 70879cae8..062a1cb3a 100644 --- a/api/Dockerfile.alerts +++ b/api/Dockerfile.alerts @@ -1,7 +1,7 @@ FROM python:3.10-alpine LABEL Maintainer="Rajesh Rajendran" LABEL Maintainer="KRAIEM Taha Yassine" -RUN apk add --no-cache tini +RUN apk add --no-cache build-base tini ARG envarg ENV APP_NAME=alerts \ pg_minconn=2 \ diff --git a/api/Dockerfile.alerts.dockerignore b/api/Dockerfile.alerts.dockerignore new file mode 100644 index 000000000..3539023b4 --- /dev/null +++ b/api/Dockerfile.alerts.dockerignore @@ -0,0 +1,10 @@ +# ignore .git and .cache folders +.git +.cache +**/build.sh +**/build_*.sh +**/*deploy.sh + +app.py +entrypoint_alerts.sh +requirements.txt \ No newline at end of file From e5fcd52686841fd61ddae31afb4ddea65c58823c Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Thu, 14 Jul 2022 13:55:08 +0200 Subject: [PATCH 022/553] fix(ui) - onboarding user form --- .../Client/Roles/components/RoleForm/RoleForm.tsx | 1 - frontend/app/components/Client/Users/UsersView.tsx | 5 +++-- .../Client/Users/components/UserList/UserList.tsx | 6 ++++-- .../Users/components/UserListItem/UserListItem.tsx | 14 +++++++++----- .../components/ManageUsersTab/ManageUsersTab.js | 7 +++---- .../app/components/Session_/Inspector/index.js | 2 -- .../Session_/Player/Controls/Timeline.js | 1 - .../components/shared/CodeSnippet/CodeSnippet.tsx | 1 - frontend/app/duck/assignments.js | 1 - frontend/app/mstore/funnelStore.ts | 1 - 10 files changed, 19 insertions(+), 20 deletions(-) diff --git a/frontend/app/components/Client/Roles/components/RoleForm/RoleForm.tsx b/frontend/app/components/Client/Roles/components/RoleForm/RoleForm.tsx index 578cc75ad..7aed70131 100644 --- a/frontend/app/components/Client/Roles/components/RoleForm/RoleForm.tsx +++ b/frontend/app/components/Client/Roles/components/RoleForm/RoleForm.tsx @@ -49,7 +49,6 @@ const RoleForm = (props: Props) => { } const writeOption = ({ name, value }: any) => { - console.log('name', name); if (name === 'permissions') { onChangePermissions(value) } else if (name === 'projects') { diff --git a/frontend/app/components/Client/Users/UsersView.tsx b/frontend/app/components/Client/Users/UsersView.tsx index 3b4d0c95f..a1240eb37 100644 --- a/frontend/app/components/Client/Users/UsersView.tsx +++ b/frontend/app/components/Client/Users/UsersView.tsx @@ -10,12 +10,13 @@ import { connect } from 'react-redux'; import AddUserButton from './components/AddUserButton'; interface Props { + isOnboarding?: boolean; account: any; isEnterprise: boolean; limits: any; } function UsersView(props: Props) { - const { account, limits, isEnterprise } = props; + const { account, limits, isEnterprise, isOnboarding = false } = props; const { userStore, roleStore } = useStore(); const userCount = useObserver(() => userStore.list.length); const roles = useObserver(() => roleStore.list); @@ -49,7 +50,7 @@ function UsersView(props: Props) {
- +
); } diff --git a/frontend/app/components/Client/Users/components/UserList/UserList.tsx b/frontend/app/components/Client/Users/components/UserList/UserList.tsx index cae5ffd43..a86e6d7d7 100644 --- a/frontend/app/components/Client/Users/components/UserList/UserList.tsx +++ b/frontend/app/components/Client/Users/components/UserList/UserList.tsx @@ -9,10 +9,11 @@ import UserForm from '../UserForm'; import AnimatedSVG, { ICONS } from 'Shared/AnimatedSVG/AnimatedSVG'; interface Props { + isOnboarding?: boolean; isEnterprise?: boolean; } function UserList(props: Props) { - const { isEnterprise = false } = props; + const { isEnterprise = false, isOnboarding = false } = props; const { userStore } = useStore(); const loading = useObserver(() => userStore.loading); const users = useObserver(() => userStore.list); @@ -55,7 +56,7 @@ function UserList(props: Props) {
Name
Role
-
Created On
+ {!isOnboarding &&
Created On
}
@@ -67,6 +68,7 @@ function UserList(props: Props) { generateInvite={() => userStore.generateInviteCode(user.userId)} copyInviteCode={() => userStore.copyInviteCode(user.userId)} isEnterprise={isEnterprise} + isOnboarding={isOnboarding} /> ))} diff --git a/frontend/app/components/Client/Users/components/UserListItem/UserListItem.tsx b/frontend/app/components/Client/Users/components/UserListItem/UserListItem.tsx index 8628f029f..d7b7d0d55 100644 --- a/frontend/app/components/Client/Users/components/UserListItem/UserListItem.tsx +++ b/frontend/app/components/Client/Users/components/UserListItem/UserListItem.tsx @@ -2,7 +2,7 @@ import React from 'react'; import { Icon, Popup } from 'UI'; import { checkForRecent } from 'App/date'; - +import cn from 'classnames'; const AdminPrivilegeLabel = ({ user }) => { return ( @@ -13,6 +13,7 @@ const AdminPrivilegeLabel = ({ user }) => { ) } interface Props { + isOnboarding?: boolean; user: any; editHandler?: any; generateInvite?: any; @@ -26,6 +27,7 @@ function UserListItem(props: Props) { generateInvite = () => {}, copyInviteCode = () => {}, isEnterprise = false, + isOnboarding = false } = props; return (
@@ -41,11 +43,13 @@ function UserListItem(props: Props) { )}
-
- {user.createdAt && checkForRecent(user.createdAt, 'LLL dd, yyyy, hh:mm a')} -
+ {!isOnboarding && ( +
+ {user.createdAt && checkForRecent(user.createdAt, 'LLL dd, yyyy, hh:mm a')} +
+ )} -
+
{!user.isJoined && user.invitationLink && !user.isExpiredInvite && ( diff --git a/frontend/app/components/Onboarding/components/ManageUsersTab/ManageUsersTab.js b/frontend/app/components/Onboarding/components/ManageUsersTab/ManageUsersTab.js index c4d414395..1f4763247 100644 --- a/frontend/app/components/Onboarding/components/ManageUsersTab/ManageUsersTab.js +++ b/frontend/app/components/Onboarding/components/ManageUsersTab/ManageUsersTab.js @@ -1,5 +1,5 @@ +import UsersView from 'App/components/Client/Users/UsersView' import React from 'react' -import ManageUsers from '../../../Client/ManageUsers' export default function ManageUsersTab() { return ( @@ -9,9 +9,8 @@ export default function ManageUsersTab() { 👨‍💻
Invite Collaborators
- - - + +
diff --git a/frontend/app/components/Session_/Inspector/index.js b/frontend/app/components/Session_/Inspector/index.js index e3fda7096..f76834fee 100644 --- a/frontend/app/components/Session_/Inspector/index.js +++ b/frontend/app/components/Session_/Inspector/index.js @@ -38,9 +38,7 @@ export default function Inspector () { const onKeyPress = e => { if (e.key === 'Backspace' || e.key === 'Delete') { const elem = selectedElementRef.current; - console.log(elem) if (elem !== null && elem.parentElement !== null) { - console.log('a?') elem.parentElement.removeChild(elem); setSelectedElement(null); } diff --git a/frontend/app/components/Session_/Player/Controls/Timeline.js b/frontend/app/components/Session_/Player/Controls/Timeline.js index 828715499..3acdb4c11 100644 --- a/frontend/app/components/Session_/Player/Controls/Timeline.js +++ b/frontend/app/components/Session_/Player/Controls/Timeline.js @@ -95,7 +95,6 @@ export default class Timeline extends React.PureComponent { const { endTime } = this.props; const p = e.nativeEvent.offsetX / e.target.offsetWidth; const time = Math.max(Math.round(p * endTime), 0); - console.log(p, time, e, endTime) this.props.jump(time); } diff --git a/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx b/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx index 76f752f70..c9629bf7a 100644 --- a/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx +++ b/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx @@ -21,7 +21,6 @@ interface Props { } function CodeSnippet(props: Props) { const { host, projectKey, ingestPoint, defaultInputMode, obscureTextNumbers, obscureTextEmails } = props; - console.log('defaultInputMode', defaultInputMode) const codeSnippet = `