From 700ef0dcc611c866588da6e6f798978785ad7c61 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Thu, 5 May 2022 15:26:10 +0200 Subject: [PATCH 01/25] Made standart project layout for ender service --- backend/build.sh | 2 +- backend/{services => cmd}/ender/main.go | 27 +++++++++---------- .../ender => internal}/builder/builder.go | 0 .../ender => internal}/builder/builderMap.go | 0 .../builder/clikRageDetector.go | 0 .../builder/cpuIssueFinder.go | 0 .../builder/deadClickDetector.go | 0 .../builder/domDropDetector.go | 0 .../builder/inputEventBuilder.go | 0 .../builder/memoryIssueFinder.go | 0 .../builder/pageEventBuilder.go | 0 .../builder/performanceTrackAggrBuilder.go | 0 backend/internal/config/ender/config.go | 25 +++++++++++++++++ backend/services/ender/build_hack | 0 14 files changed, 38 insertions(+), 16 deletions(-) rename backend/{services => cmd}/ender/main.go (69%) rename backend/{services/ender => internal}/builder/builder.go (100%) rename backend/{services/ender => internal}/builder/builderMap.go (100%) rename backend/{services/ender => internal}/builder/clikRageDetector.go (100%) rename backend/{services/ender => internal}/builder/cpuIssueFinder.go (100%) rename backend/{services/ender => internal}/builder/deadClickDetector.go (100%) rename backend/{services/ender => internal}/builder/domDropDetector.go (100%) rename backend/{services/ender => internal}/builder/inputEventBuilder.go (100%) rename backend/{services/ender => internal}/builder/memoryIssueFinder.go (100%) rename backend/{services/ender => internal}/builder/pageEventBuilder.go (100%) rename backend/{services/ender => internal}/builder/performanceTrackAggrBuilder.go (100%) create mode 100644 backend/internal/config/ender/config.go create mode 100644 backend/services/ender/build_hack diff --git a/backend/build.sh b/backend/build.sh index b4de3c2de..b78d71fd8 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -23,7 +23,7 @@ function build_service() { image="$1" echo "BUILDING $image" case "$image" in - http | db) + http | db | ender) echo build http docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image -f ./cmd/Dockerfile . [[ $PUSH_IMAGE -eq 1 ]] && { diff --git a/backend/services/ender/main.go b/backend/cmd/ender/main.go similarity index 69% rename from backend/services/ender/main.go rename to backend/cmd/ender/main.go index 4170a178e..b54a8dc15 100644 --- a/backend/services/ender/main.go +++ b/backend/cmd/ender/main.go @@ -2,37 +2,34 @@ package main import ( "log" + "openreplay/backend/internal/builder" + "openreplay/backend/internal/config/ender" "time" "os" "os/signal" "syscall" - "openreplay/backend/pkg/env" "openreplay/backend/pkg/intervals" logger "openreplay/backend/pkg/log" "openreplay/backend/pkg/messages" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/queue/types" - "openreplay/backend/services/ender/builder" ) func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - GROUP_EVENTS := env.String("GROUP_ENDER") - TOPIC_TRIGGER := env.String("TOPIC_TRIGGER") + cfg := ender.New() builderMap := builder.NewBuilderMap() - - statsLogger := logger.NewQueueStats(env.Int("LOG_QUEUE_STATS_INTERVAL_SEC")) - + statsLogger := logger.NewQueueStats(cfg.LoggerTimeout) producer := queue.NewProducer() consumer := queue.NewMessageConsumer( - GROUP_EVENTS, + cfg.GroupEvents, []string{ - env.String("TOPIC_RAW_WEB"), - env.String("TOPIC_RAW_IOS"), + cfg.TopicRawWeb, + cfg.TopicRawIOS, }, func(sessionID uint64, msg messages.Message, meta *types.Meta) { statsLogger.Collect(sessionID, meta) @@ -51,17 +48,17 @@ func main() { select { case sig := <-sigchan: log.Printf("Caught signal %v: terminating\n", sig) - producer.Close(2000) - consumer.CommitBack(intervals.EVENTS_BACK_COMMIT_GAP) + producer.Close(cfg.ProducerTimeout) + consumer.Commit() consumer.Close() os.Exit(0) case <-tick: builderMap.IterateReadyMessages(time.Now().UnixMilli(), func(sessionID uint64, readyMsg messages.Message) { - producer.Produce(TOPIC_TRIGGER, sessionID, messages.Encode(readyMsg)) + producer.Produce(cfg.TopicTrigger, sessionID, messages.Encode(readyMsg)) }) // TODO: why exactly do we need Flush here and not in any other place? - producer.Flush(2000) - consumer.CommitBack(intervals.EVENTS_BACK_COMMIT_GAP) + producer.Flush(cfg.ProducerTimeout) + consumer.Commit() default: if err := consumer.ConsumeNext(); err != nil { log.Fatalf("Error on consuming: %v", err) diff --git a/backend/services/ender/builder/builder.go b/backend/internal/builder/builder.go similarity index 100% rename from backend/services/ender/builder/builder.go rename to backend/internal/builder/builder.go diff --git a/backend/services/ender/builder/builderMap.go b/backend/internal/builder/builderMap.go similarity index 100% rename from backend/services/ender/builder/builderMap.go rename to backend/internal/builder/builderMap.go diff --git a/backend/services/ender/builder/clikRageDetector.go b/backend/internal/builder/clikRageDetector.go similarity index 100% rename from backend/services/ender/builder/clikRageDetector.go rename to backend/internal/builder/clikRageDetector.go diff --git a/backend/services/ender/builder/cpuIssueFinder.go b/backend/internal/builder/cpuIssueFinder.go similarity index 100% rename from backend/services/ender/builder/cpuIssueFinder.go rename to backend/internal/builder/cpuIssueFinder.go diff --git a/backend/services/ender/builder/deadClickDetector.go b/backend/internal/builder/deadClickDetector.go similarity index 100% rename from backend/services/ender/builder/deadClickDetector.go rename to backend/internal/builder/deadClickDetector.go diff --git a/backend/services/ender/builder/domDropDetector.go b/backend/internal/builder/domDropDetector.go similarity index 100% rename from backend/services/ender/builder/domDropDetector.go rename to backend/internal/builder/domDropDetector.go diff --git a/backend/services/ender/builder/inputEventBuilder.go b/backend/internal/builder/inputEventBuilder.go similarity index 100% rename from backend/services/ender/builder/inputEventBuilder.go rename to backend/internal/builder/inputEventBuilder.go diff --git a/backend/services/ender/builder/memoryIssueFinder.go b/backend/internal/builder/memoryIssueFinder.go similarity index 100% rename from backend/services/ender/builder/memoryIssueFinder.go rename to backend/internal/builder/memoryIssueFinder.go diff --git a/backend/services/ender/builder/pageEventBuilder.go b/backend/internal/builder/pageEventBuilder.go similarity index 100% rename from backend/services/ender/builder/pageEventBuilder.go rename to backend/internal/builder/pageEventBuilder.go diff --git a/backend/services/ender/builder/performanceTrackAggrBuilder.go b/backend/internal/builder/performanceTrackAggrBuilder.go similarity index 100% rename from backend/services/ender/builder/performanceTrackAggrBuilder.go rename to backend/internal/builder/performanceTrackAggrBuilder.go diff --git a/backend/internal/config/ender/config.go b/backend/internal/config/ender/config.go new file mode 100644 index 000000000..e39fbc240 --- /dev/null +++ b/backend/internal/config/ender/config.go @@ -0,0 +1,25 @@ +package ender + +import ( + "openreplay/backend/pkg/env" +) + +type Config struct { + GroupEvents string + TopicTrigger string + LoggerTimeout int + TopicRawWeb string + TopicRawIOS string + ProducerTimeout int +} + +func New() *Config { + return &Config{ + GroupEvents: env.String("GROUP_ENDER"), + TopicTrigger: env.String("TOPIC_TRIGGER"), + LoggerTimeout: env.Int("LOG_QUEUE_STATS_INTERVAL_SEC"), + TopicRawWeb: env.String("TOPIC_RAW_WEB"), + TopicRawIOS: env.String("TOPIC_RAW_IOS"), + ProducerTimeout: 2000, + } +} diff --git a/backend/services/ender/build_hack b/backend/services/ender/build_hack new file mode 100644 index 000000000..e69de29bb From f4212d6eaaf497623816b446c4515d33e5e54c8c Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Thu, 5 May 2022 17:37:05 +0200 Subject: [PATCH 02/25] Split ender into 2 services (ender and heuristics) --- backend/cmd/ender/main.go | 7 ++- backend/cmd/heuristics/main.go | 68 ++++++++++++++++++++++++++ backend/internal/builder/builder.go | 33 +++++++------ backend/internal/builder/builderMap.go | 12 ----- backend/internal/ender/builder.go | 56 +++++++++++++++++++++ backend/internal/ender/builderMap.go | 36 ++++++++++++++ backend/pkg/intervals/intervals.go | 12 ++--- 7 files changed, 186 insertions(+), 38 deletions(-) create mode 100644 backend/cmd/heuristics/main.go create mode 100644 backend/internal/ender/builder.go create mode 100644 backend/internal/ender/builderMap.go diff --git a/backend/cmd/ender/main.go b/backend/cmd/ender/main.go index b54a8dc15..dbe2fa212 100644 --- a/backend/cmd/ender/main.go +++ b/backend/cmd/ender/main.go @@ -2,8 +2,8 @@ package main import ( "log" - "openreplay/backend/internal/builder" "openreplay/backend/internal/config/ender" + builder "openreplay/backend/internal/ender" "time" "os" @@ -49,16 +49,15 @@ func main() { case sig := <-sigchan: log.Printf("Caught signal %v: terminating\n", sig) producer.Close(cfg.ProducerTimeout) - consumer.Commit() + consumer.CommitBack(intervals.EVENTS_BACK_COMMIT_GAP) consumer.Close() os.Exit(0) case <-tick: builderMap.IterateReadyMessages(time.Now().UnixMilli(), func(sessionID uint64, readyMsg messages.Message) { producer.Produce(cfg.TopicTrigger, sessionID, messages.Encode(readyMsg)) }) - // TODO: why exactly do we need Flush here and not in any other place? producer.Flush(cfg.ProducerTimeout) - consumer.Commit() + consumer.CommitBack(intervals.EVENTS_BACK_COMMIT_GAP) default: if err := consumer.ConsumeNext(); err != nil { log.Fatalf("Error on consuming: %v", err) diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go new file mode 100644 index 000000000..b33511bd1 --- /dev/null +++ b/backend/cmd/heuristics/main.go @@ -0,0 +1,68 @@ +package main + +import ( + "log" + "openreplay/backend/internal/builder" + "openreplay/backend/internal/config/ender" + "openreplay/backend/pkg/intervals" + logger "openreplay/backend/pkg/log" + "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/queue" + "openreplay/backend/pkg/queue/types" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) + + cfg := ender.New() + + builderMap := builder.NewBuilderMap() + statsLogger := logger.NewQueueStats(cfg.LoggerTimeout) + producer := queue.NewProducer() + consumer := queue.NewMessageConsumer( + cfg.GroupEvents, + []string{ + cfg.TopicRawWeb, + cfg.TopicRawIOS, + }, + func(sessionID uint64, msg messages.Message, meta *types.Meta) { + statsLogger.Collect(sessionID, meta) + builderMap.HandleMessage(sessionID, msg, msg.Meta().Index) + }, + false, + ) + + tick := time.Tick(intervals.EVENTS_COMMIT_INTERVAL * time.Millisecond) + + sigchan := make(chan os.Signal, 1) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + + log.Printf("Ender service started\n") + for { + select { + case sig := <-sigchan: + log.Printf("Caught signal %v: terminating\n", sig) + producer.Close(cfg.ProducerTimeout) + consumer.Commit() + consumer.Close() + os.Exit(0) + case <-tick: + builderMap.IterateReadyMessages(time.Now().UnixMilli(), func(sessionID uint64, readyMsg messages.Message) { + producer.Produce(cfg.TopicTrigger, sessionID, messages.Encode(readyMsg)) + }) + producer.Flush(cfg.ProducerTimeout) + consumer.Commit() + default: + if err := consumer.ConsumeNext(); err != nil { + log.Fatalf("Error on consuming: %v", err) + } + } + } + + // Config + +} diff --git a/backend/internal/builder/builder.go b/backend/internal/builder/builder.go index 1a89f67b6..bd9f26b19 100644 --- a/backend/internal/builder/builder.go +++ b/backend/internal/builder/builder.go @@ -44,9 +44,9 @@ type builder struct { readyMsgs []Message timestamp uint64 lastProcessedTimestamp int64 - peBuilder *pageEventBuilder + peBuilder *pageEventBuilder // TODO: DB ptaBuilder *performanceTrackAggrBuilder - ieBuilder *inputEventBuilder + ieBuilder *inputEventBuilder // TODO: DB ciFinder *cpuIssueFinder miFinder *memoryIssueFinder ddDetector *domDropDetector @@ -117,6 +117,7 @@ func (b *builder) handleMessage(message Message, messageID uint64) { b.lastProcessedTimestamp = time.Now().UnixMilli() // Might happen before the first timestamp. + // TODO: to DB switch msg := message.(type) { case *SessionStart, *Metadata, @@ -137,7 +138,7 @@ func (b *builder) handleMessage(message Message, messageID uint64) { return } switch msg := message.(type) { - case *SetPageLocation: + case *SetPageLocation: // TODO: DB if msg.NavigationStart == 0 { b.appendReadyMessage(&PageEvent{ URL: msg.URL, @@ -154,11 +155,11 @@ func (b *builder) handleMessage(message Message, messageID uint64) { b.miFinder.HandleSetPageLocation(msg) b.ciFinder.HandleSetPageLocation(msg) } - case *PageLoadTiming: + case *PageLoadTiming: // TODO: DB if rm := b.peBuilder.HandlePageLoadTiming(msg); rm != nil { b.appendReadyMessage(rm) } - case *PageRenderTiming: + case *PageRenderTiming: // TODO: DB if rm := b.peBuilder.HandlePageRenderTiming(msg); rm != nil { b.appendReadyMessage(rm) } @@ -172,20 +173,20 @@ func (b *builder) handleMessage(message Message, messageID uint64) { if rm := b.miFinder.HandlePerformanceTrack(msg, messageID, b.timestamp); rm != nil { b.appendReadyMessage(rm) } - case *SetInputTarget: + case *SetInputTarget: // TODO: DB if rm := b.ieBuilder.HandleSetInputTarget(msg); rm != nil { b.appendReadyMessage(rm) } - case *SetInputValue: + case *SetInputValue: // TODO: DB if rm := b.ieBuilder.HandleSetInputValue(msg, messageID, b.timestamp); rm != nil { b.appendReadyMessage(rm) } - case *MouseClick: + case *MouseClick: // TODO: DB b.buildInputEvent() if rm := b.crDetector.HandleMouseClick(msg, messageID, b.timestamp); rm != nil { b.appendReadyMessage(rm) } - if msg.Label != "" { + if msg.Label != "" { // TODO: DB b.appendReadyMessage(&ClickEvent{ MessageID: messageID, Label: msg.Label, @@ -195,7 +196,7 @@ func (b *builder) handleMessage(message Message, messageID uint64) { }) } case *JSException: - b.appendReadyMessage(&ErrorEvent{ + b.appendReadyMessage(&ErrorEvent{ // TODO: DB MessageID: messageID, Timestamp: b.timestamp, Source: "js_exception", @@ -206,7 +207,7 @@ func (b *builder) handleMessage(message Message, messageID uint64) { case *ResourceTiming: tp := getResourceType(msg.Initiator, msg.URL) success := msg.Duration != 0 - b.appendReadyMessage(&ResourceEvent{ + b.appendReadyMessage(&ResourceEvent{ // TODO: DB MessageID: messageID, Timestamp: msg.Timestamp, Duration: msg.Duration, @@ -231,14 +232,14 @@ func (b *builder) handleMessage(message Message, messageID uint64) { }) } case *RawCustomEvent: - b.appendReadyMessage(&CustomEvent{ + b.appendReadyMessage(&CustomEvent{ // TODO: DB MessageID: messageID, Timestamp: b.timestamp, Name: msg.Name, Payload: msg.Payload, }) case *CustomIssue: - b.appendReadyMessage(&IssueEvent{ + b.appendReadyMessage(&IssueEvent{ // TODO: DB Type: "custom", Timestamp: b.timestamp, MessageID: messageID, @@ -246,7 +247,7 @@ func (b *builder) handleMessage(message Message, messageID uint64) { Payload: msg.Payload, }) case *Fetch: - b.appendReadyMessage(&FetchEvent{ + b.appendReadyMessage(&FetchEvent{ // TODO: DB MessageID: messageID, Timestamp: msg.Timestamp, Method: msg.Method, @@ -265,7 +266,7 @@ func (b *builder) handleMessage(message Message, messageID uint64) { }) } case *GraphQL: - b.appendReadyMessage(&GraphQLEvent{ + b.appendReadyMessage(&GraphQLEvent{ // TODO: DB MessageID: messageID, Timestamp: b.timestamp, OperationKind: msg.OperationKind, @@ -274,7 +275,7 @@ func (b *builder) handleMessage(message Message, messageID uint64) { Response: msg.Response, }) case *StateAction: - b.appendReadyMessage(&StateActionEvent{ + b.appendReadyMessage(&StateActionEvent{ // TODO: DB MessageID: messageID, Timestamp: b.timestamp, Type: msg.Type, diff --git a/backend/internal/builder/builderMap.go b/backend/internal/builder/builderMap.go index 3f3e4d6e3..b7885da92 100644 --- a/backend/internal/builder/builderMap.go +++ b/backend/internal/builder/builderMap.go @@ -26,18 +26,6 @@ func (m builderMap) HandleMessage(sessionID uint64, msg Message, messageID uint6 b.handleMessage(msg, messageID) } -func (m builderMap) IterateSessionReadyMessages(sessionID uint64, operatingTs int64, iter func(msg Message)) { - b, ok := m[sessionID] - if !ok { - return - } - sessionEnded := b.checkTimeouts(operatingTs) - b.iterateReadyMessage(iter) - if sessionEnded { - delete(m, sessionID) - } -} - func (m builderMap) IterateReadyMessages(operatingTs int64, iter func(sessionID uint64, msg Message)) { for sessionID, b := range m { sessionEnded := b.checkTimeouts(operatingTs) diff --git a/backend/internal/ender/builder.go b/backend/internal/ender/builder.go new file mode 100644 index 000000000..0389f74d1 --- /dev/null +++ b/backend/internal/ender/builder.go @@ -0,0 +1,56 @@ +package builder + +import ( + "log" + "openreplay/backend/pkg/intervals" + . "openreplay/backend/pkg/messages" +) + +type builder struct { + readyMsgs []Message + timestamp uint64 + sid uint64 +} + +func NewBuilder() *builder { + return &builder{} +} + +func (b *builder) appendReadyMessage(msg Message) { // interface is never nil even if it holds nil value + b.readyMsgs = append(b.readyMsgs, msg) +} + +func (b *builder) buildSessionEnd() { + if b.timestamp == 0 { + return + } + sessionEnd := &SessionEnd{ + Timestamp: b.timestamp, + } + b.appendReadyMessage(sessionEnd) +} + +func (b *builder) handleMessage(message Message, messageID uint64) { + timestamp := GetTimestamp(message) + if b.timestamp < timestamp { + b.timestamp = timestamp + } + + if b.timestamp == 0 { + log.Printf("Empty timestamp, sessionID: %d, messageID: %d", b.sid, messageID) + return + } +} + +func (b *builder) checkTimeouts(ts int64) bool { + if b.timestamp == 0 { + return false // There was no timestamp events yet + } + + lastTsGap := ts - int64(b.timestamp) + if lastTsGap > intervals.EVENTS_SESSION_END_TIMEOUT { + b.buildSessionEnd() + return true + } + return false +} diff --git a/backend/internal/ender/builderMap.go b/backend/internal/ender/builderMap.go new file mode 100644 index 000000000..6eba1f9ad --- /dev/null +++ b/backend/internal/ender/builderMap.go @@ -0,0 +1,36 @@ +package builder + +import ( + . "openreplay/backend/pkg/messages" +) + +type builderMap map[uint64]*builder + +func NewBuilderMap() builderMap { + return make(builderMap) +} + +func (m builderMap) GetBuilder(sessionID uint64) *builder { + b := m[sessionID] + if b == nil { + b = NewBuilder() + m[sessionID] = b + b.sid = sessionID + + } + return b +} + +func (m builderMap) HandleMessage(sessionID uint64, msg Message, messageID uint64) { + b := m.GetBuilder(sessionID) + b.handleMessage(msg, messageID) +} + +func (m builderMap) IterateReadyMessages(operatingTs int64, iter func(sessionID uint64, msg Message)) { + for sessionID, b := range m { + sessionEnded := b.checkTimeouts(operatingTs) + if sessionEnded { + delete(m, sessionID) + } + } +} diff --git a/backend/pkg/intervals/intervals.go b/backend/pkg/intervals/intervals.go index c4dfbc835..2ce13ed5e 100644 --- a/backend/pkg/intervals/intervals.go +++ b/backend/pkg/intervals/intervals.go @@ -1,11 +1,11 @@ package intervals -const EVENTS_COMMIT_INTERVAL = 30 * 1000 -const HEARTBEAT_INTERVAL = 2 * 60 * 1000 -const INTEGRATIONS_REQUEST_INTERVAL = 1 * 60 * 1000 -const EVENTS_PAGE_EVENT_TIMEOUT = 2 * 60 * 1000 -const EVENTS_INPUT_EVENT_TIMEOUT = 2 * 60 * 1000 +const EVENTS_COMMIT_INTERVAL = 30 * 1000 // как часто комитим сообщения в кафке (ender) +const HEARTBEAT_INTERVAL = 2 * 60 * 1000 // максимальный таймаут от трекера в рамках сессии +const INTEGRATIONS_REQUEST_INTERVAL = 1 * 60 * 1000 // интеграции +const EVENTS_PAGE_EVENT_TIMEOUT = 2 * 60 * 1000 // таймаут пейдж ивента +const EVENTS_INPUT_EVENT_TIMEOUT = 2 * 60 * 1000 // const EVENTS_PERFORMANCE_AGGREGATION_TIMEOUT = 2 * 60 * 1000 const EVENTS_SESSION_END_TIMEOUT = HEARTBEAT_INTERVAL + 30*1000 const EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS = HEARTBEAT_INTERVAL + 3*60*1000 -const EVENTS_BACK_COMMIT_GAP = EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS + 1*60*1000 +const EVENTS_BACK_COMMIT_GAP = EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS + 1*60*1000 // для бэк коммита From 2b3728d8dabd2fc0244f4eb661e6162e455f54a7 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 6 May 2022 12:16:24 +0200 Subject: [PATCH 03/25] Finished refactoring for session ender service --- backend/cmd/ender/main.go | 30 +++++++++---- backend/internal/ender/builder.go | 56 ----------------------- backend/internal/ender/builderMap.go | 36 --------------- backend/internal/sessionender/ender.go | 62 ++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 101 deletions(-) delete mode 100644 backend/internal/ender/builder.go delete mode 100644 backend/internal/ender/builderMap.go create mode 100644 backend/internal/sessionender/ender.go diff --git a/backend/cmd/ender/main.go b/backend/cmd/ender/main.go index dbe2fa212..5d82b67db 100644 --- a/backend/cmd/ender/main.go +++ b/backend/cmd/ender/main.go @@ -3,7 +3,7 @@ package main import ( "log" "openreplay/backend/internal/config/ender" - builder "openreplay/backend/internal/ender" + "openreplay/backend/internal/sessionender" "time" "os" @@ -20,10 +20,12 @@ import ( func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) + // Load service configuration cfg := ender.New() - builderMap := builder.NewBuilderMap() + // Init all modules statsLogger := logger.NewQueueStats(cfg.LoggerTimeout) + sessions := sessionender.New(intervals.EVENTS_SESSION_END_TIMEOUT) producer := queue.NewProducer() consumer := queue.NewMessageConsumer( cfg.GroupEvents, @@ -33,31 +35,41 @@ func main() { }, func(sessionID uint64, msg messages.Message, meta *types.Meta) { statsLogger.Collect(sessionID, meta) - builderMap.HandleMessage(sessionID, msg, msg.Meta().Index) + sessions.UpdateSession(sessionID, messages.GetTimestamp(msg)) }, false, ) - tick := time.Tick(intervals.EVENTS_COMMIT_INTERVAL * time.Millisecond) + log.Printf("Ender service started\n") sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - log.Printf("Ender service started\n") + tick := time.Tick(intervals.EVENTS_COMMIT_INTERVAL * time.Millisecond) for { select { case sig := <-sigchan: log.Printf("Caught signal %v: terminating\n", sig) producer.Close(cfg.ProducerTimeout) - consumer.CommitBack(intervals.EVENTS_BACK_COMMIT_GAP) + if err := consumer.CommitBack(intervals.EVENTS_BACK_COMMIT_GAP); err != nil { + log.Printf("can't commit messages with offset: %s", err) + } consumer.Close() os.Exit(0) case <-tick: - builderMap.IterateReadyMessages(time.Now().UnixMilli(), func(sessionID uint64, readyMsg messages.Message) { - producer.Produce(cfg.TopicTrigger, sessionID, messages.Encode(readyMsg)) + // Find ended sessions and send notification to other services + sessions.HandleEndedSessions(func(sessionID uint64, timestamp int64) bool { + msg := &messages.SessionEnd{Timestamp: uint64(timestamp)} + if err := producer.Produce(cfg.TopicTrigger, sessionID, messages.Encode(msg)); err != nil { + log.Printf("can't send message to queue: %s", err) + return false + } + return true }) producer.Flush(cfg.ProducerTimeout) - consumer.CommitBack(intervals.EVENTS_BACK_COMMIT_GAP) + if err := consumer.CommitBack(intervals.EVENTS_BACK_COMMIT_GAP); err != nil { + log.Printf("can't commit messages with offset: %s", err) + } default: if err := consumer.ConsumeNext(); err != nil { log.Fatalf("Error on consuming: %v", err) diff --git a/backend/internal/ender/builder.go b/backend/internal/ender/builder.go deleted file mode 100644 index 0389f74d1..000000000 --- a/backend/internal/ender/builder.go +++ /dev/null @@ -1,56 +0,0 @@ -package builder - -import ( - "log" - "openreplay/backend/pkg/intervals" - . "openreplay/backend/pkg/messages" -) - -type builder struct { - readyMsgs []Message - timestamp uint64 - sid uint64 -} - -func NewBuilder() *builder { - return &builder{} -} - -func (b *builder) appendReadyMessage(msg Message) { // interface is never nil even if it holds nil value - b.readyMsgs = append(b.readyMsgs, msg) -} - -func (b *builder) buildSessionEnd() { - if b.timestamp == 0 { - return - } - sessionEnd := &SessionEnd{ - Timestamp: b.timestamp, - } - b.appendReadyMessage(sessionEnd) -} - -func (b *builder) handleMessage(message Message, messageID uint64) { - timestamp := GetTimestamp(message) - if b.timestamp < timestamp { - b.timestamp = timestamp - } - - if b.timestamp == 0 { - log.Printf("Empty timestamp, sessionID: %d, messageID: %d", b.sid, messageID) - return - } -} - -func (b *builder) checkTimeouts(ts int64) bool { - if b.timestamp == 0 { - return false // There was no timestamp events yet - } - - lastTsGap := ts - int64(b.timestamp) - if lastTsGap > intervals.EVENTS_SESSION_END_TIMEOUT { - b.buildSessionEnd() - return true - } - return false -} diff --git a/backend/internal/ender/builderMap.go b/backend/internal/ender/builderMap.go deleted file mode 100644 index 6eba1f9ad..000000000 --- a/backend/internal/ender/builderMap.go +++ /dev/null @@ -1,36 +0,0 @@ -package builder - -import ( - . "openreplay/backend/pkg/messages" -) - -type builderMap map[uint64]*builder - -func NewBuilderMap() builderMap { - return make(builderMap) -} - -func (m builderMap) GetBuilder(sessionID uint64) *builder { - b := m[sessionID] - if b == nil { - b = NewBuilder() - m[sessionID] = b - b.sid = sessionID - - } - return b -} - -func (m builderMap) HandleMessage(sessionID uint64, msg Message, messageID uint64) { - b := m.GetBuilder(sessionID) - b.handleMessage(msg, messageID) -} - -func (m builderMap) IterateReadyMessages(operatingTs int64, iter func(sessionID uint64, msg Message)) { - for sessionID, b := range m { - sessionEnded := b.checkTimeouts(operatingTs) - if sessionEnded { - delete(m, sessionID) - } - } -} diff --git a/backend/internal/sessionender/ender.go b/backend/internal/sessionender/ender.go new file mode 100644 index 000000000..54bd399ac --- /dev/null +++ b/backend/internal/sessionender/ender.go @@ -0,0 +1,62 @@ +package sessionender + +import ( + "log" + "time" +) + +// EndedSessionHandler handler for ended sessions +type EndedSessionHandler func(sessionID uint64, timestamp int64) bool + +// session holds information about user's session live status +type session struct { + lastTimestamp int64 + isEnded bool +} + +// SessionEnder updates timestamp of last message for each session +type SessionEnder struct { + timeout int64 + sessions map[uint64]*session // map[sessionID]session +} + +func New(timeout int64) *SessionEnder { + return &SessionEnder{ + timeout: timeout, + sessions: make(map[uint64]*session), + } +} + +// UpdateSession save timestamp for new sessions and update for existing sessions +func (se *SessionEnder) UpdateSession(sessionID, timestamp uint64) { + currTS := int64(timestamp) + if currTS == 0 { + log.Printf("got empty timestamp for sessionID: %d", sessionID) + return + } + sess, ok := se.sessions[sessionID] + if !ok { + se.sessions[sessionID] = &session{ + lastTimestamp: currTS, + isEnded: false, + } + return + } + if currTS > sess.lastTimestamp { + sess.lastTimestamp = currTS + sess.isEnded = false + } +} + +// HandleEndedSessions runs handler for each ended session and delete information about session in successful case +func (se *SessionEnder) HandleEndedSessions(handler EndedSessionHandler) { + deadLine := time.Now().UnixMilli() - se.timeout + for sessID, sess := range se.sessions { + if sess.isEnded || sess.lastTimestamp < deadLine { + sess.isEnded = true + if handler(sessID, sess.lastTimestamp) { + delete(se.sessions, sessID) + } + } + } +} From 967034a89cedeb05473ed67e70c0e8e16c7e4b44 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 6 May 2022 16:12:06 +0200 Subject: [PATCH 04/25] Create first version of heuristics service with the same logic as old ender --- backend/build.sh | 2 +- backend/cmd/heuristics/main.go | 7 ++----- backend/services/heuristics/build_hack | 0 3 files changed, 3 insertions(+), 6 deletions(-) create mode 100644 backend/services/heuristics/build_hack diff --git a/backend/build.sh b/backend/build.sh index b78d71fd8..70a29c5af 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -23,7 +23,7 @@ function build_service() { image="$1" echo "BUILDING $image" case "$image" in - http | db | ender) + http | db | ender | heuristics) echo build http docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image -f ./cmd/Dockerfile . [[ $PUSH_IMAGE -eq 1 ]] && { diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index b33511bd1..2778685d3 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -36,12 +36,12 @@ func main() { false, ) - tick := time.Tick(intervals.EVENTS_COMMIT_INTERVAL * time.Millisecond) + log.Printf("Ender service started\n") sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - log.Printf("Ender service started\n") + tick := time.Tick(intervals.EVENTS_COMMIT_INTERVAL * time.Millisecond) for { select { case sig := <-sigchan: @@ -62,7 +62,4 @@ func main() { } } } - - // Config - } diff --git a/backend/services/heuristics/build_hack b/backend/services/heuristics/build_hack new file mode 100644 index 000000000..e69de29bb From 8c432b8ba379dc2018e9252d36bb8e28d0a1eb61 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 6 May 2022 16:39:29 +0200 Subject: [PATCH 05/25] Removed from heuristics extra logic --- backend/internal/builder/builder.go | 185 +----------------- .../inputEventBuilder.go | 0 .../pageEventBuilder.go | 0 3 files changed, 1 insertion(+), 184 deletions(-) rename backend/internal/{builder => heuristics}/inputEventBuilder.go (100%) rename backend/internal/{builder => heuristics}/pageEventBuilder.go (100%) diff --git a/backend/internal/builder/builder.go b/backend/internal/builder/builder.go index bd9f26b19..b3dc909b7 100644 --- a/backend/internal/builder/builder.go +++ b/backend/internal/builder/builder.go @@ -44,9 +44,7 @@ type builder struct { readyMsgs []Message timestamp uint64 lastProcessedTimestamp int64 - peBuilder *pageEventBuilder // TODO: DB ptaBuilder *performanceTrackAggrBuilder - ieBuilder *inputEventBuilder // TODO: DB ciFinder *cpuIssueFinder miFinder *memoryIssueFinder ddDetector *domDropDetector @@ -59,9 +57,7 @@ type builder struct { func NewBuilder() *builder { return &builder{ - peBuilder: &pageEventBuilder{}, ptaBuilder: &performanceTrackAggrBuilder{}, - ieBuilder: NewInputEventBuilder(), ciFinder: &cpuIssueFinder{}, miFinder: &memoryIssueFinder{}, ddDetector: &domDropDetector{}, @@ -82,87 +78,24 @@ func (b *builder) iterateReadyMessage(iter func(msg Message)) { b.readyMsgs = nil } -func (b *builder) buildSessionEnd() { - if b.timestamp == 0 { - return - } - sessionEnd := &SessionEnd{ - Timestamp: b.timestamp, // + delay? - } - b.appendReadyMessage(sessionEnd) -} - -func (b *builder) buildPageEvent() { - if msg := b.peBuilder.Build(); msg != nil { - b.appendReadyMessage(msg) - } -} func (b *builder) buildPerformanceTrackAggr() { if msg := b.ptaBuilder.Build(); msg != nil { b.appendReadyMessage(msg) } } -func (b *builder) buildInputEvent() { - if msg := b.ieBuilder.Build(); msg != nil { - b.appendReadyMessage(msg) - } -} func (b *builder) handleMessage(message Message, messageID uint64) { timestamp := GetTimestamp(message) - if b.timestamp < timestamp { // unnecessary? TODO: test and remove + if b.timestamp < timestamp { b.timestamp = timestamp } b.lastProcessedTimestamp = time.Now().UnixMilli() - // Might happen before the first timestamp. - // TODO: to DB - switch msg := message.(type) { - case *SessionStart, - *Metadata, - *UserID, - *UserAnonymousID: - b.appendReadyMessage(msg) - case *RawErrorEvent: - b.appendReadyMessage(&ErrorEvent{ - MessageID: messageID, - Timestamp: msg.Timestamp, - Source: msg.Source, - Name: msg.Name, - Message: msg.Message, - Payload: msg.Payload, - }) - } if b.timestamp == 0 { return } switch msg := message.(type) { - case *SetPageLocation: // TODO: DB - if msg.NavigationStart == 0 { - b.appendReadyMessage(&PageEvent{ - URL: msg.URL, - Referrer: msg.Referrer, - Loaded: false, - MessageID: messageID, - Timestamp: b.timestamp, - }) - } else { - b.buildPageEvent() - b.buildInputEvent() - b.ieBuilder.ClearLabels() - b.peBuilder.HandleSetPageLocation(msg, messageID, b.timestamp) - b.miFinder.HandleSetPageLocation(msg) - b.ciFinder.HandleSetPageLocation(msg) - } - case *PageLoadTiming: // TODO: DB - if rm := b.peBuilder.HandlePageLoadTiming(msg); rm != nil { - b.appendReadyMessage(rm) - } - case *PageRenderTiming: // TODO: DB - if rm := b.peBuilder.HandlePageRenderTiming(msg); rm != nil { - b.appendReadyMessage(rm) - } case *PerformanceTrack: if rm := b.ptaBuilder.HandlePerformanceTrack(msg, b.timestamp); rm != nil { b.appendReadyMessage(rm) @@ -173,113 +106,6 @@ func (b *builder) handleMessage(message Message, messageID uint64) { if rm := b.miFinder.HandlePerformanceTrack(msg, messageID, b.timestamp); rm != nil { b.appendReadyMessage(rm) } - case *SetInputTarget: // TODO: DB - if rm := b.ieBuilder.HandleSetInputTarget(msg); rm != nil { - b.appendReadyMessage(rm) - } - case *SetInputValue: // TODO: DB - if rm := b.ieBuilder.HandleSetInputValue(msg, messageID, b.timestamp); rm != nil { - b.appendReadyMessage(rm) - } - case *MouseClick: // TODO: DB - b.buildInputEvent() - if rm := b.crDetector.HandleMouseClick(msg, messageID, b.timestamp); rm != nil { - b.appendReadyMessage(rm) - } - if msg.Label != "" { // TODO: DB - b.appendReadyMessage(&ClickEvent{ - MessageID: messageID, - Label: msg.Label, - HesitationTime: msg.HesitationTime, - Timestamp: b.timestamp, - Selector: msg.Selector, - }) - } - case *JSException: - b.appendReadyMessage(&ErrorEvent{ // TODO: DB - MessageID: messageID, - Timestamp: b.timestamp, - Source: "js_exception", - Name: msg.Name, - Message: msg.Message, - Payload: msg.Payload, - }) - case *ResourceTiming: - tp := getResourceType(msg.Initiator, msg.URL) - success := msg.Duration != 0 - b.appendReadyMessage(&ResourceEvent{ // TODO: DB - MessageID: messageID, - Timestamp: msg.Timestamp, - Duration: msg.Duration, - TTFB: msg.TTFB, - HeaderSize: msg.HeaderSize, - EncodedBodySize: msg.EncodedBodySize, - DecodedBodySize: msg.DecodedBodySize, - URL: msg.URL, - Type: tp, - Success: success, - }) - if !success { - issueType := "missing_resource" - if tp == "fetch" { - issueType = "bad_request" - } - b.appendReadyMessage(&IssueEvent{ - Type: issueType, - MessageID: messageID, - Timestamp: msg.Timestamp, - ContextString: msg.URL, - }) - } - case *RawCustomEvent: - b.appendReadyMessage(&CustomEvent{ // TODO: DB - MessageID: messageID, - Timestamp: b.timestamp, - Name: msg.Name, - Payload: msg.Payload, - }) - case *CustomIssue: - b.appendReadyMessage(&IssueEvent{ // TODO: DB - Type: "custom", - Timestamp: b.timestamp, - MessageID: messageID, - ContextString: msg.Name, - Payload: msg.Payload, - }) - case *Fetch: - b.appendReadyMessage(&FetchEvent{ // TODO: DB - MessageID: messageID, - Timestamp: msg.Timestamp, - Method: msg.Method, - URL: msg.URL, - Request: msg.Request, - Response: msg.Response, - Status: msg.Status, - Duration: msg.Duration, - }) - if msg.Status >= 400 { - b.appendReadyMessage(&IssueEvent{ - Type: "bad_request", - MessageID: messageID, - Timestamp: msg.Timestamp, - ContextString: msg.URL, - }) - } - case *GraphQL: - b.appendReadyMessage(&GraphQLEvent{ // TODO: DB - MessageID: messageID, - Timestamp: b.timestamp, - OperationKind: msg.OperationKind, - OperationName: msg.OperationName, - Variables: msg.Variables, - Response: msg.Response, - }) - case *StateAction: - b.appendReadyMessage(&StateActionEvent{ // TODO: DB - MessageID: messageID, - Timestamp: b.timestamp, - Type: msg.Type, - }) case *CreateElementNode, *CreateTextNode: b.ddDetector.HandleNodeCreation() @@ -300,19 +126,11 @@ func (b *builder) checkTimeouts(ts int64) bool { return false // There was no timestamp events yet } - if b.peBuilder.HasInstance() && int64(b.peBuilder.GetTimestamp())+intervals.EVENTS_PAGE_EVENT_TIMEOUT < ts { - b.buildPageEvent() - } - if b.ieBuilder.HasInstance() && int64(b.ieBuilder.GetTimestamp())+intervals.EVENTS_INPUT_EVENT_TIMEOUT < ts { - b.buildInputEvent() - } if b.ptaBuilder.HasInstance() && int64(b.ptaBuilder.GetStartTimestamp())+intervals.EVENTS_PERFORMANCE_AGGREGATION_TIMEOUT < ts { b.buildPerformanceTrackAggr() } lastTsGap := ts - int64(b.timestamp) - //b.lastProcessedTimestamp - //log.Printf("checking timeouts for sess %v: %v now, %v sesstime; gap %v",b.sid, ts, b.timestamp, lastTsGap) if lastTsGap > intervals.EVENTS_SESSION_END_TIMEOUT { if rm := b.ddDetector.Build(); rm != nil { b.appendReadyMessage(rm) @@ -329,7 +147,6 @@ func (b *builder) checkTimeouts(ts int64) bool { if rm := b.dcDetector.HandleReaction(b.timestamp); rm != nil { b.appendReadyMessage(rm) } - b.buildSessionEnd() return true } return false diff --git a/backend/internal/builder/inputEventBuilder.go b/backend/internal/heuristics/inputEventBuilder.go similarity index 100% rename from backend/internal/builder/inputEventBuilder.go rename to backend/internal/heuristics/inputEventBuilder.go diff --git a/backend/internal/builder/pageEventBuilder.go b/backend/internal/heuristics/pageEventBuilder.go similarity index 100% rename from backend/internal/builder/pageEventBuilder.go rename to backend/internal/heuristics/pageEventBuilder.go From 432c0da4e2ea3c0fe0ab5c202d74c85cb809567f Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Sat, 7 May 2022 15:04:17 +0200 Subject: [PATCH 06/25] chore(backend-heuristics): Remove redundant lines --- backend/internal/builder/builder.go | 57 +++++------------------------ 1 file changed, 9 insertions(+), 48 deletions(-) diff --git a/backend/internal/builder/builder.go b/backend/internal/builder/builder.go index b3dc909b7..6b8ece5d5 100644 --- a/backend/internal/builder/builder.go +++ b/backend/internal/builder/builder.go @@ -9,37 +9,6 @@ import ( . "openreplay/backend/pkg/messages" ) -func getURLExtention(URL string) string { - u, err := url.Parse(URL) - if err != nil { - return "" - } - i := strings.LastIndex(u.Path, ".") - return u.Path[i+1:] -} - -func getResourceType(initiator string, URL string) string { - switch initiator { - case "xmlhttprequest", "fetch": - return "fetch" - case "img": - return "img" - default: - switch getURLExtention(URL) { - case "css": - return "stylesheet" - case "js": - return "script" - case "png", "gif", "jpg", "jpeg", "svg": - return "img" - case "mp4", "mkv", "ogg", "webm", "avi", "mp3": - return "media" - default: - return "other" - } - } -} - type builder struct { readyMsgs []Message timestamp uint64 @@ -50,20 +19,16 @@ type builder struct { ddDetector *domDropDetector crDetector *clickRageDetector dcDetector *deadClickDetector - integrationsWaiting bool - - sid uint64 } func NewBuilder() *builder { return &builder{ - ptaBuilder: &performanceTrackAggrBuilder{}, - ciFinder: &cpuIssueFinder{}, - miFinder: &memoryIssueFinder{}, - ddDetector: &domDropDetector{}, - crDetector: &clickRageDetector{}, - dcDetector: &deadClickDetector{}, - integrationsWaiting: true, + ptaBuilder: &performanceTrackAggrBuilder{}, + ciFinder: &cpuIssueFinder{}, + miFinder: &memoryIssueFinder{}, + ddDetector: &domDropDetector{}, + crDetector: &clickRageDetector{}, + dcDetector: &deadClickDetector{}, } } @@ -78,12 +43,6 @@ func (b *builder) iterateReadyMessage(iter func(msg Message)) { b.readyMsgs = nil } -func (b *builder) buildPerformanceTrackAggr() { - if msg := b.ptaBuilder.Build(); msg != nil { - b.appendReadyMessage(msg) - } -} - func (b *builder) handleMessage(message Message, messageID uint64) { timestamp := GetTimestamp(message) if b.timestamp < timestamp { @@ -127,7 +86,9 @@ func (b *builder) checkTimeouts(ts int64) bool { } if b.ptaBuilder.HasInstance() && int64(b.ptaBuilder.GetStartTimestamp())+intervals.EVENTS_PERFORMANCE_AGGREGATION_TIMEOUT < ts { - b.buildPerformanceTrackAggr() + if msg := b.ptaBuilder.Build(); msg != nil { + b.appendReadyMessage(msg) + } } lastTsGap := ts - int64(b.timestamp) From 62b36bd70a577853952afcba5eabea3dbd7ff1e2 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Sat, 7 May 2022 21:29:40 +0200 Subject: [PATCH 07/25] refactor(backend-heuristics): bring all sub-bilders to common interface --- backend/internal/builder/builder.go | 104 ++++------- backend/internal/builder/builderMap.go | 2 - backend/internal/builder/clikRageDetector.go | 55 +++--- backend/internal/builder/cpuIssueFinder.go | 58 +++--- backend/internal/builder/deadClickDetector.go | 63 ++++--- backend/internal/builder/domDropDetector.go | 41 +++-- backend/internal/builder/memoryIssueFinder.go | 48 ++--- .../builder/performanceTrackAggrBuilder.go | 167 +++++++++--------- backend/pkg/intervals/intervals.go | 1 - 9 files changed, 269 insertions(+), 270 deletions(-) diff --git a/backend/internal/builder/builder.go b/backend/internal/builder/builder.go index 6b8ece5d5..7e062e6b2 100644 --- a/backend/internal/builder/builder.go +++ b/backend/internal/builder/builder.go @@ -1,41 +1,34 @@ package builder import ( - "net/url" - "strings" - "time" - "openreplay/backend/pkg/intervals" . "openreplay/backend/pkg/messages" ) +type messageProcessor interface { + Handle(message Message, messageID uint64, timestamp uint64) Message + Build() Message +} + type builder struct { - readyMsgs []Message - timestamp uint64 - lastProcessedTimestamp int64 - ptaBuilder *performanceTrackAggrBuilder - ciFinder *cpuIssueFinder - miFinder *memoryIssueFinder - ddDetector *domDropDetector - crDetector *clickRageDetector - dcDetector *deadClickDetector + readyMsgs []Message + timestamp uint64 + processors []messageProcessor } func NewBuilder() *builder { return &builder{ - ptaBuilder: &performanceTrackAggrBuilder{}, - ciFinder: &cpuIssueFinder{}, - miFinder: &memoryIssueFinder{}, - ddDetector: &domDropDetector{}, - crDetector: &clickRageDetector{}, - dcDetector: &deadClickDetector{}, + processors: []messageProcessor{ + &performanceTrackAggrBuilder{}, + &cpuIssueFinder{}, + &memoryIssueFinder{}, + // &domDropDetector{}, + &clickRageDetector{}, + &deadClickDetector{}, + }, } } -func (b *builder) appendReadyMessage(msg Message) { // interface is never nil even if it holds nil value - b.readyMsgs = append(b.readyMsgs, msg) -} - func (b *builder) iterateReadyMessage(iter func(msg Message)) { for _, readyMsg := range b.readyMsgs { iter(readyMsg) @@ -48,65 +41,38 @@ func (b *builder) handleMessage(message Message, messageID uint64) { if b.timestamp < timestamp { b.timestamp = timestamp } - - b.lastProcessedTimestamp = time.Now().UnixMilli() - if b.timestamp == 0 { + // in case of SessionStart. TODO: make timestamp system transparent return } - switch msg := message.(type) { - case *PerformanceTrack: - if rm := b.ptaBuilder.HandlePerformanceTrack(msg, b.timestamp); rm != nil { - b.appendReadyMessage(rm) + + for _, p := range b.processors { + /* If nil is not returned explicitely by Handle, but as the typed nil + ("var i *IssueEvent; return i;") + The `rm != nil` will be true. + TODO: enforce nil to be nil(?) or add `isNil() bool` to the Message types + because this part is expected to be etendable by user with custom messageProcessor's. + Use of reflrction will be probably bad on millions of messages? + */ + if rm := p.Handle(message, messageID, b.timestamp); rm != nil { + b.readyMsgs = append(b.readyMsgs, rm) } - if rm := b.ciFinder.HandlePerformanceTrack(msg, messageID, b.timestamp); rm != nil { - b.appendReadyMessage(rm) - } - if rm := b.miFinder.HandlePerformanceTrack(msg, messageID, b.timestamp); rm != nil { - b.appendReadyMessage(rm) - } - case *CreateElementNode, - *CreateTextNode: - b.ddDetector.HandleNodeCreation() - case *RemoveNode: - b.ddDetector.HandleNodeRemoval(b.timestamp) - case *CreateDocument: - if rm := b.ddDetector.Build(); rm != nil { - b.appendReadyMessage(rm) - } - } - if rm := b.dcDetector.HandleMessage(message, messageID, b.timestamp); rm != nil { - b.appendReadyMessage(rm) } } func (b *builder) checkTimeouts(ts int64) bool { if b.timestamp == 0 { - return false // There was no timestamp events yet - } - - if b.ptaBuilder.HasInstance() && int64(b.ptaBuilder.GetStartTimestamp())+intervals.EVENTS_PERFORMANCE_AGGREGATION_TIMEOUT < ts { - if msg := b.ptaBuilder.Build(); msg != nil { - b.appendReadyMessage(msg) - } + return false // SessionStart happened only } lastTsGap := ts - int64(b.timestamp) + // Maybe listen for `trigger` and react on SessionEnd instead (less reliable) if lastTsGap > intervals.EVENTS_SESSION_END_TIMEOUT { - if rm := b.ddDetector.Build(); rm != nil { - b.appendReadyMessage(rm) - } - if rm := b.ciFinder.Build(); rm != nil { - b.appendReadyMessage(rm) - } - if rm := b.miFinder.Build(); rm != nil { - b.appendReadyMessage(rm) - } - if rm := b.crDetector.Build(); rm != nil { - b.appendReadyMessage(rm) - } - if rm := b.dcDetector.HandleReaction(b.timestamp); rm != nil { - b.appendReadyMessage(rm) + for _, p := range b.processors { + // TODO: same as above + if rm := p.Build(); rm != nil { + b.readyMsgs = append(b.readyMsgs, rm) + } } return true } diff --git a/backend/internal/builder/builderMap.go b/backend/internal/builder/builderMap.go index b7885da92..6caf18e4f 100644 --- a/backend/internal/builder/builderMap.go +++ b/backend/internal/builder/builderMap.go @@ -15,8 +15,6 @@ func (m builderMap) GetBuilder(sessionID uint64) *builder { if b == nil { b = NewBuilder() m[sessionID] = b - b.sid = sessionID - } return b } diff --git a/backend/internal/builder/clikRageDetector.go b/backend/internal/builder/clikRageDetector.go index f25efbcd9..1140027b3 100644 --- a/backend/internal/builder/clikRageDetector.go +++ b/backend/internal/builder/clikRageDetector.go @@ -6,7 +6,7 @@ import ( . "openreplay/backend/pkg/messages" ) -const CLICK_TIME_DIFF = 300 +const MAX_TIME_DIFF = 300 const MIN_CLICKS_IN_A_ROW = 3 type clickRageDetector struct { @@ -17,39 +17,50 @@ type clickRageDetector struct { countsInARow int } -func (crd *clickRageDetector) Build() *IssueEvent { - var i *IssueEvent - if crd.countsInARow >= MIN_CLICKS_IN_A_ROW { - payload, _ := json.Marshal(struct{ Count int }{crd.countsInARow}) - i = &IssueEvent{ - Type: "click_rage", - ContextString: crd.lastLabel, - Payload: string(payload), // TODO: json encoder - Timestamp: crd.firstInARawTimestamp, - MessageID: crd.firstInARawMessageId, - } - } +func (crd *clickRageDetector) reset() { crd.lastTimestamp = 0 crd.lastLabel = "" crd.firstInARawTimestamp = 0 crd.firstInARawMessageId = 0 crd.countsInARow = 0 - return i } -func (crd *clickRageDetector) HandleMouseClick(msg *MouseClick, messageID uint64, timestamp uint64) *IssueEvent { - if crd.lastTimestamp+CLICK_TIME_DIFF > timestamp && crd.lastLabel == msg.Label { - crd.lastTimestamp = timestamp - crd.countsInARow += 1 - return nil +func (crd *clickRageDetector) Build() Message { + if crd.countsInARow >= MIN_CLICKS_IN_A_ROW { + payload, _ := json.Marshal(struct{ Count int }{crd.countsInARow}) + i := &IssueEvent{ + Type: "click_rage", + ContextString: crd.lastLabel, + Payload: string(payload), // TODO: json message field type + Timestamp: crd.firstInARawTimestamp, + MessageID: crd.firstInARawMessageId, + } + crd.reset() + return i } - i := crd.Build() - if msg.Label != "" { + crd.reset() + return nil +} + +func (crd *clickRageDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { + switch msg := message.(type) { + case *MouseClick: + // TODO: check if we it is ok to capture clickrages without the connected CleckEvent in db. + if msg.Label == "" { + return crd.Build() + } + if crd.lastLabel == msg.Label && timestamp-crd.lastTimestamp < MAX_TIME_DIFF { + crd.lastTimestamp = timestamp + crd.countsInARow += 1 + return nil + } + i := crd.Build() crd.lastTimestamp = timestamp crd.lastLabel = msg.Label crd.firstInARawTimestamp = timestamp crd.firstInARawMessageId = messageID crd.countsInARow = 1 + return i } - return i + return nil } diff --git a/backend/internal/builder/cpuIssueFinder.go b/backend/internal/builder/cpuIssueFinder.go index 1af867ea3..feb694a86 100644 --- a/backend/internal/builder/cpuIssueFinder.go +++ b/backend/internal/builder/cpuIssueFinder.go @@ -18,7 +18,7 @@ type cpuIssueFinder struct { contextString string } -func (f *cpuIssueFinder) Build() *IssueEvent { +func (f *cpuIssueFinder) Build() Message { if f.startTimestamp == 0 { return nil } @@ -47,35 +47,35 @@ func (f *cpuIssueFinder) Build() *IssueEvent { } } -func (f *cpuIssueFinder) HandleSetPageLocation(msg *SetPageLocation) { - f.contextString = msg.URL -} - -func (f *cpuIssueFinder) HandlePerformanceTrack(msg *PerformanceTrack, messageID uint64, timestamp uint64) *IssueEvent { - dt := performance.TimeDiff(timestamp, f.lastTimestamp) - if dt == 0 { - return nil // TODO: handle error - } - - f.lastTimestamp = timestamp - - if msg.Frames == -1 || msg.Ticks == -1 { - return f.Build() - } - - cpuRate := performance.CPURate(msg.Ticks, dt) - - if cpuRate >= CPU_THRESHOLD { - if f.startTimestamp == 0 { - f.startTimestamp = timestamp - f.startMessageID = messageID +func (f *cpuIssueFinder) Handle(message Message, messageID uint64, timestamp uint64) Message { + switch msg := message.(type) { + case *PerformanceTrack: + dt := performance.TimeDiff(timestamp, f.lastTimestamp) + if dt == 0 { + return nil // TODO: handle error } - if f.maxRate < cpuRate { - f.maxRate = cpuRate - } - } else { - return f.Build() - } + f.lastTimestamp = timestamp + + if msg.Frames == -1 || msg.Ticks == -1 { + return f.Build() + } + + cpuRate := performance.CPURate(msg.Ticks, dt) + + if cpuRate >= CPU_THRESHOLD { + if f.startTimestamp == 0 { + f.startTimestamp = timestamp + f.startMessageID = messageID + } + if f.maxRate < cpuRate { + f.maxRate = cpuRate + } + } else { + return f.Build() + } + case *SetPageLocation: + f.contextString = msg.URL + } return nil } diff --git a/backend/internal/builder/deadClickDetector.go b/backend/internal/builder/deadClickDetector.go index de977b7bd..f83c0bedd 100644 --- a/backend/internal/builder/deadClickDetector.go +++ b/backend/internal/builder/deadClickDetector.go @@ -7,50 +7,61 @@ import ( const CLICK_RELATION_TIME = 1400 type deadClickDetector struct { - lastMouseClick *MouseClick - lastTimestamp uint64 - lastMessageID uint64 - inputIDSet map[uint64]bool + lastTimestamp uint64 + lastMouseClick *MouseClick + lastClickTimestamp uint64 + lastMessageID uint64 + inputIDSet map[uint64]bool } -func (d *deadClickDetector) HandleReaction(timestamp uint64) *IssueEvent { - var i *IssueEvent - if d.lastMouseClick != nil && d.lastTimestamp+CLICK_RELATION_TIME < timestamp { - i = &IssueEvent{ - Type: "dead_click", - ContextString: d.lastMouseClick.Label, - Timestamp: d.lastTimestamp, - MessageID: d.lastMessageID, - } - } +func (d *deadClickDetector) reset() { d.inputIDSet = nil d.lastMouseClick = nil - d.lastTimestamp = 0 + d.lastClickTimestamp = 0 d.lastMessageID = 0 +} + +func (d *deadClickDetector) handleReaction(timestamp uint64) Message { + if d.lastMouseClick == nil || d.lastClickTimestamp+CLICK_RELATION_TIME > timestamp { // riaction is instant + d.reset() + return nil + } + i := &IssueEvent{ + Type: "dead_click", + ContextString: d.lastMouseClick.Label, + Timestamp: d.lastClickTimestamp, + MessageID: d.lastMessageID, + } + d.reset() return i } -func (d *deadClickDetector) HandleMessage(msg Message, messageID uint64, timestamp uint64) *IssueEvent { - var i *IssueEvent - switch m := msg.(type) { +func (d *deadClickDetector) Build() Message { + return d.handleReaction(d.lastTimestamp) +} + +func (d *deadClickDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { + d.lastTimestamp = timestamp + switch msg := message.(type) { case *SetInputTarget: if d.inputIDSet == nil { d.inputIDSet = make(map[uint64]bool) } - d.inputIDSet[m.ID] = true + d.inputIDSet[msg.ID] = true case *CreateDocument: d.inputIDSet = nil case *MouseClick: - if m.Label == "" { + if msg.Label == "" { return nil } - i = d.HandleReaction(timestamp) - if d.inputIDSet[m.ID] { // ignore if input + i := d.handleReaction(timestamp) + if d.inputIDSet[msg.ID] { // ignore if input return i } - d.lastMouseClick = m - d.lastTimestamp = timestamp + d.lastMouseClick = msg + d.lastClickTimestamp = timestamp d.lastMessageID = messageID + return i case *SetNodeAttribute, *RemoveNodeAttribute, *CreateElementNode, @@ -60,7 +71,7 @@ func (d *deadClickDetector) HandleMessage(msg Message, messageID uint64, timesta *SetCSSData, *CSSInsertRule, *CSSDeleteRule: - i = d.HandleReaction(timestamp) + return d.handleReaction(timestamp) } - return i + return nil } diff --git a/backend/internal/builder/domDropDetector.go b/backend/internal/builder/domDropDetector.go index 3643038c1..473937a9d 100644 --- a/backend/internal/builder/domDropDetector.go +++ b/backend/internal/builder/domDropDetector.go @@ -4,36 +4,45 @@ import ( . "openreplay/backend/pkg/messages" ) +const DROP_WINDOW = 200 //ms +const CRITICAL_COUNT = 1 // Our login page contains 20. But on crush it removes only roots (1-3 nodes). +// TODO: smart detection (making whole DOM tree would eat all memory) + type domDropDetector struct { removedCount int lastDropTimestamp uint64 } -const DROP_WINDOW = 200 //ms -const CRITICAL_COUNT = 1 // Our login page contains 20. But on crush it removes only roots (1-3 nodes). - -func (dd *domDropDetector) HandleNodeCreation() { +func (dd *domDropDetector) reset() { dd.removedCount = 0 dd.lastDropTimestamp = 0 } -func (dd *domDropDetector) HandleNodeRemoval(ts uint64) { - if dd.lastDropTimestamp+DROP_WINDOW > ts { - dd.removedCount += 1 - } else { - dd.removedCount = 1 +func (dd *domDropDetector) Handle(message Message, _ uint64, timestamp uint64) Message { + switch message.(type) { + case *CreateElementNode, + *CreateTextNode: + dd.removedCount = 0 + dd.lastDropTimestamp = 0 + case *RemoveNode: + if dd.lastDropTimestamp+DROP_WINDOW > timestamp { + dd.removedCount += 1 + } else { + dd.removedCount = 1 + } + dd.lastDropTimestamp = timestamp } - dd.lastDropTimestamp = ts + return nil } -func (dd *domDropDetector) Build() *DOMDrop { - var domDrop *DOMDrop +func (dd *domDropDetector) Build() Message { if dd.removedCount >= CRITICAL_COUNT { - domDrop = &DOMDrop{ + domDrop := &DOMDrop{ Timestamp: dd.lastDropTimestamp, } + dd.reset() + return domDrop } - dd.removedCount = 0 - dd.lastDropTimestamp = 0 - return domDrop + dd.reset() + return nil } diff --git a/backend/internal/builder/memoryIssueFinder.go b/backend/internal/builder/memoryIssueFinder.go index 0d6d71420..2f04343bc 100644 --- a/backend/internal/builder/memoryIssueFinder.go +++ b/backend/internal/builder/memoryIssueFinder.go @@ -19,7 +19,7 @@ type memoryIssueFinder struct { contextString string } -func (f *memoryIssueFinder) Build() *IssueEvent { +func (f *memoryIssueFinder) Build() Message { if f.startTimestamp == 0 { return nil } @@ -37,34 +37,34 @@ func (f *memoryIssueFinder) Build() *IssueEvent { return i } -func (f *memoryIssueFinder) HandleSetPageLocation(msg *SetPageLocation) { - f.contextString = msg.URL -} +func (f *memoryIssueFinder) Handle(message Message, messageID uint64, timestamp uint64) Message { + switch msg := message.(type) { + case *PerformanceTrack: + if f.count < MIN_COUNT { + f.sum += float64(msg.UsedJSHeapSize) + f.count++ + return nil + } + + average := f.sum / f.count + rate := int(math.Round(float64(msg.UsedJSHeapSize) / average * 100)) -func (f *memoryIssueFinder) HandlePerformanceTrack(msg *PerformanceTrack, messageID uint64, timestamp uint64) *IssueEvent { - if f.count < MIN_COUNT { f.sum += float64(msg.UsedJSHeapSize) f.count++ - return nil - } - average := f.sum / f.count - rate := int(math.Round(float64(msg.UsedJSHeapSize) / average * 100)) - - f.sum += float64(msg.UsedJSHeapSize) - f.count++ - - if rate >= MEM_RATE_THRESHOLD { - if f.startTimestamp == 0 { - f.startTimestamp = timestamp - f.startMessageID = messageID + if rate >= MEM_RATE_THRESHOLD { + if f.startTimestamp == 0 { + f.startTimestamp = timestamp + f.startMessageID = messageID + } + if f.rate < rate { + f.rate = rate + } + } else { + return f.Build() } - if f.rate < rate { - f.rate = rate - } - } else { - return f.Build() + case *SetPageLocation: + f.contextString = msg.URL } - return nil } diff --git a/backend/internal/builder/performanceTrackAggrBuilder.go b/backend/internal/builder/performanceTrackAggrBuilder.go index 70b751f55..4396e8a05 100644 --- a/backend/internal/builder/performanceTrackAggrBuilder.go +++ b/backend/internal/builder/performanceTrackAggrBuilder.go @@ -7,100 +7,105 @@ import ( "openreplay/backend/pkg/messages/performance" ) +const AGGREGATION_WINDOW = 2 * 60 * 1000 + type performanceTrackAggrBuilder struct { - performanceTrackAggr *PerformanceTrackAggr - lastTimestamp uint64 - count float64 - sumFrameRate float64 - sumTickRate float64 - sumTotalJSHeapSize float64 - sumUsedJSHeapSize float64 + *PerformanceTrackAggr + lastTimestamp uint64 + count float64 + sumFrameRate float64 + sumTickRate float64 + sumTotalJSHeapSize float64 + sumUsedJSHeapSize float64 } func (b *performanceTrackAggrBuilder) start(timestamp uint64) { - b.performanceTrackAggr = &PerformanceTrackAggr{ + b.PerformanceTrackAggr = &PerformanceTrackAggr{ TimestampStart: timestamp, } b.lastTimestamp = timestamp } -func (b *performanceTrackAggrBuilder) HandlePerformanceTrack(msg *PerformanceTrack, timestamp uint64) *PerformanceTrackAggr { - if msg.Frames == -1 || msg.Ticks == -1 || !b.HasInstance() { - performanceTrackAggr := b.Build() - b.start(timestamp) - return performanceTrackAggr - } - - dt := performance.TimeDiff(timestamp, b.lastTimestamp) - if dt == 0 { - return nil // TODO: handle error - } - - frameRate := performance.FrameRate(msg.Frames, dt) - tickRate := performance.TickRate(msg.Ticks, dt) - - fps := uint64(math.Round(frameRate)) - cpu := performance.CPURateFromTickRate(tickRate) - if fps < b.performanceTrackAggr.MinFPS || b.performanceTrackAggr.MinFPS == 0 { - b.performanceTrackAggr.MinFPS = fps - } - if fps > b.performanceTrackAggr.MaxFPS { - b.performanceTrackAggr.MaxFPS = fps - } - if cpu < b.performanceTrackAggr.MinCPU || b.performanceTrackAggr.MinCPU == 0 { - b.performanceTrackAggr.MinCPU = cpu - } - if cpu > b.performanceTrackAggr.MaxCPU { - b.performanceTrackAggr.MaxCPU = cpu - } - if msg.TotalJSHeapSize < b.performanceTrackAggr.MinTotalJSHeapSize || b.performanceTrackAggr.MinTotalJSHeapSize == 0 { - b.performanceTrackAggr.MinTotalJSHeapSize = msg.TotalJSHeapSize - } - if msg.TotalJSHeapSize > b.performanceTrackAggr.MaxTotalJSHeapSize { - b.performanceTrackAggr.MaxTotalJSHeapSize = msg.TotalJSHeapSize - } - if msg.UsedJSHeapSize < b.performanceTrackAggr.MinUsedJSHeapSize || b.performanceTrackAggr.MinUsedJSHeapSize == 0 { - b.performanceTrackAggr.MinUsedJSHeapSize = msg.UsedJSHeapSize - } - if msg.UsedJSHeapSize > b.performanceTrackAggr.MaxUsedJSHeapSize { - b.performanceTrackAggr.MaxUsedJSHeapSize = msg.UsedJSHeapSize - } - b.sumFrameRate += frameRate - b.sumTickRate += tickRate - b.sumTotalJSHeapSize += float64(msg.TotalJSHeapSize) - b.sumUsedJSHeapSize += float64(msg.UsedJSHeapSize) - b.count += 1 - b.lastTimestamp = timestamp - return nil -} - -func (b *performanceTrackAggrBuilder) HasInstance() bool { - return b.performanceTrackAggr != nil -} - -func (b *performanceTrackAggrBuilder) GetStartTimestamp() uint64 { - if b.performanceTrackAggr == nil { - return 0 - } - return b.performanceTrackAggr.TimestampStart -} - -func (b *performanceTrackAggrBuilder) Build() *PerformanceTrackAggr { - var performanceTrackAggr *PerformanceTrackAggr - if b.HasInstance() && b.GetStartTimestamp() != b.lastTimestamp && b.count != 0 { - performanceTrackAggr = b.performanceTrackAggr - performanceTrackAggr.TimestampEnd = b.lastTimestamp - performanceTrackAggr.AvgFPS = uint64(math.Round(b.sumFrameRate / b.count)) - performanceTrackAggr.AvgCPU = 100 - uint64(math.Round(b.sumTickRate*100/b.count)) - performanceTrackAggr.AvgTotalJSHeapSize = uint64(math.Round(b.sumTotalJSHeapSize / b.count)) - performanceTrackAggr.AvgUsedJSHeapSize = uint64(math.Round(b.sumUsedJSHeapSize / b.count)) - } - b.performanceTrackAggr = nil +func (b *performanceTrackAggrBuilder) reset() { + b.PerformanceTrackAggr = nil b.count = 0 b.sumFrameRate = 0 b.sumTickRate = 0 b.sumTotalJSHeapSize = 0 b.sumUsedJSHeapSize = 0 b.lastTimestamp = 0 - return performanceTrackAggr +} + +func (b *performanceTrackAggrBuilder) Handle(message Message, _ uint64, timestamp uint64) Message { + switch msg := message.(type) { + case *PerformanceTrack: + if b.PerformanceTrackAggr == nil || msg.Frames == -1 || msg.Ticks == -1 { + pta := b.Build() + b.start(timestamp) + return pta + } + + dt := performance.TimeDiff(timestamp, b.lastTimestamp) + if dt == 0 { + return nil // shouldn't happen + } + + frameRate := performance.FrameRate(msg.Frames, dt) + tickRate := performance.TickRate(msg.Ticks, dt) + + fps := uint64(math.Round(frameRate)) + cpu := performance.CPURateFromTickRate(tickRate) + if fps < b.MinFPS || b.MinFPS == 0 { + b.MinFPS = fps + } + if fps > b.MaxFPS { + b.MaxFPS = fps + } + if cpu < b.MinCPU || b.MinCPU == 0 { + b.MinCPU = cpu + } + if cpu > b.MaxCPU { + b.MaxCPU = cpu + } + if msg.TotalJSHeapSize < b.MinTotalJSHeapSize || b.MinTotalJSHeapSize == 0 { + b.MinTotalJSHeapSize = msg.TotalJSHeapSize + } + if msg.TotalJSHeapSize > b.MaxTotalJSHeapSize { + b.MaxTotalJSHeapSize = msg.TotalJSHeapSize + } + if msg.UsedJSHeapSize < b.MinUsedJSHeapSize || b.MinUsedJSHeapSize == 0 { + b.MinUsedJSHeapSize = msg.UsedJSHeapSize + } + if msg.UsedJSHeapSize > b.MaxUsedJSHeapSize { + b.MaxUsedJSHeapSize = msg.UsedJSHeapSize + } + b.sumFrameRate += frameRate + b.sumTickRate += tickRate + b.sumTotalJSHeapSize += float64(msg.TotalJSHeapSize) + b.sumUsedJSHeapSize += float64(msg.UsedJSHeapSize) + b.count += 1 + b.lastTimestamp = timestamp + } + if b.PerformanceTrackAggr != nil && + timestamp-b.PerformanceTrackAggr.TimestampStart >= AGGREGATION_WINDOW { + return b.Build() + } + return nil +} + +func (b *performanceTrackAggrBuilder) Build() Message { + if b.PerformanceTrackAggr == nil { + return nil + } + if b.count != 0 && b.PerformanceTrackAggr.TimestampStart < b.lastTimestamp { // the last one shouldn't happen + b.PerformanceTrackAggr.TimestampEnd = b.lastTimestamp + b.PerformanceTrackAggr.AvgFPS = uint64(math.Round(b.sumFrameRate / b.count)) + b.PerformanceTrackAggr.AvgCPU = 100 - uint64(math.Round(b.sumTickRate*100/b.count)) + b.PerformanceTrackAggr.AvgTotalJSHeapSize = uint64(math.Round(b.sumTotalJSHeapSize / b.count)) + b.PerformanceTrackAggr.AvgUsedJSHeapSize = uint64(math.Round(b.sumUsedJSHeapSize / b.count)) + b.reset() + return b.PerformanceTrackAggr + } + b.reset() + return nil } diff --git a/backend/pkg/intervals/intervals.go b/backend/pkg/intervals/intervals.go index 2ce13ed5e..649ceca1a 100644 --- a/backend/pkg/intervals/intervals.go +++ b/backend/pkg/intervals/intervals.go @@ -5,7 +5,6 @@ const HEARTBEAT_INTERVAL = 2 * 60 * 1000 // максимальный const INTEGRATIONS_REQUEST_INTERVAL = 1 * 60 * 1000 // интеграции const EVENTS_PAGE_EVENT_TIMEOUT = 2 * 60 * 1000 // таймаут пейдж ивента const EVENTS_INPUT_EVENT_TIMEOUT = 2 * 60 * 1000 // -const EVENTS_PERFORMANCE_AGGREGATION_TIMEOUT = 2 * 60 * 1000 const EVENTS_SESSION_END_TIMEOUT = HEARTBEAT_INTERVAL + 30*1000 const EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS = HEARTBEAT_INTERVAL + 3*60*1000 const EVENTS_BACK_COMMIT_GAP = EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS + 1*60*1000 // для бэк коммита From 6ab6d342c035a65cde98fbe73b5509db03e8a344 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Sat, 7 May 2022 22:16:15 +0200 Subject: [PATCH 08/25] chore(backend-heuristics/db): remove redundant --- backend/cmd/db/main.go | 2 +- backend/internal/heuristics/heuristics.go | 40 +++---------------- .../internal/heuristics/inputEventBuilder.go | 2 +- .../internal/heuristics/pageEventBuilder.go | 2 +- backend/internal/heuristics/session.go | 10 +++-- 5 files changed, 15 insertions(+), 41 deletions(-) diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index 962057213..f6cd481a7 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -60,7 +60,7 @@ func main() { } // Handle heuristics and save to temporary queue in memory - heurFinder.HandleMessage(session, msg) + heurFinder.HandleMessage(sessionID, msg) // Process saved heuristics messages as usual messages above in the code heurFinder.IterateSessionReadyMessages(sessionID, func(msg messages.Message) { diff --git a/backend/internal/heuristics/heuristics.go b/backend/internal/heuristics/heuristics.go index 677574951..c55ad33b3 100644 --- a/backend/internal/heuristics/heuristics.go +++ b/backend/internal/heuristics/heuristics.go @@ -1,43 +1,26 @@ package heuristics import ( - . "openreplay/backend/pkg/db/types" . "openreplay/backend/pkg/messages" ) -type MessageHandler interface { - HandleMessage(Message) -} -type ReadyMessagesIterator interface { - IterateReadyMessages(func(Message)) -} - -type Handler interface { - MessageHandler - ReadyMessagesIterator -} - type mainHandler map[uint64]*sessHandler func NewHandler() mainHandler { return make(mainHandler) } -func (m mainHandler) getSessHandler(session *Session) *sessHandler { - if session == nil { - //AAAA - return nil - } - s := m[session.SessionID] +func (m mainHandler) getSessHandler(sessionID uint64) *sessHandler { + s := m[sessionID] if s == nil { - s = newSessHandler(session) - m[session.SessionID] = s + s = newSessHandler() + m[sessionID] = s } return s } -func (m mainHandler) HandleMessage(session *Session, msg Message) { - s := m.getSessHandler(session) +func (m mainHandler) HandleMessage(sessionID uint64, msg Message) { + s := m.getSessHandler(sessionID) s.HandleMessage(msg) } @@ -51,14 +34,3 @@ func (m mainHandler) IterateSessionReadyMessages(sessionID uint64, iter func(msg delete(m, sessionID) } } - -func (m mainHandler) IterateReadyMessages(iter func(sessionID uint64, msg Message)) { - for sessionID, s := range m { - s.IterateReadyMessages(func(msg Message) { - iter(sessionID, msg) - }) - if s.IsEnded() { - delete(m, sessionID) - } - } -} diff --git a/backend/internal/heuristics/inputEventBuilder.go b/backend/internal/heuristics/inputEventBuilder.go index ce1b710ca..624e15e47 100644 --- a/backend/internal/heuristics/inputEventBuilder.go +++ b/backend/internal/heuristics/inputEventBuilder.go @@ -1,4 +1,4 @@ -package builder +package heuristics import ( . "openreplay/backend/pkg/messages" diff --git a/backend/internal/heuristics/pageEventBuilder.go b/backend/internal/heuristics/pageEventBuilder.go index 2b0665894..96a1b287e 100644 --- a/backend/internal/heuristics/pageEventBuilder.go +++ b/backend/internal/heuristics/pageEventBuilder.go @@ -1,4 +1,4 @@ -package builder +package heuristics import ( . "openreplay/backend/pkg/messages" diff --git a/backend/internal/heuristics/session.go b/backend/internal/heuristics/session.go index 3946bf918..a49db948b 100644 --- a/backend/internal/heuristics/session.go +++ b/backend/internal/heuristics/session.go @@ -1,19 +1,21 @@ package heuristics import ( - . "openreplay/backend/pkg/db/types" . "openreplay/backend/pkg/messages" ) +type Handler interface { + HandleMessage(Message) + IterateReadyMessages(func(Message)) +} + type sessHandler struct { - session *Session handlers []Handler ended bool } -func newSessHandler(session *Session) *sessHandler { +func newSessHandler() *sessHandler { return &sessHandler{ - session: session, handlers: []Handler{ new(clickrage), new(performanceAggregator), From ca9d76624b505ecb31b43890512e21cf3ff33a08 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 9 May 2022 16:51:10 +0200 Subject: [PATCH 09/25] feat(backend/heuristics): message handlers refactoring --- backend/cmd/heuristics/main.go | 30 ++++++++- backend/internal/builder/builder.go | 19 ++---- backend/internal/builder/builderMap.go | 31 ++++++---- .../internal/handlers/custom/customHandler.go | 16 +++++ .../ios/appNotResponding.go} | 51 ++++++++++------ .../ios/clickRage.go} | 61 +++++++++++-------- .../ios/performanceAggregator.go} | 59 ++++++++++-------- backend/internal/handlers/messageProcessor.go | 11 ++++ .../readyMessageStore.go | 8 +-- .../web/clickRage.go} | 12 ++-- .../web/cpuIssue.go} | 10 +-- .../web/deadClick.go} | 14 +++-- .../web/domDrop.go} | 4 +- .../web/memoryIssue.go} | 10 +-- .../web/performanceAggregator.go} | 12 ++-- backend/internal/heuristics/session.go | 6 +- 16 files changed, 223 insertions(+), 131 deletions(-) create mode 100644 backend/internal/handlers/custom/customHandler.go rename backend/internal/{heuristics/anr.go => handlers/ios/appNotResponding.go} (66%) rename backend/internal/{heuristics/clickrage.go => handlers/ios/clickRage.go} (60%) rename backend/internal/{heuristics/performance.go => handlers/ios/performanceAggregator.go} (78%) create mode 100644 backend/internal/handlers/messageProcessor.go rename backend/internal/{heuristics => handlers}/readyMessageStore.go (51%) rename backend/internal/{builder/clikRageDetector.go => handlers/web/clickRage.go} (85%) rename backend/internal/{builder/cpuIssueFinder.go => handlers/web/cpuIssue.go} (86%) rename backend/internal/{builder/deadClickDetector.go => handlers/web/deadClick.go} (82%) rename backend/internal/{builder/domDropDetector.go => handlers/web/domDrop.go} (94%) rename backend/internal/{builder/memoryIssueFinder.go => handlers/web/memoryIssue.go} (83%) rename backend/internal/{builder/performanceTrackAggrBuilder.go => handlers/web/performanceAggregator.go} (89%) diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index 2778685d3..f5e3b675f 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -4,6 +4,10 @@ import ( "log" "openreplay/backend/internal/builder" "openreplay/backend/internal/config/ender" + "openreplay/backend/internal/handlers" + "openreplay/backend/internal/handlers/custom" + "openreplay/backend/internal/handlers/ios" + "openreplay/backend/internal/handlers/web" "openreplay/backend/pkg/intervals" logger "openreplay/backend/pkg/log" "openreplay/backend/pkg/messages" @@ -18,10 +22,32 @@ import ( func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) + // Load service configuration cfg := ender.New() - builderMap := builder.NewBuilderMap() + // Declare message handlers we want to apply for each incoming message + msgHandlers := []handlers.MessageProcessor{ + // web handlers + &web.ClickRageDetector{}, + &web.CpuIssueDetector{}, + &web.DeadClickDetector{}, + &web.MemoryIssueDetector{}, + &web.PerformanceAggregator{}, + // iOS handlers + &ios.AppNotResponding{}, + &ios.ClickRageDetector{}, + &ios.PerformanceAggregator{}, + // Other handlers (you can add your custom handlers here) + &custom.CustomHandler{}, + } + + // Create handler's aggregator + builderMap := builder.NewBuilderMap(msgHandlers...) + + // Init logger statsLogger := logger.NewQueueStats(cfg.LoggerTimeout) + + // Init producer and consumer for data bus producer := queue.NewProducer() consumer := queue.NewMessageConsumer( cfg.GroupEvents, @@ -36,7 +62,7 @@ func main() { false, ) - log.Printf("Ender service started\n") + log.Printf("Heuristics service started\n") sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) diff --git a/backend/internal/builder/builder.go b/backend/internal/builder/builder.go index 7e062e6b2..c35457d62 100644 --- a/backend/internal/builder/builder.go +++ b/backend/internal/builder/builder.go @@ -1,31 +1,20 @@ package builder import ( + "openreplay/backend/internal/handlers" "openreplay/backend/pkg/intervals" . "openreplay/backend/pkg/messages" ) -type messageProcessor interface { - Handle(message Message, messageID uint64, timestamp uint64) Message - Build() Message -} - type builder struct { readyMsgs []Message timestamp uint64 - processors []messageProcessor + processors []handlers.MessageProcessor } -func NewBuilder() *builder { +func NewBuilder(handlers ...handlers.MessageProcessor) *builder { return &builder{ - processors: []messageProcessor{ - &performanceTrackAggrBuilder{}, - &cpuIssueFinder{}, - &memoryIssueFinder{}, - // &domDropDetector{}, - &clickRageDetector{}, - &deadClickDetector{}, - }, + processors: handlers, } } diff --git a/backend/internal/builder/builderMap.go b/backend/internal/builder/builderMap.go index 6caf18e4f..5bc01e78d 100644 --- a/backend/internal/builder/builderMap.go +++ b/backend/internal/builder/builderMap.go @@ -1,37 +1,44 @@ package builder import ( + "openreplay/backend/internal/handlers" . "openreplay/backend/pkg/messages" ) -type builderMap map[uint64]*builder - -func NewBuilderMap() builderMap { - return make(builderMap) +type builderMap struct { + handlers []handlers.MessageProcessor + sessions map[uint64]*builder } -func (m builderMap) GetBuilder(sessionID uint64) *builder { - b := m[sessionID] +func NewBuilderMap(handlers ...handlers.MessageProcessor) *builderMap { + return &builderMap{ + handlers: handlers, + sessions: make(map[uint64]*builder), + } +} + +func (m *builderMap) GetBuilder(sessionID uint64) *builder { + b := m.sessions[sessionID] if b == nil { - b = NewBuilder() - m[sessionID] = b + b = NewBuilder(m.handlers...) + m.sessions[sessionID] = b } return b } -func (m builderMap) HandleMessage(sessionID uint64, msg Message, messageID uint64) { +func (m *builderMap) HandleMessage(sessionID uint64, msg Message, messageID uint64) { b := m.GetBuilder(sessionID) b.handleMessage(msg, messageID) } -func (m builderMap) IterateReadyMessages(operatingTs int64, iter func(sessionID uint64, msg Message)) { - for sessionID, b := range m { +func (m *builderMap) IterateReadyMessages(operatingTs int64, iter func(sessionID uint64, msg Message)) { + for sessionID, b := range m.sessions { sessionEnded := b.checkTimeouts(operatingTs) b.iterateReadyMessage(func(msg Message) { iter(sessionID, msg) }) if sessionEnded { - delete(m, sessionID) + delete(m.sessions, sessionID) } } } diff --git a/backend/internal/handlers/custom/customHandler.go b/backend/internal/handlers/custom/customHandler.go new file mode 100644 index 000000000..9b191189e --- /dev/null +++ b/backend/internal/handlers/custom/customHandler.go @@ -0,0 +1,16 @@ +package custom + +import . "openreplay/backend/pkg/messages" + +type CustomHandler struct { + lastTimestamp uint64 +} + +func (h *CustomHandler) Handle(message Message, messageID uint64, timestamp uint64) Message { + h.lastTimestamp = timestamp + return nil +} + +func (h *CustomHandler) Build() Message { + return nil +} diff --git a/backend/internal/heuristics/anr.go b/backend/internal/handlers/ios/appNotResponding.go similarity index 66% rename from backend/internal/heuristics/anr.go rename to backend/internal/handlers/ios/appNotResponding.go index 7cec8fc97..1241648db 100644 --- a/backend/internal/heuristics/anr.go +++ b/backend/internal/handlers/ios/appNotResponding.go @@ -1,34 +1,23 @@ -package heuristics +package ios import ( + "openreplay/backend/internal/handlers" . "openreplay/backend/pkg/messages" ) +// app is not responding detector + const MIN_TIME_AFTER_LAST_HEARTBEAT = 60 * 1000 -type anr struct { - readyMessageStore +type AppNotResponding struct { + handlers.ReadyMessageStore lastLabel string lastHeartbeatTimestamp uint64 lastHeartbeatIndex uint64 } -func (h *anr) buildIf(timestamp uint64) { - if h.lastHeartbeatTimestamp != 0 && h.lastHeartbeatTimestamp+MIN_TIME_AFTER_LAST_HEARTBEAT <= timestamp { - m := &IOSIssueEvent{ - Type: "anr", - ContextString: h.lastLabel, - } - m.Timestamp = h.lastHeartbeatTimestamp - m.Index = h.lastHeartbeatIndex // Associated Index/ MessageID ? - h.append(m) - h.lastHeartbeatTimestamp = 0 - h.lastHeartbeatIndex = 0 - } -} - -func (h *anr) HandleMessage(msg Message) { - switch m := msg.(type) { +func (h *AppNotResponding) Handle(message Message, messageID uint64, timestamp uint64) Message { + switch m := message.(type) { case *IOSClickEvent: h.buildIf(m.Timestamp) h.lastLabel = m.Label @@ -46,4 +35,28 @@ func (h *anr) HandleMessage(msg Message) { case *IOSSessionEnd: h.buildIf(m.Timestamp) } + return nil +} + +func (h *AppNotResponding) Build() Message { + //TODO implement me + panic("implement me") +} + +func (h *AppNotResponding) buildIf(timestamp uint64) { + if h.lastHeartbeatTimestamp != 0 && h.lastHeartbeatTimestamp+MIN_TIME_AFTER_LAST_HEARTBEAT <= timestamp { + m := &IOSIssueEvent{ + Type: "anr", + ContextString: h.lastLabel, + } + m.Timestamp = h.lastHeartbeatTimestamp + m.Index = h.lastHeartbeatIndex // Associated Index/ MessageID ? + h.Append(m) + h.lastHeartbeatTimestamp = 0 + h.lastHeartbeatIndex = 0 + } +} + +func (h *AppNotResponding) HandleMessage(msg Message) { + // TODO: delete it } diff --git a/backend/internal/heuristics/clickrage.go b/backend/internal/handlers/ios/clickRage.go similarity index 60% rename from backend/internal/heuristics/clickrage.go rename to backend/internal/handlers/ios/clickRage.go index 4d19bf92e..2707c04d5 100644 --- a/backend/internal/heuristics/clickrage.go +++ b/backend/internal/handlers/ios/clickRage.go @@ -1,14 +1,17 @@ -package heuristics +package ios import ( + "openreplay/backend/internal/handlers" + "openreplay/backend/internal/handlers/web" . "openreplay/backend/pkg/messages" ) const CLICK_TIME_DIFF = 200 -const MIN_CLICKS_IN_A_ROW = 3 -type clickrage struct { - readyMessageStore +//const MIN_CLICKS_IN_A_ROW = 3 + +type ClickRageDetector struct { + handlers.ReadyMessageStore lastTimestamp uint64 lastLabel string firstInARawTimestamp uint64 @@ -16,30 +19,13 @@ type clickrage struct { countsInARow int } -func (h *clickrage) build() { - if h.countsInARow >= MIN_CLICKS_IN_A_ROW { - m := &IOSIssueEvent{ - Type: "click_rage", - ContextString: h.lastLabel, - } - m.Timestamp = h.firstInARawTimestamp - m.Index = h.firstInARawSeqIndex // Associated Index/ MessageID ? - h.append(m) - } - h.lastTimestamp = 0 - h.lastLabel = "" - h.firstInARawTimestamp = 0 - h.firstInARawSeqIndex = 0 - h.countsInARow = 0 -} - -func (h *clickrage) HandleMessage(msg Message) { - switch m := msg.(type) { +func (h *ClickRageDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { + switch m := message.(type) { case *IOSClickEvent: if h.lastTimestamp+CLICK_TIME_DIFF < m.Timestamp && h.lastLabel == m.Label { h.lastTimestamp = m.Timestamp h.countsInARow += 1 - return + return nil } h.build() if m.Label != "" { @@ -52,4 +38,31 @@ func (h *clickrage) HandleMessage(msg Message) { case *IOSSessionEnd: h.build() } + return nil +} + +func (h *ClickRageDetector) Build() Message { + //TODO implement me + panic("implement me") +} + +func (h *ClickRageDetector) build() { + if h.countsInARow >= web.MIN_CLICKS_IN_A_ROW { + m := &IOSIssueEvent{ + Type: "click_rage", + ContextString: h.lastLabel, + } + m.Timestamp = h.firstInARawTimestamp + m.Index = h.firstInARawSeqIndex // Associated Index/ MessageID ? + h.Append(m) + } + h.lastTimestamp = 0 + h.lastLabel = "" + h.firstInARawTimestamp = 0 + h.firstInARawSeqIndex = 0 + h.countsInARow = 0 +} + +func (h *ClickRageDetector) HandleMessage(msg Message) { + // TODO: delete it } diff --git a/backend/internal/heuristics/performance.go b/backend/internal/handlers/ios/performanceAggregator.go similarity index 78% rename from backend/internal/heuristics/performance.go rename to backend/internal/handlers/ios/performanceAggregator.go index c7494a793..1525127b8 100644 --- a/backend/internal/heuristics/performance.go +++ b/backend/internal/handlers/ios/performanceAggregator.go @@ -1,6 +1,7 @@ -package heuristics +package ios import ( + "openreplay/backend/internal/handlers" . "openreplay/backend/pkg/messages" ) @@ -18,8 +19,8 @@ func (va *valueAggregator) aggregate() uint64 { return uint64(va.sum / va.count) } -type performanceAggregator struct { - readyMessageStore +type PerformanceAggregator struct { + handlers.ReadyMessageStore pa *IOSPerformanceAggregated fps valueAggregator cpu valueAggregator @@ -27,30 +28,11 @@ type performanceAggregator struct { battery valueAggregator } -func (h *performanceAggregator) build(timestamp uint64) { - if h.pa == nil { - return - } - h.pa.TimestampEnd = timestamp - h.pa.AvgFPS = h.fps.aggregate() - h.pa.AvgCPU = h.cpu.aggregate() - h.pa.AvgMemory = h.memory.aggregate() - h.pa.AvgBattery = h.battery.aggregate() - - h.append(h.pa) - - h.pa = &IOSPerformanceAggregated{} - for _, agg := range []valueAggregator{h.fps, h.cpu, h.memory, h.battery} { - agg.sum = 0 - agg.count = 0 - } -} - -func (h *performanceAggregator) HandleMessage(msg Message) { +func (h *PerformanceAggregator) Handle(message Message, messageID uint64, timestamp uint64) Message { if h.pa == nil { h.pa = &IOSPerformanceAggregated{} // TODO: struct type in messages } - switch m := msg.(type) { // TODO: All Timestampe messages + switch m := message.(type) { // TODO: All Timestampe messages case *IOSPerformanceEvent: if h.pa.TimestampStart == 0 { h.pa.TimestampStart = m.Timestamp @@ -99,4 +81,33 @@ func (h *performanceAggregator) HandleMessage(msg Message) { case *IOSSessionEnd: h.build(m.Timestamp) } + return nil +} + +func (h *PerformanceAggregator) Build() Message { + //TODO implement me + panic("implement me") +} + +func (h *PerformanceAggregator) build(timestamp uint64) { + if h.pa == nil { + return + } + h.pa.TimestampEnd = timestamp + h.pa.AvgFPS = h.fps.aggregate() + h.pa.AvgCPU = h.cpu.aggregate() + h.pa.AvgMemory = h.memory.aggregate() + h.pa.AvgBattery = h.battery.aggregate() + + h.Append(h.pa) + + h.pa = &IOSPerformanceAggregated{} + for _, agg := range []valueAggregator{h.fps, h.cpu, h.memory, h.battery} { + agg.sum = 0 + agg.count = 0 + } +} + +func (h *PerformanceAggregator) HandleMessage(msg Message) { + // TODO: delete it } diff --git a/backend/internal/handlers/messageProcessor.go b/backend/internal/handlers/messageProcessor.go new file mode 100644 index 000000000..c4235c18b --- /dev/null +++ b/backend/internal/handlers/messageProcessor.go @@ -0,0 +1,11 @@ +package handlers + +import . "openreplay/backend/pkg/messages" + +// Heuristic interface - common interface for user's realisations +// U can create your own message handler and easily connect to heuristics service + +type MessageProcessor interface { + Handle(message Message, messageID uint64, timestamp uint64) Message + Build() Message +} diff --git a/backend/internal/heuristics/readyMessageStore.go b/backend/internal/handlers/readyMessageStore.go similarity index 51% rename from backend/internal/heuristics/readyMessageStore.go rename to backend/internal/handlers/readyMessageStore.go index bbe77585d..c0c386571 100644 --- a/backend/internal/heuristics/readyMessageStore.go +++ b/backend/internal/handlers/readyMessageStore.go @@ -1,18 +1,18 @@ -package heuristics +package handlers import ( . "openreplay/backend/pkg/messages" ) -type readyMessageStore struct { +type ReadyMessageStore struct { store []Message } -func (s *readyMessageStore) append(msg Message) { +func (s *ReadyMessageStore) Append(msg Message) { s.store = append(s.store, msg) } -func (s *readyMessageStore) IterateReadyMessages(cb func(msg Message)) { +func (s *ReadyMessageStore) IterateReadyMessages(cb func(msg Message)) { for _, msg := range s.store { cb(msg) } diff --git a/backend/internal/builder/clikRageDetector.go b/backend/internal/handlers/web/clickRage.go similarity index 85% rename from backend/internal/builder/clikRageDetector.go rename to backend/internal/handlers/web/clickRage.go index 1140027b3..db22a9667 100644 --- a/backend/internal/builder/clikRageDetector.go +++ b/backend/internal/handlers/web/clickRage.go @@ -1,4 +1,4 @@ -package builder +package web import ( "encoding/json" @@ -6,10 +6,12 @@ import ( . "openreplay/backend/pkg/messages" ) +// TODO: Description of click rage detector + const MAX_TIME_DIFF = 300 const MIN_CLICKS_IN_A_ROW = 3 -type clickRageDetector struct { +type ClickRageDetector struct { lastTimestamp uint64 lastLabel string firstInARawTimestamp uint64 @@ -17,7 +19,7 @@ type clickRageDetector struct { countsInARow int } -func (crd *clickRageDetector) reset() { +func (crd *ClickRageDetector) reset() { crd.lastTimestamp = 0 crd.lastLabel = "" crd.firstInARawTimestamp = 0 @@ -25,7 +27,7 @@ func (crd *clickRageDetector) reset() { crd.countsInARow = 0 } -func (crd *clickRageDetector) Build() Message { +func (crd *ClickRageDetector) Build() Message { if crd.countsInARow >= MIN_CLICKS_IN_A_ROW { payload, _ := json.Marshal(struct{ Count int }{crd.countsInARow}) i := &IssueEvent{ @@ -42,7 +44,7 @@ func (crd *clickRageDetector) Build() Message { return nil } -func (crd *clickRageDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { +func (crd *ClickRageDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { switch msg := message.(type) { case *MouseClick: // TODO: check if we it is ok to capture clickrages without the connected CleckEvent in db. diff --git a/backend/internal/builder/cpuIssueFinder.go b/backend/internal/handlers/web/cpuIssue.go similarity index 86% rename from backend/internal/builder/cpuIssueFinder.go rename to backend/internal/handlers/web/cpuIssue.go index feb694a86..5cc12be68 100644 --- a/backend/internal/builder/cpuIssueFinder.go +++ b/backend/internal/handlers/web/cpuIssue.go @@ -1,4 +1,4 @@ -package builder +package web import ( "encoding/json" @@ -7,10 +7,12 @@ import ( "openreplay/backend/pkg/messages/performance" ) +// TODO: Description of cpu issue detector + const CPU_THRESHOLD = 70 // % out of 100 const CPU_MIN_DURATION_TRIGGER = 6 * 1000 -type cpuIssueFinder struct { +type CpuIssueDetector struct { startTimestamp uint64 startMessageID uint64 lastTimestamp uint64 @@ -18,7 +20,7 @@ type cpuIssueFinder struct { contextString string } -func (f *cpuIssueFinder) Build() Message { +func (f *CpuIssueDetector) Build() Message { if f.startTimestamp == 0 { return nil } @@ -47,7 +49,7 @@ func (f *cpuIssueFinder) Build() Message { } } -func (f *cpuIssueFinder) Handle(message Message, messageID uint64, timestamp uint64) Message { +func (f *CpuIssueDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { switch msg := message.(type) { case *PerformanceTrack: dt := performance.TimeDiff(timestamp, f.lastTimestamp) diff --git a/backend/internal/builder/deadClickDetector.go b/backend/internal/handlers/web/deadClick.go similarity index 82% rename from backend/internal/builder/deadClickDetector.go rename to backend/internal/handlers/web/deadClick.go index f83c0bedd..a04da9be9 100644 --- a/backend/internal/builder/deadClickDetector.go +++ b/backend/internal/handlers/web/deadClick.go @@ -1,12 +1,14 @@ -package builder +package web import ( . "openreplay/backend/pkg/messages" ) +// TODO: Description of dead click detector + const CLICK_RELATION_TIME = 1400 -type deadClickDetector struct { +type DeadClickDetector struct { lastTimestamp uint64 lastMouseClick *MouseClick lastClickTimestamp uint64 @@ -14,14 +16,14 @@ type deadClickDetector struct { inputIDSet map[uint64]bool } -func (d *deadClickDetector) reset() { +func (d *DeadClickDetector) reset() { d.inputIDSet = nil d.lastMouseClick = nil d.lastClickTimestamp = 0 d.lastMessageID = 0 } -func (d *deadClickDetector) handleReaction(timestamp uint64) Message { +func (d *DeadClickDetector) handleReaction(timestamp uint64) Message { if d.lastMouseClick == nil || d.lastClickTimestamp+CLICK_RELATION_TIME > timestamp { // riaction is instant d.reset() return nil @@ -36,11 +38,11 @@ func (d *deadClickDetector) handleReaction(timestamp uint64) Message { return i } -func (d *deadClickDetector) Build() Message { +func (d *DeadClickDetector) Build() Message { return d.handleReaction(d.lastTimestamp) } -func (d *deadClickDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { +func (d *DeadClickDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { d.lastTimestamp = timestamp switch msg := message.(type) { case *SetInputTarget: diff --git a/backend/internal/builder/domDropDetector.go b/backend/internal/handlers/web/domDrop.go similarity index 94% rename from backend/internal/builder/domDropDetector.go rename to backend/internal/handlers/web/domDrop.go index 473937a9d..c89fab2c4 100644 --- a/backend/internal/builder/domDropDetector.go +++ b/backend/internal/handlers/web/domDrop.go @@ -1,9 +1,11 @@ -package builder +package web import ( . "openreplay/backend/pkg/messages" ) +// TODO: Description of dom drop detector + const DROP_WINDOW = 200 //ms const CRITICAL_COUNT = 1 // Our login page contains 20. But on crush it removes only roots (1-3 nodes). // TODO: smart detection (making whole DOM tree would eat all memory) diff --git a/backend/internal/builder/memoryIssueFinder.go b/backend/internal/handlers/web/memoryIssue.go similarity index 83% rename from backend/internal/builder/memoryIssueFinder.go rename to backend/internal/handlers/web/memoryIssue.go index 2f04343bc..ac8ca8a14 100644 --- a/backend/internal/builder/memoryIssueFinder.go +++ b/backend/internal/handlers/web/memoryIssue.go @@ -1,4 +1,4 @@ -package builder +package web import ( "encoding/json" @@ -7,10 +7,12 @@ import ( . "openreplay/backend/pkg/messages" ) +// TODO: Description of memory issue detector + const MIN_COUNT = 3 const MEM_RATE_THRESHOLD = 300 // % to average -type memoryIssueFinder struct { +type MemoryIssueDetector struct { startMessageID uint64 startTimestamp uint64 rate int @@ -19,7 +21,7 @@ type memoryIssueFinder struct { contextString string } -func (f *memoryIssueFinder) Build() Message { +func (f *MemoryIssueDetector) Build() Message { if f.startTimestamp == 0 { return nil } @@ -37,7 +39,7 @@ func (f *memoryIssueFinder) Build() Message { return i } -func (f *memoryIssueFinder) Handle(message Message, messageID uint64, timestamp uint64) Message { +func (f *MemoryIssueDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { switch msg := message.(type) { case *PerformanceTrack: if f.count < MIN_COUNT { diff --git a/backend/internal/builder/performanceTrackAggrBuilder.go b/backend/internal/handlers/web/performanceAggregator.go similarity index 89% rename from backend/internal/builder/performanceTrackAggrBuilder.go rename to backend/internal/handlers/web/performanceAggregator.go index 4396e8a05..a7bf79f9f 100644 --- a/backend/internal/builder/performanceTrackAggrBuilder.go +++ b/backend/internal/handlers/web/performanceAggregator.go @@ -1,4 +1,4 @@ -package builder +package web import ( "math" @@ -9,7 +9,7 @@ import ( const AGGREGATION_WINDOW = 2 * 60 * 1000 -type performanceTrackAggrBuilder struct { +type PerformanceAggregator struct { *PerformanceTrackAggr lastTimestamp uint64 count float64 @@ -19,14 +19,14 @@ type performanceTrackAggrBuilder struct { sumUsedJSHeapSize float64 } -func (b *performanceTrackAggrBuilder) start(timestamp uint64) { +func (b *PerformanceAggregator) start(timestamp uint64) { b.PerformanceTrackAggr = &PerformanceTrackAggr{ TimestampStart: timestamp, } b.lastTimestamp = timestamp } -func (b *performanceTrackAggrBuilder) reset() { +func (b *PerformanceAggregator) reset() { b.PerformanceTrackAggr = nil b.count = 0 b.sumFrameRate = 0 @@ -36,7 +36,7 @@ func (b *performanceTrackAggrBuilder) reset() { b.lastTimestamp = 0 } -func (b *performanceTrackAggrBuilder) Handle(message Message, _ uint64, timestamp uint64) Message { +func (b *PerformanceAggregator) Handle(message Message, _ uint64, timestamp uint64) Message { switch msg := message.(type) { case *PerformanceTrack: if b.PerformanceTrackAggr == nil || msg.Frames == -1 || msg.Ticks == -1 { @@ -93,7 +93,7 @@ func (b *performanceTrackAggrBuilder) Handle(message Message, _ uint64, timestam return nil } -func (b *performanceTrackAggrBuilder) Build() Message { +func (b *PerformanceAggregator) Build() Message { if b.PerformanceTrackAggr == nil { return nil } diff --git a/backend/internal/heuristics/session.go b/backend/internal/heuristics/session.go index a49db948b..3c7951750 100644 --- a/backend/internal/heuristics/session.go +++ b/backend/internal/heuristics/session.go @@ -16,11 +16,7 @@ type sessHandler struct { func newSessHandler() *sessHandler { return &sessHandler{ - handlers: []Handler{ - new(clickrage), - new(performanceAggregator), - new(anr), - }, + handlers: []Handler{}, } } From 47007eb9d776db1b1fa0a9d635ead42d9ba7560a Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 10 May 2022 14:11:41 +0200 Subject: [PATCH 10/25] feat(backend/db): prepared db service for refactoring --- backend/cmd/db/main.go | 31 +- backend/cmd/heuristics/main.go | 1 + backend/internal/builder/builder.go | 8 + backend/internal/builder/builderMap.go | 11 + backend/internal/config/db/config.go | 2 + .../custom}/inputEventBuilder.go | 19 +- .../internal/handlers/custom/mainHandler.go | 288 ++++++++++++++++++ .../custom}/pageEventBuilder.go | 22 +- backend/internal/heuristics/heuristics.go | 36 --- backend/internal/heuristics/session.go | 43 --- 10 files changed, 368 insertions(+), 93 deletions(-) rename backend/internal/{heuristics => handlers/custom}/inputEventBuilder.go (80%) create mode 100644 backend/internal/handlers/custom/mainHandler.go rename backend/internal/{heuristics => handlers/custom}/pageEventBuilder.go (83%) delete mode 100644 backend/internal/heuristics/heuristics.go delete mode 100644 backend/internal/heuristics/session.go diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index f6cd481a7..564fcbab5 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -2,9 +2,12 @@ package main import ( "log" + "openreplay/backend/internal/builder" "openreplay/backend/internal/config/db" "openreplay/backend/internal/datasaver" - "openreplay/backend/internal/heuristics" + "openreplay/backend/internal/handlers" + "openreplay/backend/internal/handlers/custom" + "openreplay/backend/pkg/intervals" "time" "os" @@ -28,8 +31,17 @@ func main() { pg := cache.NewPGCache(postgres.NewConn(cfg.Postgres), cfg.ProjectExpirationTimeoutMs) defer pg.Close() + // Declare message handlers we want to apply for each incoming message + msgHandlers := []handlers.MessageProcessor{ + custom.NewMainHandler(), + custom.NewInputEventBuilder(), + custom.NewPageEventBuilder(), + } + + // Create handler's aggregator + builderMap := builder.NewBuilderMap(msgHandlers...) + // Init modules - heurFinder := heuristics.NewHandler() saver := datasaver.New(pg) statsLogger := logger.NewQueueStats(cfg.LoggerTimeout) @@ -42,6 +54,7 @@ func main() { if !postgres.IsPkeyViolation(err) { log.Printf("Message Insertion Error %v, SessionID: %v, Message: %v", err, sessionID, msg) } + // TODO: can we lose data here because of db error? return } @@ -60,10 +73,10 @@ func main() { } // Handle heuristics and save to temporary queue in memory - heurFinder.HandleMessage(sessionID, msg) + builderMap.HandleMessage(sessionID, msg, msg.Meta().Index) // Process saved heuristics messages as usual messages above in the code - heurFinder.IterateSessionReadyMessages(sessionID, func(msg messages.Message) { + builderMap.IterateSessionReadyMessages(sessionID, func(msg messages.Message) { // TODO: DRY code (carefully with the return statement logic) if err := saver.InsertMessage(sessionID, msg); err != nil { if !postgres.IsPkeyViolation(err) { @@ -82,8 +95,9 @@ func main() { consumer := queue.NewMessageConsumer( cfg.GroupDB, []string{ + cfg.TopicRawWeb, // TODO: is it necessary or not? cfg.TopicRawIOS, - cfg.TopicTrigger, + cfg.TopicTrigger, // to receive SessionEnd events }, handler, false, @@ -94,19 +108,22 @@ func main() { sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - tick := time.Tick(cfg.CommitBatchTimeout) + commitTick := time.Tick(cfg.CommitBatchTimeout) + checkTick := time.Tick(intervals.EVENTS_COMMIT_INTERVAL * time.Millisecond) for { select { case sig := <-sigchan: log.Printf("Caught signal %v: terminating\n", sig) consumer.Close() os.Exit(0) - case <-tick: + case <-commitTick: pg.CommitBatches() // TODO?: separate stats & regular messages if err := consumer.Commit(); err != nil { log.Printf("Error on consumer commit: %v", err) } + case <-checkTick: + // checkTimeout default: err := consumer.ConsumeNext() if err != nil { diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index f5e3b675f..ddfd29095 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -54,6 +54,7 @@ func main() { []string{ cfg.TopicRawWeb, cfg.TopicRawIOS, + cfg.TopicTrigger, // to receive SessionEnd events }, func(sessionID uint64, msg messages.Message, meta *types.Meta) { statsLogger.Collect(sessionID, meta) diff --git a/backend/internal/builder/builder.go b/backend/internal/builder/builder.go index c35457d62..a00ad194a 100644 --- a/backend/internal/builder/builder.go +++ b/backend/internal/builder/builder.go @@ -10,6 +10,7 @@ type builder struct { readyMsgs []Message timestamp uint64 processors []handlers.MessageProcessor + ended bool } func NewBuilder(handlers ...handlers.MessageProcessor) *builder { @@ -35,6 +36,13 @@ func (b *builder) handleMessage(message Message, messageID uint64) { return } + if _, isEnd := message.(*IOSSessionEnd); isEnd { + b.ended = true + } + if _, isEnd := message.(*SessionEnd); isEnd { + b.ended = true + } + for _, p := range b.processors { /* If nil is not returned explicitely by Handle, but as the typed nil ("var i *IssueEvent; return i;") diff --git a/backend/internal/builder/builderMap.go b/backend/internal/builder/builderMap.go index 5bc01e78d..6b2c22bec 100644 --- a/backend/internal/builder/builderMap.go +++ b/backend/internal/builder/builderMap.go @@ -42,3 +42,14 @@ func (m *builderMap) IterateReadyMessages(operatingTs int64, iter func(sessionID } } } + +func (m *builderMap) IterateSessionReadyMessages(sessionID uint64, iter func(msg Message)) { + session, ok := m.sessions[sessionID] + if !ok { + return + } + session.iterateReadyMessage(iter) + if session.ended { + delete(m.sessions, sessionID) + } +} diff --git a/backend/internal/config/db/config.go b/backend/internal/config/db/config.go index fb35a199c..e074399dc 100644 --- a/backend/internal/config/db/config.go +++ b/backend/internal/config/db/config.go @@ -10,6 +10,7 @@ type Config struct { ProjectExpirationTimeoutMs int64 LoggerTimeout int GroupDB string + TopicRawWeb string TopicRawIOS string TopicTrigger string CommitBatchTimeout time.Duration @@ -21,6 +22,7 @@ func New() *Config { ProjectExpirationTimeoutMs: 1000 * 60 * 20, LoggerTimeout: env.Int("LOG_QUEUE_STATS_INTERVAL_SEC"), GroupDB: env.String("GROUP_DB"), + TopicRawWeb: env.String("TOPIC_RAW_WEB"), TopicRawIOS: env.String("TOPIC_RAW_IOS"), TopicTrigger: env.String("TOPIC_TRIGGER"), CommitBatchTimeout: 15 * time.Second, diff --git a/backend/internal/heuristics/inputEventBuilder.go b/backend/internal/handlers/custom/inputEventBuilder.go similarity index 80% rename from backend/internal/heuristics/inputEventBuilder.go rename to backend/internal/handlers/custom/inputEventBuilder.go index 624e15e47..770e714af 100644 --- a/backend/internal/heuristics/inputEventBuilder.go +++ b/backend/internal/handlers/custom/inputEventBuilder.go @@ -1,4 +1,4 @@ -package heuristics +package custom import ( . "openreplay/backend/pkg/messages" @@ -12,6 +12,17 @@ type inputEventBuilder struct { inputID uint64 } +func (b *inputEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message { + //TODO implement me + panic("implement me") +} + +func (b *inputEventBuilder) Build() Message { + // b.build() + //TODO implement me + panic("implement me") +} + func NewInputEventBuilder() *inputEventBuilder { ieBuilder := &inputEventBuilder{} ieBuilder.ClearLabels() @@ -25,7 +36,7 @@ func (b *inputEventBuilder) ClearLabels() { func (b *inputEventBuilder) HandleSetInputTarget(msg *SetInputTarget) *InputEvent { var inputEvent *InputEvent if b.inputID != msg.ID { - inputEvent = b.Build() + inputEvent = b.build() b.inputID = msg.ID } b.inputLabels[msg.ID] = msg.Label @@ -35,7 +46,7 @@ func (b *inputEventBuilder) HandleSetInputTarget(msg *SetInputTarget) *InputEven func (b *inputEventBuilder) HandleSetInputValue(msg *SetInputValue, messageID uint64, timestamp uint64) *InputEvent { var inputEvent *InputEvent if b.inputID != msg.ID { - inputEvent = b.Build() + inputEvent = b.build() b.inputID = msg.ID } if b.inputEvent == nil { @@ -63,7 +74,7 @@ func (b *inputEventBuilder) GetTimestamp() uint64 { return b.inputEvent.Timestamp } -func (b *inputEventBuilder) Build() *InputEvent { +func (b *inputEventBuilder) build() *InputEvent { if b.inputEvent == nil { return nil } diff --git a/backend/internal/handlers/custom/mainHandler.go b/backend/internal/handlers/custom/mainHandler.go new file mode 100644 index 000000000..7e653c250 --- /dev/null +++ b/backend/internal/handlers/custom/mainHandler.go @@ -0,0 +1,288 @@ +package custom + +import ( + "net/url" + "openreplay/backend/pkg/intervals" + "strings" + "time" + + . "openreplay/backend/pkg/messages" +) + +func getURLExtention(URL string) string { + u, err := url.Parse(URL) + if err != nil { + return "" + } + i := strings.LastIndex(u.Path, ".") + return u.Path[i+1:] +} + +func getResourceType(initiator string, URL string) string { + switch initiator { + case "xmlhttprequest", "fetch": + return "fetch" + case "img": + return "img" + default: + switch getURLExtention(URL) { + case "css": + return "stylesheet" + case "js": + return "script" + case "png", "gif", "jpg", "jpeg", "svg": + return "img" + case "mp4", "mkv", "ogg", "webm", "avi", "mp3": + return "media" + default: + return "other" + } + } +} + +type builder struct { + readyMsgs []Message + timestamp uint64 + lastProcessedTimestamp int64 + peBuilder *pageEventBuilder + ieBuilder *inputEventBuilder + integrationsWaiting bool + sid uint64 +} + +func (b *builder) Build() Message { + //TODO implement me + panic("implement me") +} + +func NewMainHandler() *builder { + return &builder{ + peBuilder: &pageEventBuilder{}, + ieBuilder: NewInputEventBuilder(), + integrationsWaiting: true, + } +} + +func (b *builder) appendReadyMessage(msg Message) { // interface is never nil even if it holds nil value + b.readyMsgs = append(b.readyMsgs, msg) +} + +func (b *builder) iterateReadyMessage(iter func(msg Message)) { + for _, readyMsg := range b.readyMsgs { + iter(readyMsg) + } + b.readyMsgs = nil +} + +func (b *builder) buildPageEvent() { + if msg := b.peBuilder.Build(); msg != nil { + b.appendReadyMessage(msg) + } +} + +func (b *builder) buildInputEvent() { + if msg := b.ieBuilder.Build(); msg != nil { + b.appendReadyMessage(msg) + } +} + +func (b *builder) Handle(message Message, messageID uint64, timestamp uint64) Message { + b.timestamp = timestamp + b.lastProcessedTimestamp = time.Now().UnixMilli() + + // Might happen before the first timestamp. + switch msg := message.(type) { + case *SessionStart, + *Metadata, + *UserID, + *UserAnonymousID: + b.appendReadyMessage(msg) + case *RawErrorEvent: + b.appendReadyMessage(&ErrorEvent{ + MessageID: messageID, + Timestamp: msg.Timestamp, + Source: msg.Source, + Name: msg.Name, + Message: msg.Message, + Payload: msg.Payload, + }) + } + if b.timestamp == 0 { + return nil + } + switch msg := message.(type) { + case *SetPageLocation: + if msg.NavigationStart == 0 { + b.appendReadyMessage(&PageEvent{ + URL: msg.URL, + Referrer: msg.Referrer, + Loaded: false, + MessageID: messageID, + Timestamp: b.timestamp, + }) + } else { + b.buildPageEvent() + b.buildInputEvent() + b.ieBuilder.ClearLabels() + b.peBuilder.HandleSetPageLocation(msg, messageID, b.timestamp) + // TODO: what to do with this code? + //b.miFinder.HandleSetPageLocation(msg) + //b.ciFinder.HandleSetPageLocation(msg) + } + case *PageLoadTiming: + if rm := b.peBuilder.HandlePageLoadTiming(msg); rm != nil { + b.appendReadyMessage(rm) + } + case *PageRenderTiming: + if rm := b.peBuilder.HandlePageRenderTiming(msg); rm != nil { + b.appendReadyMessage(rm) + } + case *PerformanceTrack: + // TODO: what to do with this code? + //if rm := b.ptaBuilder.HandlePerformanceTrack(msg, b.timestamp); rm != nil { + // b.appendReadyMessage(rm) + //} + //if rm := b.ciFinder.HandlePerformanceTrack(msg, messageID, b.timestamp); rm != nil { + // b.appendReadyMessage(rm) + //} + //if rm := b.miFinder.HandlePerformanceTrack(msg, messageID, b.timestamp); rm != nil { + // b.appendReadyMessage(rm) + //} + case *SetInputTarget: + if rm := b.ieBuilder.HandleSetInputTarget(msg); rm != nil { + b.appendReadyMessage(rm) + } + case *SetInputValue: + if rm := b.ieBuilder.HandleSetInputValue(msg, messageID, b.timestamp); rm != nil { + b.appendReadyMessage(rm) + } + case *MouseClick: + b.buildInputEvent() + // TODO: what to do with this code? + //if rm := b.crDetector.HandleMouseClick(msg, messageID, b.timestamp); rm != nil { + // b.appendReadyMessage(rm) + //} + if msg.Label != "" { + b.appendReadyMessage(&ClickEvent{ + MessageID: messageID, + Label: msg.Label, + HesitationTime: msg.HesitationTime, + Timestamp: b.timestamp, + Selector: msg.Selector, + }) + } + case *JSException: + b.appendReadyMessage(&ErrorEvent{ + MessageID: messageID, + Timestamp: b.timestamp, + Source: "js_exception", + Name: msg.Name, + Message: msg.Message, + Payload: msg.Payload, + }) + case *ResourceTiming: + tp := getResourceType(msg.Initiator, msg.URL) + success := msg.Duration != 0 + b.appendReadyMessage(&ResourceEvent{ + MessageID: messageID, + Timestamp: msg.Timestamp, + Duration: msg.Duration, + TTFB: msg.TTFB, + HeaderSize: msg.HeaderSize, + EncodedBodySize: msg.EncodedBodySize, + DecodedBodySize: msg.DecodedBodySize, + URL: msg.URL, + Type: tp, + Success: success, + }) + if !success { + issueType := "missing_resource" + if tp == "fetch" { + issueType = "bad_request" + } + b.appendReadyMessage(&IssueEvent{ + Type: issueType, + MessageID: messageID, + Timestamp: msg.Timestamp, + ContextString: msg.URL, + }) + } + case *RawCustomEvent: + b.appendReadyMessage(&CustomEvent{ + MessageID: messageID, + Timestamp: b.timestamp, + Name: msg.Name, + Payload: msg.Payload, + }) + case *CustomIssue: + b.appendReadyMessage(&IssueEvent{ + Type: "custom", + Timestamp: b.timestamp, + MessageID: messageID, + ContextString: msg.Name, + Payload: msg.Payload, + }) + case *Fetch: + b.appendReadyMessage(&FetchEvent{ + MessageID: messageID, + Timestamp: msg.Timestamp, + Method: msg.Method, + URL: msg.URL, + Request: msg.Request, + Response: msg.Response, + Status: msg.Status, + Duration: msg.Duration, + }) + if msg.Status >= 400 { + b.appendReadyMessage(&IssueEvent{ + Type: "bad_request", + MessageID: messageID, + Timestamp: msg.Timestamp, + ContextString: msg.URL, + }) + } + case *GraphQL: + b.appendReadyMessage(&GraphQLEvent{ + MessageID: messageID, + Timestamp: b.timestamp, + OperationKind: msg.OperationKind, + OperationName: msg.OperationName, + Variables: msg.Variables, + Response: msg.Response, + }) + case *StateAction: + b.appendReadyMessage(&StateActionEvent{ + MessageID: messageID, + Timestamp: b.timestamp, + Type: msg.Type, + }) + // TODO: what to do with this code? + //case *CreateElementNode, *CreateTextNode: + // b.ddDetector.HandleNodeCreation() + //case *RemoveNode: + // b.ddDetector.HandleNodeRemoval(b.timestamp) + //case *CreateDocument: + // if rm := b.ddDetector.Build(); rm != nil { + // b.appendReadyMessage(rm) + // } + } + // TODO: what to do with this code? + //if rm := b.dcDetector.HandleMessage(message, messageID, b.timestamp); rm != nil { + // b.appendReadyMessage(rm) + //} + return nil +} + +func (b *builder) checkTimeouts(ts int64) bool { + if b.timestamp == 0 { + return false // There was no timestamp events yet + } + + if b.peBuilder.HasInstance() && int64(b.peBuilder.GetTimestamp())+intervals.EVENTS_PAGE_EVENT_TIMEOUT < ts { + b.buildPageEvent() + } + if b.ieBuilder.HasInstance() && int64(b.ieBuilder.GetTimestamp())+intervals.EVENTS_INPUT_EVENT_TIMEOUT < ts { + b.buildInputEvent() + } + return false +} diff --git a/backend/internal/heuristics/pageEventBuilder.go b/backend/internal/handlers/custom/pageEventBuilder.go similarity index 83% rename from backend/internal/heuristics/pageEventBuilder.go rename to backend/internal/handlers/custom/pageEventBuilder.go index 96a1b287e..765fd31a2 100644 --- a/backend/internal/heuristics/pageEventBuilder.go +++ b/backend/internal/handlers/custom/pageEventBuilder.go @@ -1,4 +1,4 @@ -package heuristics +package custom import ( . "openreplay/backend/pkg/messages" @@ -9,9 +9,25 @@ type pageEventBuilder struct { firstTimingHandled bool } +func (b *pageEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message { + //TODO implement me + panic("implement me") +} + +func (b *pageEventBuilder) Build() Message { + // b.build() + //TODO implement me + panic("implement me") +} + +func NewPageEventBuilder() *pageEventBuilder { + ieBuilder := &pageEventBuilder{} + return ieBuilder +} + func (b *pageEventBuilder) buildIfTimingsComplete() *PageEvent { if b.firstTimingHandled { - return b.Build() + return b.build() } b.firstTimingHandled = true return nil @@ -83,7 +99,7 @@ func (b *pageEventBuilder) GetTimestamp() uint64 { return b.pageEvent.Timestamp } -func (b *pageEventBuilder) Build() *PageEvent { +func (b *pageEventBuilder) build() *PageEvent { pageEvent := b.pageEvent b.pageEvent = nil b.firstTimingHandled = false diff --git a/backend/internal/heuristics/heuristics.go b/backend/internal/heuristics/heuristics.go deleted file mode 100644 index c55ad33b3..000000000 --- a/backend/internal/heuristics/heuristics.go +++ /dev/null @@ -1,36 +0,0 @@ -package heuristics - -import ( - . "openreplay/backend/pkg/messages" -) - -type mainHandler map[uint64]*sessHandler - -func NewHandler() mainHandler { - return make(mainHandler) -} - -func (m mainHandler) getSessHandler(sessionID uint64) *sessHandler { - s := m[sessionID] - if s == nil { - s = newSessHandler() - m[sessionID] = s - } - return s -} - -func (m mainHandler) HandleMessage(sessionID uint64, msg Message) { - s := m.getSessHandler(sessionID) - s.HandleMessage(msg) -} - -func (m mainHandler) IterateSessionReadyMessages(sessionID uint64, iter func(msg Message)) { - s, ok := m[sessionID] - if !ok { - return - } - s.IterateReadyMessages(iter) - if s.IsEnded() { - delete(m, sessionID) - } -} diff --git a/backend/internal/heuristics/session.go b/backend/internal/heuristics/session.go deleted file mode 100644 index 3c7951750..000000000 --- a/backend/internal/heuristics/session.go +++ /dev/null @@ -1,43 +0,0 @@ -package heuristics - -import ( - . "openreplay/backend/pkg/messages" -) - -type Handler interface { - HandleMessage(Message) - IterateReadyMessages(func(Message)) -} - -type sessHandler struct { - handlers []Handler - ended bool -} - -func newSessHandler() *sessHandler { - return &sessHandler{ - handlers: []Handler{}, - } -} - -func (s *sessHandler) HandleMessage(msg Message) { - for _, h := range s.handlers { - h.HandleMessage(msg) - } - if _, isEnd := msg.(*IOSSessionEnd); isEnd { - s.ended = true - } - if _, isEnd := msg.(*SessionEnd); isEnd { - s.ended = true - } -} - -func (s *sessHandler) IterateReadyMessages(cb func(msg Message)) { - for _, h := range s.handlers { - h.IterateReadyMessages(cb) - } -} - -func (s *sessHandler) IsEnded() bool { - return s.ended -} From 26e23d594f31079e8c991768407ee67c88240aca Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 10 May 2022 15:40:55 +0200 Subject: [PATCH 11/25] feat(backend/handlers): refactored web and ios message handlers --- .../internal/handlers/ios/appNotResponding.go | 40 +++++++++++-------- backend/internal/handlers/ios/clickRage.go | 34 ++++++++-------- .../handlers/ios/performanceAggregator.go | 32 +++++++++------ backend/internal/handlers/web/clickRage.go | 27 ++++++++----- backend/internal/handlers/web/cpuIssue.go | 14 ++++++- backend/internal/handlers/web/deadClick.go | 38 ++++++++++++------ backend/internal/handlers/web/domDrop.go | 11 +++-- backend/internal/handlers/web/memoryIssue.go | 27 +++++++++---- .../handlers/web/performanceAggregator.go | 6 +++ backend/pkg/messages/messages.go | 2 +- 10 files changed, 148 insertions(+), 83 deletions(-) diff --git a/backend/internal/handlers/ios/appNotResponding.go b/backend/internal/handlers/ios/appNotResponding.go index 1241648db..097361c00 100644 --- a/backend/internal/handlers/ios/appNotResponding.go +++ b/backend/internal/handlers/ios/appNotResponding.go @@ -3,9 +3,17 @@ package ios import ( "openreplay/backend/internal/handlers" . "openreplay/backend/pkg/messages" + "time" ) -// app is not responding detector +/* + Handler name: AppNotResponding + Input events: IOSClickEvent, + IOSInputEvent, + IOSPerformanceEvent, + IOSSessionEnd + Output event: IOSIssueEvent +*/ const MIN_TIME_AFTER_LAST_HEARTBEAT = 60 * 1000 @@ -17,46 +25,44 @@ type AppNotResponding struct { } func (h *AppNotResponding) Handle(message Message, messageID uint64, timestamp uint64) Message { + var event Message = nil switch m := message.(type) { case *IOSClickEvent: - h.buildIf(m.Timestamp) + event = h.build(m.Timestamp) h.lastLabel = m.Label h.lastHeartbeatTimestamp = m.Timestamp h.lastHeartbeatIndex = m.Index case *IOSInputEvent: - h.buildIf(m.Timestamp) + event = h.build(m.Timestamp) h.lastLabel = m.Label h.lastHeartbeatTimestamp = m.Timestamp h.lastHeartbeatIndex = m.Index case *IOSPerformanceEvent: - h.buildIf(m.Timestamp) + event = h.build(m.Timestamp) h.lastHeartbeatTimestamp = m.Timestamp h.lastHeartbeatIndex = m.Index case *IOSSessionEnd: - h.buildIf(m.Timestamp) + event = h.build(m.Timestamp) } - return nil + return event } func (h *AppNotResponding) Build() Message { - //TODO implement me - panic("implement me") + return h.build(uint64(time.Now().Unix())) } -func (h *AppNotResponding) buildIf(timestamp uint64) { +func (h *AppNotResponding) build(timestamp uint64) Message { if h.lastHeartbeatTimestamp != 0 && h.lastHeartbeatTimestamp+MIN_TIME_AFTER_LAST_HEARTBEAT <= timestamp { - m := &IOSIssueEvent{ + event := &IOSIssueEvent{ Type: "anr", ContextString: h.lastLabel, + Timestamp: h.lastHeartbeatTimestamp, } - m.Timestamp = h.lastHeartbeatTimestamp - m.Index = h.lastHeartbeatIndex // Associated Index/ MessageID ? - h.Append(m) + event.Index = h.lastHeartbeatIndex // Associated Index/ MessageID ? + // Reset h.lastHeartbeatTimestamp = 0 h.lastHeartbeatIndex = 0 + return event } -} - -func (h *AppNotResponding) HandleMessage(msg Message) { - // TODO: delete it + return nil } diff --git a/backend/internal/handlers/ios/clickRage.go b/backend/internal/handlers/ios/clickRage.go index 2707c04d5..6562e05c1 100644 --- a/backend/internal/handlers/ios/clickRage.go +++ b/backend/internal/handlers/ios/clickRage.go @@ -6,9 +6,14 @@ import ( . "openreplay/backend/pkg/messages" ) -const CLICK_TIME_DIFF = 200 +/* + Handler name: ClickRage + Input events: IOSClickEvent, + IOSSessionEnd + Output event: IOSIssueEvent +*/ -//const MIN_CLICKS_IN_A_ROW = 3 +const CLICK_TIME_DIFF = 200 type ClickRageDetector struct { handlers.ReadyMessageStore @@ -20,6 +25,7 @@ type ClickRageDetector struct { } func (h *ClickRageDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { + var event Message = nil switch m := message.(type) { case *IOSClickEvent: if h.lastTimestamp+CLICK_TIME_DIFF < m.Timestamp && h.lastLabel == m.Label { @@ -27,7 +33,7 @@ func (h *ClickRageDetector) Handle(message Message, messageID uint64, timestamp h.countsInARow += 1 return nil } - h.build() + event = h.Build() if m.Label != "" { h.lastTimestamp = m.Timestamp h.lastLabel = m.Label @@ -36,33 +42,25 @@ func (h *ClickRageDetector) Handle(message Message, messageID uint64, timestamp h.countsInARow = 1 } case *IOSSessionEnd: - h.build() + event = h.Build() } - return nil + return event } func (h *ClickRageDetector) Build() Message { - //TODO implement me - panic("implement me") -} - -func (h *ClickRageDetector) build() { if h.countsInARow >= web.MIN_CLICKS_IN_A_ROW { - m := &IOSIssueEvent{ + event := &IOSIssueEvent{ Type: "click_rage", ContextString: h.lastLabel, } - m.Timestamp = h.firstInARawTimestamp - m.Index = h.firstInARawSeqIndex // Associated Index/ MessageID ? - h.Append(m) + event.Timestamp = h.firstInARawTimestamp + event.Index = h.firstInARawSeqIndex // Associated Index/ MessageID ? + return event } h.lastTimestamp = 0 h.lastLabel = "" h.firstInARawTimestamp = 0 h.firstInARawSeqIndex = 0 h.countsInARow = 0 -} - -func (h *ClickRageDetector) HandleMessage(msg Message) { - // TODO: delete it + return nil } diff --git a/backend/internal/handlers/ios/performanceAggregator.go b/backend/internal/handlers/ios/performanceAggregator.go index 1525127b8..b4bc812c7 100644 --- a/backend/internal/handlers/ios/performanceAggregator.go +++ b/backend/internal/handlers/ios/performanceAggregator.go @@ -3,8 +3,16 @@ package ios import ( "openreplay/backend/internal/handlers" . "openreplay/backend/pkg/messages" + "time" ) +/* + Handler name: PerformanceAggregator + Input events: IOSPerformanceEvent, + IOSSessionEnd + Output event: IssueEvent +*/ + const AGGR_TIME = 15 * 60 * 1000 type valueAggregator struct { @@ -32,13 +40,14 @@ func (h *PerformanceAggregator) Handle(message Message, messageID uint64, timest if h.pa == nil { h.pa = &IOSPerformanceAggregated{} // TODO: struct type in messages } - switch m := message.(type) { // TODO: All Timestampe messages + var event Message = nil + switch m := message.(type) { // TODO: All Timestamp messages case *IOSPerformanceEvent: if h.pa.TimestampStart == 0 { h.pa.TimestampStart = m.Timestamp } if h.pa.TimestampStart+AGGR_TIME <= m.Timestamp { - h.build(m.Timestamp) + event = h.build(m.Timestamp) } switch m.Name { case "fps": @@ -79,35 +88,32 @@ func (h *PerformanceAggregator) Handle(message Message, messageID uint64, timest } } case *IOSSessionEnd: - h.build(m.Timestamp) + event = h.build(m.Timestamp) } - return nil + return event } func (h *PerformanceAggregator) Build() Message { - //TODO implement me - panic("implement me") + return h.build(uint64(time.Now().Unix())) } -func (h *PerformanceAggregator) build(timestamp uint64) { +func (h *PerformanceAggregator) build(timestamp uint64) Message { if h.pa == nil { - return + return nil } + h.pa.TimestampEnd = timestamp h.pa.AvgFPS = h.fps.aggregate() h.pa.AvgCPU = h.cpu.aggregate() h.pa.AvgMemory = h.memory.aggregate() h.pa.AvgBattery = h.battery.aggregate() - h.Append(h.pa) + event := h.pa h.pa = &IOSPerformanceAggregated{} for _, agg := range []valueAggregator{h.fps, h.cpu, h.memory, h.battery} { agg.sum = 0 agg.count = 0 } -} - -func (h *PerformanceAggregator) HandleMessage(msg Message) { - // TODO: delete it + return event } diff --git a/backend/internal/handlers/web/clickRage.go b/backend/internal/handlers/web/clickRage.go index db22a9667..e22eb6454 100644 --- a/backend/internal/handlers/web/clickRage.go +++ b/backend/internal/handlers/web/clickRage.go @@ -2,11 +2,16 @@ package web import ( "encoding/json" + "log" . "openreplay/backend/pkg/messages" ) -// TODO: Description of click rage detector +/* + Handler name: ClickRage + Input event: MouseClick + Output event: IssueEvent +*/ const MAX_TIME_DIFF = 300 const MIN_CLICKS_IN_A_ROW = 3 @@ -28,26 +33,28 @@ func (crd *ClickRageDetector) reset() { } func (crd *ClickRageDetector) Build() Message { + defer crd.reset() if crd.countsInARow >= MIN_CLICKS_IN_A_ROW { - payload, _ := json.Marshal(struct{ Count int }{crd.countsInARow}) - i := &IssueEvent{ + payload, err := json.Marshal(struct{ Count int }{crd.countsInARow}) + if err != nil { + log.Printf("can't marshal ClickRage payload to json: %s", err) + } + event := &IssueEvent{ Type: "click_rage", ContextString: crd.lastLabel, - Payload: string(payload), // TODO: json message field type + Payload: string(payload), Timestamp: crd.firstInARawTimestamp, MessageID: crd.firstInARawMessageId, } - crd.reset() - return i + return event } - crd.reset() return nil } func (crd *ClickRageDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { switch msg := message.(type) { case *MouseClick: - // TODO: check if we it is ok to capture clickrages without the connected CleckEvent in db. + // TODO: check if we it is ok to capture clickRage event without the connected ClickEvent in db. if msg.Label == "" { return crd.Build() } @@ -56,13 +63,13 @@ func (crd *ClickRageDetector) Handle(message Message, messageID uint64, timestam crd.countsInARow += 1 return nil } - i := crd.Build() + event := crd.Build() crd.lastTimestamp = timestamp crd.lastLabel = msg.Label crd.firstInARawTimestamp = timestamp crd.firstInARawMessageId = messageID crd.countsInARow = 1 - return i + return event } return nil } diff --git a/backend/internal/handlers/web/cpuIssue.go b/backend/internal/handlers/web/cpuIssue.go index 5cc12be68..56f483e8b 100644 --- a/backend/internal/handlers/web/cpuIssue.go +++ b/backend/internal/handlers/web/cpuIssue.go @@ -2,12 +2,18 @@ package web import ( "encoding/json" + "log" . "openreplay/backend/pkg/messages" "openreplay/backend/pkg/messages/performance" ) -// TODO: Description of cpu issue detector +/* + Handler name: CpuIssue + Input events: PerformanceTrack, + SetPageLocation + Output event: IssueEvent +*/ const CPU_THRESHOLD = 70 // % out of 100 const CPU_MIN_DURATION_TRIGGER = 6 * 1000 @@ -36,10 +42,14 @@ func (f *CpuIssueDetector) Build() Message { return nil } - payload, _ := json.Marshal(struct { + payload, err := json.Marshal(struct { Duration uint64 Rate uint64 }{duration, maxRate}) + if err != nil { + log.Printf("can't marshal CpuIssue payload to json: %s", err) + } + return &IssueEvent{ Type: "cpu", Timestamp: timestamp, diff --git a/backend/internal/handlers/web/deadClick.go b/backend/internal/handlers/web/deadClick.go index a04da9be9..6377b074e 100644 --- a/backend/internal/handlers/web/deadClick.go +++ b/backend/internal/handlers/web/deadClick.go @@ -4,7 +4,22 @@ import ( . "openreplay/backend/pkg/messages" ) -// TODO: Description of dead click detector +/* + Handler name: DeadClick + Input events: SetInputTarget, + CreateDocument, + MouseClick, + SetNodeAttribute, + RemoveNodeAttribute, + CreateElementNode, + CreateTextNode, + MoveNode, + RemoveNode, + SetCSSData, + CSSInsertRule, + CSSDeleteRule + Output event: IssueEvent +*/ const CLICK_RELATION_TIME = 1400 @@ -23,23 +38,22 @@ func (d *DeadClickDetector) reset() { d.lastMessageID = 0 } -func (d *DeadClickDetector) handleReaction(timestamp uint64) Message { - if d.lastMouseClick == nil || d.lastClickTimestamp+CLICK_RELATION_TIME > timestamp { // riaction is instant - d.reset() +func (d *DeadClickDetector) build(timestamp uint64) Message { + defer d.reset() + if d.lastMouseClick == nil || d.lastClickTimestamp+CLICK_RELATION_TIME > timestamp { // reaction is instant return nil } - i := &IssueEvent{ + event := &IssueEvent{ Type: "dead_click", ContextString: d.lastMouseClick.Label, Timestamp: d.lastClickTimestamp, MessageID: d.lastMessageID, } - d.reset() - return i + return event } func (d *DeadClickDetector) Build() Message { - return d.handleReaction(d.lastTimestamp) + return d.build(d.lastTimestamp) } func (d *DeadClickDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { @@ -56,14 +70,14 @@ func (d *DeadClickDetector) Handle(message Message, messageID uint64, timestamp if msg.Label == "" { return nil } - i := d.handleReaction(timestamp) + event := d.build(timestamp) if d.inputIDSet[msg.ID] { // ignore if input - return i + return event } d.lastMouseClick = msg d.lastClickTimestamp = timestamp d.lastMessageID = messageID - return i + return event case *SetNodeAttribute, *RemoveNodeAttribute, *CreateElementNode, @@ -73,7 +87,7 @@ func (d *DeadClickDetector) Handle(message Message, messageID uint64, timestamp *SetCSSData, *CSSInsertRule, *CSSDeleteRule: - return d.handleReaction(timestamp) + return d.build(timestamp) } return nil } diff --git a/backend/internal/handlers/web/domDrop.go b/backend/internal/handlers/web/domDrop.go index c89fab2c4..4a3ec2065 100644 --- a/backend/internal/handlers/web/domDrop.go +++ b/backend/internal/handlers/web/domDrop.go @@ -4,7 +4,13 @@ import ( . "openreplay/backend/pkg/messages" ) -// TODO: Description of dom drop detector +/* + Handler name: DomDrop + Input events: CreateElementNode, + CreateTextNode, + RemoveNode + Output event: DOMDrop +*/ const DROP_WINDOW = 200 //ms const CRITICAL_COUNT = 1 // Our login page contains 20. But on crush it removes only roots (1-3 nodes). @@ -38,13 +44,12 @@ func (dd *domDropDetector) Handle(message Message, _ uint64, timestamp uint64) M } func (dd *domDropDetector) Build() Message { + defer dd.reset() if dd.removedCount >= CRITICAL_COUNT { domDrop := &DOMDrop{ Timestamp: dd.lastDropTimestamp, } - dd.reset() return domDrop } - dd.reset() return nil } diff --git a/backend/internal/handlers/web/memoryIssue.go b/backend/internal/handlers/web/memoryIssue.go index ac8ca8a14..487c396a9 100644 --- a/backend/internal/handlers/web/memoryIssue.go +++ b/backend/internal/handlers/web/memoryIssue.go @@ -2,12 +2,18 @@ package web import ( "encoding/json" + "log" "math" . "openreplay/backend/pkg/messages" ) -// TODO: Description of memory issue detector +/* + Handler name: MemoryIssue + Input events: PerformanceTrack, + SetPageLocation + Output event: IssueEvent +*/ const MIN_COUNT = 3 const MEM_RATE_THRESHOLD = 300 // % to average @@ -21,22 +27,29 @@ type MemoryIssueDetector struct { contextString string } +func (f *MemoryIssueDetector) reset() { + f.startTimestamp = 0 + f.startMessageID = 0 + f.rate = 0 +} + func (f *MemoryIssueDetector) Build() Message { if f.startTimestamp == 0 { return nil } - payload, _ := json.Marshal(struct{ Rate int }{f.rate - 100}) - i := &IssueEvent{ + payload, err := json.Marshal(struct{ Rate int }{f.rate - 100}) + if err != nil { + log.Printf("can't marshal MemoryIssue payload to json: %s", err) + } + event := &IssueEvent{ Type: "memory", Timestamp: f.startTimestamp, MessageID: f.startMessageID, ContextString: f.contextString, Payload: string(payload), } - f.startTimestamp = 0 - f.startMessageID = 0 - f.rate = 0 - return i + f.reset() + return event } func (f *MemoryIssueDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { diff --git a/backend/internal/handlers/web/performanceAggregator.go b/backend/internal/handlers/web/performanceAggregator.go index a7bf79f9f..928cedeb9 100644 --- a/backend/internal/handlers/web/performanceAggregator.go +++ b/backend/internal/handlers/web/performanceAggregator.go @@ -7,6 +7,12 @@ import ( "openreplay/backend/pkg/messages/performance" ) +/* + Handler name: PerformanceAggregator + Input event: PerformanceTrack + Output event: PerformanceTrackAggr +*/ + const AGGREGATION_WINDOW = 2 * 60 * 1000 type PerformanceAggregator struct { diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index e9aec5788..418c47342 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -1168,7 +1168,7 @@ type IssueEvent struct { Type string ContextString string Context string - Payload string + Payload string // TODO: check, maybe it's better to use empty interface here } func (msg *IssueEvent) Encode() []byte { From c77966a78997ea78a9c114e81a11e07d2576d10c Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Wed, 11 May 2022 16:45:31 +0200 Subject: [PATCH 12/25] feat(backend/handlers): removed unix timestamp from header builders --- backend/cmd/db/main.go | 17 ++++--- backend/cmd/heuristics/main.go | 2 +- backend/internal/builder/builder.go | 44 +++++-------------- backend/internal/builder/builderMap.go | 12 +++-- .../internal/handlers/custom/mainHandler.go | 31 ------------- .../internal/handlers/ios/appNotResponding.go | 5 ++- .../handlers/ios/performanceAggregator.go | 15 ++++--- 7 files changed, 39 insertions(+), 87 deletions(-) diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index 564fcbab5..8236586c2 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -7,7 +7,6 @@ import ( "openreplay/backend/internal/datasaver" "openreplay/backend/internal/handlers" "openreplay/backend/internal/handlers/custom" - "openreplay/backend/pkg/intervals" "time" "os" @@ -33,9 +32,9 @@ func main() { // Declare message handlers we want to apply for each incoming message msgHandlers := []handlers.MessageProcessor{ - custom.NewMainHandler(), - custom.NewInputEventBuilder(), - custom.NewPageEventBuilder(), + custom.NewMainHandler(), // TODO: separate to several handler + //custom.NewInputEventBuilder(), + //custom.NewPageEventBuilder(), } // Create handler's aggregator @@ -54,7 +53,6 @@ func main() { if !postgres.IsPkeyViolation(err) { log.Printf("Message Insertion Error %v, SessionID: %v, Message: %v", err, sessionID, msg) } - // TODO: can we lose data here because of db error? return } @@ -95,7 +93,7 @@ func main() { consumer := queue.NewMessageConsumer( cfg.GroupDB, []string{ - cfg.TopicRawWeb, // TODO: is it necessary or not? + cfg.TopicRawWeb, cfg.TopicRawIOS, cfg.TopicTrigger, // to receive SessionEnd events }, @@ -109,7 +107,6 @@ func main() { signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) commitTick := time.Tick(cfg.CommitBatchTimeout) - checkTick := time.Tick(intervals.EVENTS_COMMIT_INTERVAL * time.Millisecond) for { select { case sig := <-sigchan: @@ -118,12 +115,14 @@ func main() { os.Exit(0) case <-commitTick: pg.CommitBatches() + // TODO: ee commit stats !!! + //if err := commitStats(); err != nil { + // log.Printf("Error on stats commit: %v", err) + //} // TODO?: separate stats & regular messages if err := consumer.Commit(); err != nil { log.Printf("Error on consumer commit: %v", err) } - case <-checkTick: - // checkTimeout default: err := consumer.ConsumeNext() if err != nil { diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index ddfd29095..9c77cb4ba 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -78,7 +78,7 @@ func main() { consumer.Close() os.Exit(0) case <-tick: - builderMap.IterateReadyMessages(time.Now().UnixMilli(), func(sessionID uint64, readyMsg messages.Message) { + builderMap.IterateReadyMessages(func(sessionID uint64, readyMsg messages.Message) { producer.Produce(cfg.TopicTrigger, sessionID, messages.Encode(readyMsg)) }) producer.Flush(cfg.ProducerTimeout) diff --git a/backend/internal/builder/builder.go b/backend/internal/builder/builder.go index a00ad194a..4916764cd 100644 --- a/backend/internal/builder/builder.go +++ b/backend/internal/builder/builder.go @@ -2,7 +2,6 @@ package builder import ( "openreplay/backend/internal/handlers" - "openreplay/backend/pkg/intervals" . "openreplay/backend/pkg/messages" ) @@ -26,6 +25,15 @@ func (b *builder) iterateReadyMessage(iter func(msg Message)) { b.readyMsgs = nil } +func (b *builder) checkSessionEnd(message Message) { + if _, isEnd := message.(*IOSSessionEnd); isEnd { + b.ended = true + } + if _, isEnd := message.(*SessionEnd); isEnd { + b.ended = true + } +} + func (b *builder) handleMessage(message Message, messageID uint64) { timestamp := GetTimestamp(message) if b.timestamp < timestamp { @@ -36,42 +44,10 @@ func (b *builder) handleMessage(message Message, messageID uint64) { return } - if _, isEnd := message.(*IOSSessionEnd); isEnd { - b.ended = true - } - if _, isEnd := message.(*SessionEnd); isEnd { - b.ended = true - } - + b.checkSessionEnd(message) for _, p := range b.processors { - /* If nil is not returned explicitely by Handle, but as the typed nil - ("var i *IssueEvent; return i;") - The `rm != nil` will be true. - TODO: enforce nil to be nil(?) or add `isNil() bool` to the Message types - because this part is expected to be etendable by user with custom messageProcessor's. - Use of reflrction will be probably bad on millions of messages? - */ if rm := p.Handle(message, messageID, b.timestamp); rm != nil { b.readyMsgs = append(b.readyMsgs, rm) } } } - -func (b *builder) checkTimeouts(ts int64) bool { - if b.timestamp == 0 { - return false // SessionStart happened only - } - - lastTsGap := ts - int64(b.timestamp) - // Maybe listen for `trigger` and react on SessionEnd instead (less reliable) - if lastTsGap > intervals.EVENTS_SESSION_END_TIMEOUT { - for _, p := range b.processors { - // TODO: same as above - if rm := p.Build(); rm != nil { - b.readyMsgs = append(b.readyMsgs, rm) - } - } - return true - } - return false -} diff --git a/backend/internal/builder/builderMap.go b/backend/internal/builder/builderMap.go index 6b2c22bec..b393bcd28 100644 --- a/backend/internal/builder/builderMap.go +++ b/backend/internal/builder/builderMap.go @@ -31,13 +31,19 @@ func (m *builderMap) HandleMessage(sessionID uint64, msg Message, messageID uint b.handleMessage(msg, messageID) } -func (m *builderMap) IterateReadyMessages(operatingTs int64, iter func(sessionID uint64, msg Message)) { +func (m *builderMap) IterateReadyMessages(iter func(sessionID uint64, msg Message)) { for sessionID, b := range m.sessions { - sessionEnded := b.checkTimeouts(operatingTs) + if b.ended { + for _, p := range b.processors { + if rm := p.Build(); rm != nil { + b.readyMsgs = append(b.readyMsgs, rm) + } + } + } b.iterateReadyMessage(func(msg Message) { iter(sessionID, msg) }) - if sessionEnded { + if b.ended { delete(m.sessions, sessionID) } } diff --git a/backend/internal/handlers/custom/mainHandler.go b/backend/internal/handlers/custom/mainHandler.go index 7e653c250..52a6278c0 100644 --- a/backend/internal/handlers/custom/mainHandler.go +++ b/backend/internal/handlers/custom/mainHandler.go @@ -125,9 +125,6 @@ func (b *builder) Handle(message Message, messageID uint64, timestamp uint64) Me b.buildInputEvent() b.ieBuilder.ClearLabels() b.peBuilder.HandleSetPageLocation(msg, messageID, b.timestamp) - // TODO: what to do with this code? - //b.miFinder.HandleSetPageLocation(msg) - //b.ciFinder.HandleSetPageLocation(msg) } case *PageLoadTiming: if rm := b.peBuilder.HandlePageLoadTiming(msg); rm != nil { @@ -137,17 +134,6 @@ func (b *builder) Handle(message Message, messageID uint64, timestamp uint64) Me if rm := b.peBuilder.HandlePageRenderTiming(msg); rm != nil { b.appendReadyMessage(rm) } - case *PerformanceTrack: - // TODO: what to do with this code? - //if rm := b.ptaBuilder.HandlePerformanceTrack(msg, b.timestamp); rm != nil { - // b.appendReadyMessage(rm) - //} - //if rm := b.ciFinder.HandlePerformanceTrack(msg, messageID, b.timestamp); rm != nil { - // b.appendReadyMessage(rm) - //} - //if rm := b.miFinder.HandlePerformanceTrack(msg, messageID, b.timestamp); rm != nil { - // b.appendReadyMessage(rm) - //} case *SetInputTarget: if rm := b.ieBuilder.HandleSetInputTarget(msg); rm != nil { b.appendReadyMessage(rm) @@ -158,10 +144,6 @@ func (b *builder) Handle(message Message, messageID uint64, timestamp uint64) Me } case *MouseClick: b.buildInputEvent() - // TODO: what to do with this code? - //if rm := b.crDetector.HandleMouseClick(msg, messageID, b.timestamp); rm != nil { - // b.appendReadyMessage(rm) - //} if msg.Label != "" { b.appendReadyMessage(&ClickEvent{ MessageID: messageID, @@ -256,20 +238,7 @@ func (b *builder) Handle(message Message, messageID uint64, timestamp uint64) Me Timestamp: b.timestamp, Type: msg.Type, }) - // TODO: what to do with this code? - //case *CreateElementNode, *CreateTextNode: - // b.ddDetector.HandleNodeCreation() - //case *RemoveNode: - // b.ddDetector.HandleNodeRemoval(b.timestamp) - //case *CreateDocument: - // if rm := b.ddDetector.Build(); rm != nil { - // b.appendReadyMessage(rm) - // } } - // TODO: what to do with this code? - //if rm := b.dcDetector.HandleMessage(message, messageID, b.timestamp); rm != nil { - // b.appendReadyMessage(rm) - //} return nil } diff --git a/backend/internal/handlers/ios/appNotResponding.go b/backend/internal/handlers/ios/appNotResponding.go index 097361c00..b5f6cd2f0 100644 --- a/backend/internal/handlers/ios/appNotResponding.go +++ b/backend/internal/handlers/ios/appNotResponding.go @@ -3,7 +3,6 @@ package ios import ( "openreplay/backend/internal/handlers" . "openreplay/backend/pkg/messages" - "time" ) /* @@ -22,9 +21,11 @@ type AppNotResponding struct { lastLabel string lastHeartbeatTimestamp uint64 lastHeartbeatIndex uint64 + lastTimestamp uint64 } func (h *AppNotResponding) Handle(message Message, messageID uint64, timestamp uint64) Message { + h.lastTimestamp = timestamp var event Message = nil switch m := message.(type) { case *IOSClickEvent: @@ -48,7 +49,7 @@ func (h *AppNotResponding) Handle(message Message, messageID uint64, timestamp u } func (h *AppNotResponding) Build() Message { - return h.build(uint64(time.Now().Unix())) + return h.build(h.lastTimestamp) } func (h *AppNotResponding) build(timestamp uint64) Message { diff --git a/backend/internal/handlers/ios/performanceAggregator.go b/backend/internal/handlers/ios/performanceAggregator.go index b4bc812c7..2a9401748 100644 --- a/backend/internal/handlers/ios/performanceAggregator.go +++ b/backend/internal/handlers/ios/performanceAggregator.go @@ -3,7 +3,6 @@ package ios import ( "openreplay/backend/internal/handlers" . "openreplay/backend/pkg/messages" - "time" ) /* @@ -29,14 +28,16 @@ func (va *valueAggregator) aggregate() uint64 { type PerformanceAggregator struct { handlers.ReadyMessageStore - pa *IOSPerformanceAggregated - fps valueAggregator - cpu valueAggregator - memory valueAggregator - battery valueAggregator + pa *IOSPerformanceAggregated + fps valueAggregator + cpu valueAggregator + memory valueAggregator + battery valueAggregator + lastTimestamp uint64 } func (h *PerformanceAggregator) Handle(message Message, messageID uint64, timestamp uint64) Message { + h.lastTimestamp = timestamp if h.pa == nil { h.pa = &IOSPerformanceAggregated{} // TODO: struct type in messages } @@ -94,7 +95,7 @@ func (h *PerformanceAggregator) Handle(message Message, messageID uint64, timest } func (h *PerformanceAggregator) Build() Message { - return h.build(uint64(time.Now().Unix())) + return h.build(h.lastTimestamp) } func (h *PerformanceAggregator) build(timestamp uint64) Message { From e65fa58ab56e3198e46e298090f396c36ec111d0 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 11 May 2022 18:51:55 +0200 Subject: [PATCH 13/25] refactor(backend-internal): dry builder --- backend/internal/builder/builder.go | 4 +-- backend/internal/builder/builderMap.go | 45 +++++++++++++++----------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/backend/internal/builder/builder.go b/backend/internal/builder/builder.go index 4916764cd..ff3d91e1b 100644 --- a/backend/internal/builder/builder.go +++ b/backend/internal/builder/builder.go @@ -18,7 +18,7 @@ func NewBuilder(handlers ...handlers.MessageProcessor) *builder { } } -func (b *builder) iterateReadyMessage(iter func(msg Message)) { +func (b *builder) iterateReadyMessages(iter func(msg Message)) { for _, readyMsg := range b.readyMsgs { iter(readyMsg) } @@ -44,10 +44,10 @@ func (b *builder) handleMessage(message Message, messageID uint64) { return } - b.checkSessionEnd(message) for _, p := range b.processors { if rm := p.Handle(message, messageID, b.timestamp); rm != nil { b.readyMsgs = append(b.readyMsgs, rm) } } + b.checkSessionEnd(message) } diff --git a/backend/internal/builder/builderMap.go b/backend/internal/builder/builderMap.go index b393bcd28..f6d81b995 100644 --- a/backend/internal/builder/builderMap.go +++ b/backend/internal/builder/builderMap.go @@ -20,7 +20,7 @@ func NewBuilderMap(handlers ...handlers.MessageProcessor) *builderMap { func (m *builderMap) GetBuilder(sessionID uint64) *builder { b := m.sessions[sessionID] if b == nil { - b = NewBuilder(m.handlers...) + b = NewBuilder(m.handlers...) // Should create new instances m.sessions[sessionID] = b } return b @@ -31,21 +31,29 @@ func (m *builderMap) HandleMessage(sessionID uint64, msg Message, messageID uint b.handleMessage(msg, messageID) } -func (m *builderMap) IterateReadyMessages(iter func(sessionID uint64, msg Message)) { - for sessionID, b := range m.sessions { - if b.ended { - for _, p := range b.processors { - if rm := p.Build(); rm != nil { - b.readyMsgs = append(b.readyMsgs, rm) - } +func (m *builderMap) iterateSessionReadyMessages(sessionID uint64, b *builder, iter func(msg Message)) { + if b.ended { + for _, p := range b.processors { + if rm := p.Build(); rm != nil { + b.readyMsgs = append(b.readyMsgs, rm) } } - b.iterateReadyMessage(func(msg Message) { - iter(sessionID, msg) - }) - if b.ended { - delete(m.sessions, sessionID) - } + } + b.iterateReadyMessage(iter) + if b.ended { + delete(m.sessions, sessionID) + } +} + +func (m *builderMap) IterateReadyMessages(iter func(sessionID uint64, msg Message)) { + for sessionID, session := range m.sessions { + m.iterateSessionReadyMessages( + sessionID, + session, + func(msg Message) { + iter(sessionID, msg) + }, + ) } } @@ -54,8 +62,9 @@ func (m *builderMap) IterateSessionReadyMessages(sessionID uint64, iter func(msg if !ok { return } - session.iterateReadyMessage(iter) - if session.ended { - delete(m.sessions, sessionID) - } + m.iterateSessionReadyMessages( + sessionID, + session, + inter, + ) } From a6f8857b8912747491236f2e7440673678f0f2ec Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 11 May 2022 19:04:14 +0200 Subject: [PATCH 14/25] refactor-fix(backend-heuristics/db): create handlers for each session separately --- backend/cmd/db/main.go | 14 ++++++----- backend/cmd/heuristics/main.go | 32 ++++++++++++++------------ backend/internal/builder/builderMap.go | 12 +++++----- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index 8236586c2..d3ba45017 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -30,15 +30,17 @@ func main() { pg := cache.NewPGCache(postgres.NewConn(cfg.Postgres), cfg.ProjectExpirationTimeoutMs) defer pg.Close() - // Declare message handlers we want to apply for each incoming message - msgHandlers := []handlers.MessageProcessor{ - custom.NewMainHandler(), // TODO: separate to several handler - //custom.NewInputEventBuilder(), - //custom.NewPageEventBuilder(), + // HandlersFabric returns the list of message handlers we want to be applied to each incoming message. + handlersFabric := func() { + return []handlers.MessageProcessor{ + custom.NewMainHandler(), // TODO: separate to several handler + //custom.NewInputEventBuilder(), + //custom.NewPageEventBuilder(), + } } // Create handler's aggregator - builderMap := builder.NewBuilderMap(msgHandlers...) + builderMap := builder.NewBuilderMap(handlersFabric) // Init modules saver := datasaver.New(pg) diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index 9c77cb4ba..5543f85e1 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -25,24 +25,26 @@ func main() { // Load service configuration cfg := ender.New() - // Declare message handlers we want to apply for each incoming message - msgHandlers := []handlers.MessageProcessor{ - // web handlers - &web.ClickRageDetector{}, - &web.CpuIssueDetector{}, - &web.DeadClickDetector{}, - &web.MemoryIssueDetector{}, - &web.PerformanceAggregator{}, - // iOS handlers - &ios.AppNotResponding{}, - &ios.ClickRageDetector{}, - &ios.PerformanceAggregator{}, - // Other handlers (you can add your custom handlers here) - &custom.CustomHandler{}, + // HandlersFabric returns the list of message handlers we want to be applied to each incoming message. + handlersFabric := func() { + return []handlers.MessageProcessor{ + // web handlers + &web.ClickRageDetector{}, + &web.CpuIssueDetector{}, + &web.DeadClickDetector{}, + &web.MemoryIssueDetector{}, + &web.PerformanceAggregator{}, + // iOS handlers + &ios.AppNotResponding{}, + &ios.ClickRageDetector{}, + &ios.PerformanceAggregator{}, + // Other handlers (you can add your custom handlers here) + &custom.CustomHandler{}, + } } // Create handler's aggregator - builderMap := builder.NewBuilderMap(msgHandlers...) + builderMap := builder.NewBuilderMap(handlersFabric) // Init logger statsLogger := logger.NewQueueStats(cfg.LoggerTimeout) diff --git a/backend/internal/builder/builderMap.go b/backend/internal/builder/builderMap.go index f6d81b995..ed47abdce 100644 --- a/backend/internal/builder/builderMap.go +++ b/backend/internal/builder/builderMap.go @@ -6,21 +6,21 @@ import ( ) type builderMap struct { - handlers []handlers.MessageProcessor - sessions map[uint64]*builder + handlersFabric func() []handlers.MessageProcessor + sessions map[uint64]*builder } -func NewBuilderMap(handlers ...handlers.MessageProcessor) *builderMap { +func NewBuilderMap(handlersFabric func() []handlers.MessageProcessor) *builderMap { return &builderMap{ - handlers: handlers, - sessions: make(map[uint64]*builder), + handlersFabric: handlersFabric, + sessions: make(map[uint64]*builder), } } func (m *builderMap) GetBuilder(sessionID uint64) *builder { b := m.sessions[sessionID] if b == nil { - b = NewBuilder(m.handlers...) // Should create new instances + b = NewBuilder(m.handlersFabric()) // Should create new instances m.sessions[sessionID] = b } return b From 85b87e17dfbf91eab8e3ebe239fd3e668f224cf1 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 11 May 2022 21:14:23 +0200 Subject: [PATCH 15/25] refactor(backend/internals): builder: message order & timestamps check --- backend/internal/builder/builder.go | 30 ++++++++++++++++++-------- backend/internal/builder/builderMap.go | 6 +++++- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/backend/internal/builder/builder.go b/backend/internal/builder/builder.go index ff3d91e1b..38a64ab54 100644 --- a/backend/internal/builder/builder.go +++ b/backend/internal/builder/builder.go @@ -6,10 +6,12 @@ import ( ) type builder struct { - readyMsgs []Message - timestamp uint64 - processors []handlers.MessageProcessor - ended bool + readyMsgs []Message + timestamp uint64 + lastMessageID uint64 + lastSystemTimestamp int64 + processors []handlers.MessageProcessor + ended bool } func NewBuilder(handlers ...handlers.MessageProcessor) *builder { @@ -35,15 +37,25 @@ func (b *builder) checkSessionEnd(message Message) { } func (b *builder) handleMessage(message Message, messageID uint64) { - timestamp := GetTimestamp(message) - if b.timestamp < timestamp { - b.timestamp = timestamp + if messageID < b.lastMessageID { + // May happen in case of duplicated messages in kafka (if `idempotence: false`) + return } - if b.timestamp == 0 { - // in case of SessionStart. TODO: make timestamp system transparent + timestamp := GetTimestamp(message) + if timestamp == 0 { + // May happen in case of messages that are single-in-batch, + // e.g. SessionStart or RawErrorEvent (emitted by `integrations`). + + // TODO: make timestamp system transparent; + return + } + if timestamp < b.timestamp { + // Shouldn't happen after messageID check which is done above. TODO: log this case. return } + b.timestamp = timestamp + b.lastSystemTimestamp = time.Now().UnixMilli() for _, p := range b.processors { if rm := p.Handle(message, messageID, b.timestamp); rm != nil { b.readyMsgs = append(b.readyMsgs, rm) diff --git a/backend/internal/builder/builderMap.go b/backend/internal/builder/builderMap.go index ed47abdce..fcd52b0cc 100644 --- a/backend/internal/builder/builderMap.go +++ b/backend/internal/builder/builderMap.go @@ -1,10 +1,14 @@ package builder import ( + "time" + "openreplay/backend/internal/handlers" . "openreplay/backend/pkg/messages" ) +const FORCE_DELETE_TIMEOUT = 4 * time.Hour + type builderMap struct { handlersFabric func() []handlers.MessageProcessor sessions map[uint64]*builder @@ -32,7 +36,7 @@ func (m *builderMap) HandleMessage(sessionID uint64, msg Message, messageID uint } func (m *builderMap) iterateSessionReadyMessages(sessionID uint64, b *builder, iter func(msg Message)) { - if b.ended { + if b.ended || b.lastSystemTimestamp+FORCE_DELETE_TIMEOUT < time.Now().UnixMilli() { for _, p := range b.processors { if rm := p.Build(); rm != nil { b.readyMsgs = append(b.readyMsgs, rm) From 6d2bfc0e77d6379f07445fc1efa94820722b31d6 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 11 May 2022 21:25:41 +0200 Subject: [PATCH 16/25] fix(backend/internals): builder codefix --- backend/internal/builder/builder.go | 16 +++++++++------- backend/internal/builder/builderMap.go | 8 ++++---- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/backend/internal/builder/builder.go b/backend/internal/builder/builder.go index 38a64ab54..dd7bb675a 100644 --- a/backend/internal/builder/builder.go +++ b/backend/internal/builder/builder.go @@ -1,17 +1,19 @@ package builder import ( + "time" + "openreplay/backend/internal/handlers" . "openreplay/backend/pkg/messages" ) type builder struct { - readyMsgs []Message - timestamp uint64 - lastMessageID uint64 - lastSystemTimestamp int64 - processors []handlers.MessageProcessor - ended bool + readyMsgs []Message + timestamp uint64 + lastMessageID uint64 + lastSystemTime time.Time + processors []handlers.MessageProcessor + ended bool } func NewBuilder(handlers ...handlers.MessageProcessor) *builder { @@ -55,7 +57,7 @@ func (b *builder) handleMessage(message Message, messageID uint64) { } b.timestamp = timestamp - b.lastSystemTimestamp = time.Now().UnixMilli() + b.lastSystemTime = time.Now() for _, p := range b.processors { if rm := p.Handle(message, messageID, b.timestamp); rm != nil { b.readyMsgs = append(b.readyMsgs, rm) diff --git a/backend/internal/builder/builderMap.go b/backend/internal/builder/builderMap.go index fcd52b0cc..af2ecf0d5 100644 --- a/backend/internal/builder/builderMap.go +++ b/backend/internal/builder/builderMap.go @@ -24,7 +24,7 @@ func NewBuilderMap(handlersFabric func() []handlers.MessageProcessor) *builderMa func (m *builderMap) GetBuilder(sessionID uint64) *builder { b := m.sessions[sessionID] if b == nil { - b = NewBuilder(m.handlersFabric()) // Should create new instances + b = NewBuilder(m.handlersFabric()...) // Should create new instances m.sessions[sessionID] = b } return b @@ -36,14 +36,14 @@ func (m *builderMap) HandleMessage(sessionID uint64, msg Message, messageID uint } func (m *builderMap) iterateSessionReadyMessages(sessionID uint64, b *builder, iter func(msg Message)) { - if b.ended || b.lastSystemTimestamp+FORCE_DELETE_TIMEOUT < time.Now().UnixMilli() { + if b.ended || b.lastSystemTime.Add(FORCE_DELETE_TIMEOUT).Before(time.Now()) { for _, p := range b.processors { if rm := p.Build(); rm != nil { b.readyMsgs = append(b.readyMsgs, rm) } } } - b.iterateReadyMessage(iter) + b.iterateReadyMessages(iter) if b.ended { delete(m.sessions, sessionID) } @@ -69,6 +69,6 @@ func (m *builderMap) IterateSessionReadyMessages(sessionID uint64, iter func(msg m.iterateSessionReadyMessages( sessionID, session, - inter, + iter, ) } From 88bec7ab6065ade07f3e4ecf1941df27ae5f8c9f Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 11 May 2022 21:27:18 +0200 Subject: [PATCH 17/25] refactor(): separate ieBuilder, peBuilder & networkIssueDeterctor from EventMapper --- backend/cmd/db/main.go | 6 +- backend/cmd/heuristics/main.go | 1 + .../internal/handlers/custom/eventMapper.go | 135 +++++++++ .../handlers/custom/inputEventBuilder.go | 95 +++---- .../internal/handlers/custom/mainHandler.go | 257 ------------------ .../handlers/custom/pageEventBuilder.go | 159 ++++++----- .../handlers/ios/performanceAggregator.go | 10 +- backend/internal/handlers/web/networkIssue.go | 47 ++++ backend/pkg/intervals/intervals.go | 2 - 9 files changed, 309 insertions(+), 403 deletions(-) create mode 100644 backend/internal/handlers/custom/eventMapper.go delete mode 100644 backend/internal/handlers/custom/mainHandler.go create mode 100644 backend/internal/handlers/web/networkIssue.go diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index d3ba45017..d3d786242 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -33,9 +33,9 @@ func main() { // HandlersFabric returns the list of message handlers we want to be applied to each incoming message. handlersFabric := func() { return []handlers.MessageProcessor{ - custom.NewMainHandler(), // TODO: separate to several handler - //custom.NewInputEventBuilder(), - //custom.NewPageEventBuilder(), + custom.EventMapper{}, + custom.NewInputEventBuilder(), + custom.NewPageEventBuilder(), } } diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index 5543f85e1..6712c927c 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -33,6 +33,7 @@ func main() { &web.CpuIssueDetector{}, &web.DeadClickDetector{}, &web.MemoryIssueDetector{}, + &web.NetworkIssueDetector{}, &web.PerformanceAggregator{}, // iOS handlers &ios.AppNotResponding{}, diff --git a/backend/internal/handlers/custom/eventMapper.go b/backend/internal/handlers/custom/eventMapper.go new file mode 100644 index 000000000..5d118ff7d --- /dev/null +++ b/backend/internal/handlers/custom/eventMapper.go @@ -0,0 +1,135 @@ +package custom + +import ( + "net/url" + "strings" + + . "openreplay/backend/pkg/messages" +) + +func getURLExtention(URL string) string { + u, err := url.Parse(URL) + if err != nil { + return "" + } + i := strings.LastIndex(u.Path, ".") + return u.Path[i+1:] +} + +func getResourceType(initiator string, URL string) string { + switch initiator { + case "xmlhttprequest", "fetch": + return "fetch" + case "img": + return "img" + default: + switch getURLExtention(URL) { + case "css": + return "stylesheet" + case "js": + return "script" + case "png", "gif", "jpg", "jpeg", "svg": + return "img" + case "mp4", "mkv", "ogg", "webm", "avi", "mp3": + return "media" + default: + return "other" + } + } +} + +type EventMapper struct{} + +func (b *EventMapper) Build() Message { + return nil +} + +func (b *EventMapper) Handle(message Message, messageID uint64, timestamp uint64) Message { + switch msg := message.(type) { + case *RawErrorEvent: + // !!! This won't be handled because the Meta() timestamp emitted by `integrations` will be 0 + // TODO: move to db directly + return &ErrorEvent{ + MessageID: messageID, + Timestamp: msg.Timestamp, + Source: msg.Source, + Name: msg.Name, + Message: msg.Message, + Payload: msg.Payload, + } + case *MouseClick: + if msg.Label != "" { + return &ClickEvent{ + MessageID: messageID, + Label: msg.Label, + HesitationTime: msg.HesitationTime, + Timestamp: timestamp, + Selector: msg.Selector, + } + } + case *JSException: + return &ErrorEvent{ + MessageID: messageID, + Timestamp: timestamp, + Source: "js_exception", + Name: msg.Name, + Message: msg.Message, + Payload: msg.Payload, + } + case *ResourceTiming: + return &ResourceEvent{ + MessageID: messageID, + Timestamp: msg.Timestamp, + Duration: msg.Duration, + TTFB: msg.TTFB, + HeaderSize: msg.HeaderSize, + EncodedBodySize: msg.EncodedBodySize, + DecodedBodySize: msg.DecodedBodySize, + URL: msg.URL, + Type: getResourceType(msg.Initiator, msg.URL), + Success: msg.Duration != 0, + } + case *RawCustomEvent: + return &CustomEvent{ + MessageID: messageID, + Timestamp: timestamp, + Name: msg.Name, + Payload: msg.Payload, + } + case *CustomIssue: + return &IssueEvent{ + Type: "custom", + Timestamp: timestamp, + MessageID: messageID, + ContextString: msg.Name, + Payload: msg.Payload, + } + case *Fetch: + return &FetchEvent{ + MessageID: messageID, + Timestamp: msg.Timestamp, + Method: msg.Method, + URL: msg.URL, + Request: msg.Request, + Response: msg.Response, + Status: msg.Status, + Duration: msg.Duration, + } + case *GraphQL: + return &GraphQLEvent{ + MessageID: messageID, + Timestamp: timestamp, + OperationKind: msg.OperationKind, + OperationName: msg.OperationName, + Variables: msg.Variables, + Response: msg.Response, + } + case *StateAction: + return &StateActionEvent{ + MessageID: messageID, + Timestamp: timestamp, + Type: msg.Type, + } + } + return nil +} diff --git a/backend/internal/handlers/custom/inputEventBuilder.go b/backend/internal/handlers/custom/inputEventBuilder.go index 770e714af..e07470f37 100644 --- a/backend/internal/handlers/custom/inputEventBuilder.go +++ b/backend/internal/handlers/custom/inputEventBuilder.go @@ -4,6 +4,8 @@ import ( . "openreplay/backend/pkg/messages" ) +const INPUT_EVENT_TIMEOUT = 1 * 60 * 1000 + type inputLabels map[uint64]string type inputEventBuilder struct { @@ -12,78 +14,63 @@ type inputEventBuilder struct { inputID uint64 } -func (b *inputEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message { - //TODO implement me - panic("implement me") -} - -func (b *inputEventBuilder) Build() Message { - // b.build() - //TODO implement me - panic("implement me") -} - func NewInputEventBuilder() *inputEventBuilder { ieBuilder := &inputEventBuilder{} - ieBuilder.ClearLabels() + ieBuilder.clearLabels() return ieBuilder } -func (b *inputEventBuilder) ClearLabels() { +func (b *inputEventBuilder) clearLabels() { b.inputLabels = make(inputLabels) } -func (b *inputEventBuilder) HandleSetInputTarget(msg *SetInputTarget) *InputEvent { - var inputEvent *InputEvent - if b.inputID != msg.ID { - inputEvent = b.build() - b.inputID = msg.ID - } - b.inputLabels[msg.ID] = msg.Label - return inputEvent -} - -func (b *inputEventBuilder) HandleSetInputValue(msg *SetInputValue, messageID uint64, timestamp uint64) *InputEvent { - var inputEvent *InputEvent - if b.inputID != msg.ID { - inputEvent = b.build() - b.inputID = msg.ID - } - if b.inputEvent == nil { - b.inputEvent = &InputEvent{ - MessageID: messageID, - Timestamp: timestamp, - Value: msg.Value, - ValueMasked: msg.Mask > 0, +func (b *inputEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message { + var inputEvent Message = nil + switch msg := message.(type) { + case *SetInputTarget: + if b.inputID != msg.ID { + inputEvent = b.Build() + b.inputID = msg.ID } - } else { - b.inputEvent.Value = msg.Value - b.inputEvent.ValueMasked = msg.Mask > 0 + b.inputLabels[msg.ID] = msg.Label + return inputEvent + case *SetInputValue: + if b.inputID != msg.ID { + inputEvent = b.Build() + b.inputID = msg.ID + } + if b.inputEvent == nil { + b.inputEvent = &InputEvent{ + MessageID: messageID, + Timestamp: timestamp, + Value: msg.Value, + ValueMasked: msg.Mask > 0, + } + } else { + b.inputEvent.Value = msg.Value + b.inputEvent.ValueMasked = msg.Mask > 0 + } + return inputEvent + case *CreateDocument: + inputEvent = b.Build() + b.clearLabels() + return inputEvent + case *MouseClick: + return b.Build() } - return inputEvent -} -func (b *inputEventBuilder) HasInstance() bool { - return b.inputEvent != nil -} - -func (b *inputEventBuilder) GetTimestamp() uint64 { - if b.inputEvent == nil { - return 0 + if b.inputEvent != nil && b.inputEvent.Timestamp+INPUT_EVENT_TIMEOUT < timestamp { + return b.Build() } - return b.inputEvent.Timestamp + return nil } -func (b *inputEventBuilder) build() *InputEvent { +func (b *inputEventBuilder) Build() Message { if b.inputEvent == nil { return nil } inputEvent := b.inputEvent - label, exists := b.inputLabels[b.inputID] - if !exists { - return nil - } - inputEvent.Label = label + inputEvent.Label = b.inputLabels[b.inputID] // might be empty string b.inputEvent = nil return inputEvent diff --git a/backend/internal/handlers/custom/mainHandler.go b/backend/internal/handlers/custom/mainHandler.go deleted file mode 100644 index 52a6278c0..000000000 --- a/backend/internal/handlers/custom/mainHandler.go +++ /dev/null @@ -1,257 +0,0 @@ -package custom - -import ( - "net/url" - "openreplay/backend/pkg/intervals" - "strings" - "time" - - . "openreplay/backend/pkg/messages" -) - -func getURLExtention(URL string) string { - u, err := url.Parse(URL) - if err != nil { - return "" - } - i := strings.LastIndex(u.Path, ".") - return u.Path[i+1:] -} - -func getResourceType(initiator string, URL string) string { - switch initiator { - case "xmlhttprequest", "fetch": - return "fetch" - case "img": - return "img" - default: - switch getURLExtention(URL) { - case "css": - return "stylesheet" - case "js": - return "script" - case "png", "gif", "jpg", "jpeg", "svg": - return "img" - case "mp4", "mkv", "ogg", "webm", "avi", "mp3": - return "media" - default: - return "other" - } - } -} - -type builder struct { - readyMsgs []Message - timestamp uint64 - lastProcessedTimestamp int64 - peBuilder *pageEventBuilder - ieBuilder *inputEventBuilder - integrationsWaiting bool - sid uint64 -} - -func (b *builder) Build() Message { - //TODO implement me - panic("implement me") -} - -func NewMainHandler() *builder { - return &builder{ - peBuilder: &pageEventBuilder{}, - ieBuilder: NewInputEventBuilder(), - integrationsWaiting: true, - } -} - -func (b *builder) appendReadyMessage(msg Message) { // interface is never nil even if it holds nil value - b.readyMsgs = append(b.readyMsgs, msg) -} - -func (b *builder) iterateReadyMessage(iter func(msg Message)) { - for _, readyMsg := range b.readyMsgs { - iter(readyMsg) - } - b.readyMsgs = nil -} - -func (b *builder) buildPageEvent() { - if msg := b.peBuilder.Build(); msg != nil { - b.appendReadyMessage(msg) - } -} - -func (b *builder) buildInputEvent() { - if msg := b.ieBuilder.Build(); msg != nil { - b.appendReadyMessage(msg) - } -} - -func (b *builder) Handle(message Message, messageID uint64, timestamp uint64) Message { - b.timestamp = timestamp - b.lastProcessedTimestamp = time.Now().UnixMilli() - - // Might happen before the first timestamp. - switch msg := message.(type) { - case *SessionStart, - *Metadata, - *UserID, - *UserAnonymousID: - b.appendReadyMessage(msg) - case *RawErrorEvent: - b.appendReadyMessage(&ErrorEvent{ - MessageID: messageID, - Timestamp: msg.Timestamp, - Source: msg.Source, - Name: msg.Name, - Message: msg.Message, - Payload: msg.Payload, - }) - } - if b.timestamp == 0 { - return nil - } - switch msg := message.(type) { - case *SetPageLocation: - if msg.NavigationStart == 0 { - b.appendReadyMessage(&PageEvent{ - URL: msg.URL, - Referrer: msg.Referrer, - Loaded: false, - MessageID: messageID, - Timestamp: b.timestamp, - }) - } else { - b.buildPageEvent() - b.buildInputEvent() - b.ieBuilder.ClearLabels() - b.peBuilder.HandleSetPageLocation(msg, messageID, b.timestamp) - } - case *PageLoadTiming: - if rm := b.peBuilder.HandlePageLoadTiming(msg); rm != nil { - b.appendReadyMessage(rm) - } - case *PageRenderTiming: - if rm := b.peBuilder.HandlePageRenderTiming(msg); rm != nil { - b.appendReadyMessage(rm) - } - case *SetInputTarget: - if rm := b.ieBuilder.HandleSetInputTarget(msg); rm != nil { - b.appendReadyMessage(rm) - } - case *SetInputValue: - if rm := b.ieBuilder.HandleSetInputValue(msg, messageID, b.timestamp); rm != nil { - b.appendReadyMessage(rm) - } - case *MouseClick: - b.buildInputEvent() - if msg.Label != "" { - b.appendReadyMessage(&ClickEvent{ - MessageID: messageID, - Label: msg.Label, - HesitationTime: msg.HesitationTime, - Timestamp: b.timestamp, - Selector: msg.Selector, - }) - } - case *JSException: - b.appendReadyMessage(&ErrorEvent{ - MessageID: messageID, - Timestamp: b.timestamp, - Source: "js_exception", - Name: msg.Name, - Message: msg.Message, - Payload: msg.Payload, - }) - case *ResourceTiming: - tp := getResourceType(msg.Initiator, msg.URL) - success := msg.Duration != 0 - b.appendReadyMessage(&ResourceEvent{ - MessageID: messageID, - Timestamp: msg.Timestamp, - Duration: msg.Duration, - TTFB: msg.TTFB, - HeaderSize: msg.HeaderSize, - EncodedBodySize: msg.EncodedBodySize, - DecodedBodySize: msg.DecodedBodySize, - URL: msg.URL, - Type: tp, - Success: success, - }) - if !success { - issueType := "missing_resource" - if tp == "fetch" { - issueType = "bad_request" - } - b.appendReadyMessage(&IssueEvent{ - Type: issueType, - MessageID: messageID, - Timestamp: msg.Timestamp, - ContextString: msg.URL, - }) - } - case *RawCustomEvent: - b.appendReadyMessage(&CustomEvent{ - MessageID: messageID, - Timestamp: b.timestamp, - Name: msg.Name, - Payload: msg.Payload, - }) - case *CustomIssue: - b.appendReadyMessage(&IssueEvent{ - Type: "custom", - Timestamp: b.timestamp, - MessageID: messageID, - ContextString: msg.Name, - Payload: msg.Payload, - }) - case *Fetch: - b.appendReadyMessage(&FetchEvent{ - MessageID: messageID, - Timestamp: msg.Timestamp, - Method: msg.Method, - URL: msg.URL, - Request: msg.Request, - Response: msg.Response, - Status: msg.Status, - Duration: msg.Duration, - }) - if msg.Status >= 400 { - b.appendReadyMessage(&IssueEvent{ - Type: "bad_request", - MessageID: messageID, - Timestamp: msg.Timestamp, - ContextString: msg.URL, - }) - } - case *GraphQL: - b.appendReadyMessage(&GraphQLEvent{ - MessageID: messageID, - Timestamp: b.timestamp, - OperationKind: msg.OperationKind, - OperationName: msg.OperationName, - Variables: msg.Variables, - Response: msg.Response, - }) - case *StateAction: - b.appendReadyMessage(&StateActionEvent{ - MessageID: messageID, - Timestamp: b.timestamp, - Type: msg.Type, - }) - } - return nil -} - -func (b *builder) checkTimeouts(ts int64) bool { - if b.timestamp == 0 { - return false // There was no timestamp events yet - } - - if b.peBuilder.HasInstance() && int64(b.peBuilder.GetTimestamp())+intervals.EVENTS_PAGE_EVENT_TIMEOUT < ts { - b.buildPageEvent() - } - if b.ieBuilder.HasInstance() && int64(b.ieBuilder.GetTimestamp())+intervals.EVENTS_INPUT_EVENT_TIMEOUT < ts { - b.buildInputEvent() - } - return false -} diff --git a/backend/internal/handlers/custom/pageEventBuilder.go b/backend/internal/handlers/custom/pageEventBuilder.go index 765fd31a2..d95768983 100644 --- a/backend/internal/handlers/custom/pageEventBuilder.go +++ b/backend/internal/handlers/custom/pageEventBuilder.go @@ -4,104 +4,103 @@ import ( . "openreplay/backend/pkg/messages" ) +const PAGE_EVENT_TIMEOUT = 1 * 60 * 1000 + type pageEventBuilder struct { pageEvent *PageEvent firstTimingHandled bool } -func (b *pageEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message { - //TODO implement me - panic("implement me") -} - -func (b *pageEventBuilder) Build() Message { - // b.build() - //TODO implement me - panic("implement me") -} - func NewPageEventBuilder() *pageEventBuilder { ieBuilder := &pageEventBuilder{} return ieBuilder } -func (b *pageEventBuilder) buildIfTimingsComplete() *PageEvent { - if b.firstTimingHandled { - return b.build() +func (b *pageEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message { + switch msg := message.(type) { + case *SetPageLocation: + if msg.NavigationStart == 0 { // routing without new page loading + return &PageEvent{ + URL: msg.URL, + Referrer: msg.Referrer, + Loaded: false, + MessageID: messageID, + Timestamp: timestamp, + } + } else { + pageEvent := b.Build() + b.pageEvent = &PageEvent{ + URL: msg.URL, + Referrer: msg.Referrer, + Loaded: true, + MessageID: messageID, + Timestamp: timestamp, + } + return pageEvent + } + case *PageLoadTiming: + if b.pageEvent == nil { + break + } + if msg.RequestStart <= 30000 { + b.pageEvent.RequestStart = msg.RequestStart + } + if msg.ResponseStart <= 30000 { + b.pageEvent.ResponseStart = msg.ResponseStart + } + if msg.ResponseEnd <= 30000 { + b.pageEvent.ResponseEnd = msg.ResponseEnd + } + if msg.DomContentLoadedEventStart <= 30000 { + b.pageEvent.DomContentLoadedEventStart = msg.DomContentLoadedEventStart + } + if msg.DomContentLoadedEventEnd <= 30000 { + b.pageEvent.DomContentLoadedEventEnd = msg.DomContentLoadedEventEnd + } + if msg.LoadEventStart <= 30000 { + b.pageEvent.LoadEventStart = msg.LoadEventStart + } + if msg.LoadEventEnd <= 30000 { + b.pageEvent.LoadEventEnd = msg.LoadEventEnd + } + if msg.FirstPaint <= 30000 { + b.pageEvent.FirstPaint = msg.FirstPaint + } + if msg.FirstContentfulPaint <= 30000 { + b.pageEvent.FirstContentfulPaint = msg.FirstContentfulPaint + } + return b.buildIfTimingsComplete() + case *PageRenderTiming: + if b.pageEvent == nil { + break + } + b.pageEvent.SpeedIndex = msg.SpeedIndex + b.pageEvent.VisuallyComplete = msg.VisuallyComplete + b.pageEvent.TimeToInteractive = msg.TimeToInteractive + return b.buildIfTimingsComplete() + + } + + if b.pageEvent != nil && b.pageEvent.Timestamp+PAGE_EVENT_TIMEOUT < timestamp { + return b.Build() } - b.firstTimingHandled = true return nil } -// Only for Loaded: true -func (b *pageEventBuilder) HandleSetPageLocation(msg *SetPageLocation, messageID uint64, timestamp uint64) { - b.pageEvent = &PageEvent{ - URL: msg.URL, - Referrer: msg.Referrer, - Loaded: true, - MessageID: messageID, - Timestamp: timestamp, - } -} - -func (b *pageEventBuilder) HandlePageLoadTiming(msg *PageLoadTiming) *PageEvent { - if !b.HasInstance() { - return nil - } - if msg.RequestStart <= 30000 { - b.pageEvent.RequestStart = msg.RequestStart - } - if msg.ResponseStart <= 30000 { - b.pageEvent.ResponseStart = msg.ResponseStart - } - if msg.ResponseEnd <= 30000 { - b.pageEvent.ResponseEnd = msg.ResponseEnd - } - if msg.DomContentLoadedEventStart <= 30000 { - b.pageEvent.DomContentLoadedEventStart = msg.DomContentLoadedEventStart - } - if msg.DomContentLoadedEventEnd <= 30000 { - b.pageEvent.DomContentLoadedEventEnd = msg.DomContentLoadedEventEnd - } - if msg.LoadEventStart <= 30000 { - b.pageEvent.LoadEventStart = msg.LoadEventStart - } - if msg.LoadEventEnd <= 30000 { - b.pageEvent.LoadEventEnd = msg.LoadEventEnd - } - if msg.FirstPaint <= 30000 { - b.pageEvent.FirstPaint = msg.FirstPaint - } - if msg.FirstContentfulPaint <= 30000 { - b.pageEvent.FirstContentfulPaint = msg.FirstContentfulPaint - } - return b.buildIfTimingsComplete() -} - -func (b *pageEventBuilder) HandlePageRenderTiming(msg *PageRenderTiming) *PageEvent { - if !b.HasInstance() { - return nil - } - b.pageEvent.SpeedIndex = msg.SpeedIndex - b.pageEvent.VisuallyComplete = msg.VisuallyComplete - b.pageEvent.TimeToInteractive = msg.TimeToInteractive - return b.buildIfTimingsComplete() -} - -func (b *pageEventBuilder) HasInstance() bool { - return b.pageEvent != nil -} - -func (b *pageEventBuilder) GetTimestamp() uint64 { +func (b *pageEventBuilder) Build() Message { if b.pageEvent == nil { - return 0 + return nil } - return b.pageEvent.Timestamp -} - -func (b *pageEventBuilder) build() *PageEvent { pageEvent := b.pageEvent b.pageEvent = nil b.firstTimingHandled = false return pageEvent } + +func (b *pageEventBuilder) buildIfTimingsComplete() Message { + if b.firstTimingHandled { + return b.Build() + } + b.firstTimingHandled = true + return nil +} diff --git a/backend/internal/handlers/ios/performanceAggregator.go b/backend/internal/handlers/ios/performanceAggregator.go index 2a9401748..df87298bd 100644 --- a/backend/internal/handlers/ios/performanceAggregator.go +++ b/backend/internal/handlers/ios/performanceAggregator.go @@ -48,7 +48,7 @@ func (h *PerformanceAggregator) Handle(message Message, messageID uint64, timest h.pa.TimestampStart = m.Timestamp } if h.pa.TimestampStart+AGGR_TIME <= m.Timestamp { - event = h.build(m.Timestamp) + event = h.Build() } switch m.Name { case "fps": @@ -89,21 +89,17 @@ func (h *PerformanceAggregator) Handle(message Message, messageID uint64, timest } } case *IOSSessionEnd: - event = h.build(m.Timestamp) + event = h.Build() } return event } func (h *PerformanceAggregator) Build() Message { - return h.build(h.lastTimestamp) -} - -func (h *PerformanceAggregator) build(timestamp uint64) Message { if h.pa == nil { return nil } - h.pa.TimestampEnd = timestamp + h.pa.TimestampEnd = h.lastTimestamp h.pa.AvgFPS = h.fps.aggregate() h.pa.AvgCPU = h.cpu.aggregate() h.pa.AvgMemory = h.memory.aggregate() diff --git a/backend/internal/handlers/web/networkIssue.go b/backend/internal/handlers/web/networkIssue.go new file mode 100644 index 000000000..ed51351e5 --- /dev/null +++ b/backend/internal/handlers/web/networkIssue.go @@ -0,0 +1,47 @@ +package web + +import ( + . "openreplay/backend/pkg/messages" +) + +/* + Handler name: NetworkIssue + Input events: ResourceTiming, + Fetch + Output event: IssueEvent +*/ + +type NetworkIssueDetector struct{} + +func (f *NetworkIssueDetector) Build() Message { + return nil +} + +func (f *NetworkIssueDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { + switch msg := message.(type) { + case *ResourceTiming: + success := msg.Duration != 0 // The only available way here + if !success { + issueType := "missing_resource" + if msg.Initiator == "fetch" || msg.Initiator == "xmlhttprequest" { + issueType = "bad_request" + } + return &IssueEvent{ + Type: issueType, + MessageID: messageID, + Timestamp: msg.Timestamp, + ContextString: msg.URL, + } + } + case *Fetch: + if msg.Status >= 400 { + return &IssueEvent{ + Type: "bad_request", + MessageID: messageID, + Timestamp: msg.Timestamp, + ContextString: msg.URL, + } + } + } + return nil +} diff --git a/backend/pkg/intervals/intervals.go b/backend/pkg/intervals/intervals.go index 649ceca1a..226d79d35 100644 --- a/backend/pkg/intervals/intervals.go +++ b/backend/pkg/intervals/intervals.go @@ -3,8 +3,6 @@ package intervals const EVENTS_COMMIT_INTERVAL = 30 * 1000 // как часто комитим сообщения в кафке (ender) const HEARTBEAT_INTERVAL = 2 * 60 * 1000 // максимальный таймаут от трекера в рамках сессии const INTEGRATIONS_REQUEST_INTERVAL = 1 * 60 * 1000 // интеграции -const EVENTS_PAGE_EVENT_TIMEOUT = 2 * 60 * 1000 // таймаут пейдж ивента -const EVENTS_INPUT_EVENT_TIMEOUT = 2 * 60 * 1000 // const EVENTS_SESSION_END_TIMEOUT = HEARTBEAT_INTERVAL + 30*1000 const EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS = HEARTBEAT_INTERVAL + 3*60*1000 const EVENTS_BACK_COMMIT_GAP = EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS + 1*60*1000 // для бэк коммита From ae6af1449c8d30864155859ff3e8f14b1c3f5b9a Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Thu, 12 May 2022 09:59:09 +0200 Subject: [PATCH 18/25] feat(backend-db/heuristics): fixed errors in main files --- backend/cmd/db/main.go | 4 ++-- backend/cmd/heuristics/main.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index d3d786242..20d6ce55b 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -31,9 +31,9 @@ func main() { defer pg.Close() // HandlersFabric returns the list of message handlers we want to be applied to each incoming message. - handlersFabric := func() { + handlersFabric := func() []handlers.MessageProcessor { return []handlers.MessageProcessor{ - custom.EventMapper{}, + &custom.EventMapper{}, custom.NewInputEventBuilder(), custom.NewPageEventBuilder(), } diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index 6712c927c..6edf01a92 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -26,7 +26,7 @@ func main() { cfg := ender.New() // HandlersFabric returns the list of message handlers we want to be applied to each incoming message. - handlersFabric := func() { + handlersFabric := func() []handlers.MessageProcessor { return []handlers.MessageProcessor{ // web handlers &web.ClickRageDetector{}, From 44dae11886839152c84ce6f01218aa7be9e07f27 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 13 May 2022 17:00:09 +0200 Subject: [PATCH 19/25] feat(backend/db): fixed ee version --- backend/cmd/db/main.go | 8 +-- backend/internal/datasaver/stats.go | 8 +++ ee/backend/internal/datasaver/stats.go | 79 ++++++++++++++++++++++++ ee/backend/services/db/stats.go | 83 -------------------------- 4 files changed, 91 insertions(+), 87 deletions(-) create mode 100644 ee/backend/internal/datasaver/stats.go delete mode 100644 ee/backend/services/db/stats.go diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index 20d6ce55b..1c1d3bf0e 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -44,6 +44,7 @@ func main() { // Init modules saver := datasaver.New(pg) + saver.InitStats() statsLogger := logger.NewQueueStats(cfg.LoggerTimeout) // Handler logic @@ -117,10 +118,9 @@ func main() { os.Exit(0) case <-commitTick: pg.CommitBatches() - // TODO: ee commit stats !!! - //if err := commitStats(); err != nil { - // log.Printf("Error on stats commit: %v", err) - //} + if err := saver.CommitStats(); err != nil { + log.Printf("Error on stats commit: %v", err) + } // TODO?: separate stats & regular messages if err := consumer.Commit(); err != nil { log.Printf("Error on consumer commit: %v", err) diff --git a/backend/internal/datasaver/stats.go b/backend/internal/datasaver/stats.go index a57d91824..26efe51b5 100644 --- a/backend/internal/datasaver/stats.go +++ b/backend/internal/datasaver/stats.go @@ -5,6 +5,10 @@ import ( . "openreplay/backend/pkg/messages" ) +func (si *Saver) InitStats() { + // noop +} + func (si *Saver) InsertStats(session *Session, msg Message) error { switch m := msg.(type) { // Web @@ -17,3 +21,7 @@ func (si *Saver) InsertStats(session *Session, msg Message) error { } return nil } + +func (si *Saver) CommitStats() error { + return nil +} diff --git a/ee/backend/internal/datasaver/stats.go b/ee/backend/internal/datasaver/stats.go new file mode 100644 index 000000000..501a861aa --- /dev/null +++ b/ee/backend/internal/datasaver/stats.go @@ -0,0 +1,79 @@ +package datasaver + +import ( + "log" + "time" + + "openreplay/backend/pkg/db/clickhouse" + "openreplay/backend/pkg/env" +) + +var ch *clickhouse.Connector +var finalizeTicker <-chan time.Time + +func (si *Saver) InitStats() { + ch = clickhouse.NewConnector(env.String("CLICKHOUSE_STRING")) + if err := ch.Prepare(); err != nil { + log.Fatalf("Clickhouse prepare error: %v\n", err) + } + + finalizeTicker = time.Tick(20 * time.Minute) + +} + +func (si *Saver) InsertStats(session *Session, msg Message) error { + switch m := msg.(type) { + // Web + case *SessionEnd: + return si.pg.InsertWebSession(session) + case *PerformanceTrackAggr: + return si.pg.InsertWebPerformanceTrackAggr(session, m) + case *ClickEvent: + return si.pg.InsertWebClickEvent(session, m) + case *InputEvent: + return si.pg.InsertWebInputEvent(session, m) + // Unique for Web + case *PageEvent: + si.pg.InsertWebPageEvent(session, m) + case *ResourceEvent: + return si.pg.InsertWebResourceEvent(session, m) + case *ErrorEvent: + return si.pg.InsertWebErrorEvent(session, m) + case *LongTask: + return si.pg.InsertLongtask(session, m) + + // IOS + case *IOSSessionEnd: + return si.pg.InsertIOSSession(session) + case *IOSPerformanceAggregated: + return si.pg.InsertIOSPerformanceAggregated(session, m) + case *IOSClickEvent: + return si.pg.InsertIOSClickEvent(session, m) + case *IOSInputEvent: + return si.pg.InsertIOSInputEvent(session, m) + // Unique for Web + case *IOSScreenEnter: + //ch.InsertIOSView(session, m) + case *IOSCrash: + return si.pg.InsertIOSCrash(session, m) + case *IOSNetworkCall: + return si.pg.InsertIOSNetworkCall(session, m) + } + return nil +} + +func (si *Saver) CommitStats() error { + select { + case <-finalizeTicker: + if err := ch.FinaliseSessionsTable(); err != nil { + log.Printf("Stats: FinaliseSessionsTable returned an error. %v", err) + } + default: + } + errCommit := ch.Commit() + errPrepare := ch.Prepare() + if errCommit != nil { + return errCommit + } + return errPrepare +} diff --git a/ee/backend/services/db/stats.go b/ee/backend/services/db/stats.go deleted file mode 100644 index 9d250fc51..000000000 --- a/ee/backend/services/db/stats.go +++ /dev/null @@ -1,83 +0,0 @@ -package main - -import ( - "log" - "time" - - - . "openreplay/backend/pkg/messages" - . "openreplay/backend/pkg/db/types" - "openreplay/backend/pkg/db/clickhouse" - "openreplay/backend/pkg/env" -) - -var ch *clickhouse.Connector -var finalizeTicker <-chan time.Time - -func initStats() { - ch = clickhouse.NewConnector(env.String("CLICKHOUSE_STRING")) - if err := ch.Prepare(); err != nil { - log.Fatalf("Clickhouse prepare error: %v\n", err) - } - - finalizeTicker = time.Tick(20 * time.Minute) - -} - -func insertStats(session *Session, msg Message) error { - switch m := msg.(type) { - // Web - case *SessionEnd: - return ch.InsertWebSession(session) - case *PerformanceTrackAggr: - return ch.InsertWebPerformanceTrackAggr(session, m) - case *ClickEvent: - return ch.InsertWebClickEvent(session, m) - case *InputEvent: - return ch.InsertWebInputEvent(session, m) - // Unique for Web - case *PageEvent: - ch.InsertWebPageEvent(session, m) - case *ResourceEvent: - return ch.InsertWebResourceEvent(session, m) - case *ErrorEvent: - return ch.InsertWebErrorEvent(session, m) - case *LongTask: - return ch.InsertLongtask(session, m) - - // IOS - case *IOSSessionEnd: - return ch.InsertIOSSession(session) - case *IOSPerformanceAggregated: - return ch.InsertIOSPerformanceAggregated(session, m) - case *IOSClickEvent: - return ch.InsertIOSClickEvent(session, m) - case *IOSInputEvent: - return ch.InsertIOSInputEvent(session, m) - // Unique for Web - case *IOSScreenEnter: - //ch.InsertIOSView(session, m) - case *IOSCrash: - return ch.InsertIOSCrash(session, m) - case *IOSNetworkCall: - return ch.InsertIOSNetworkCall(session, m) - } - return nil -} - -func commitStats() error { - select { - case <-finalizeTicker: - if err := ch.FinaliseSessionsTable(); err != nil { - log.Printf("Stats: FinaliseSessionsTable returned an error. %v", err) - } - default: - } - errCommit := ch.Commit() - errPrepare := ch.Prepare() - if errCommit != nil { - return errCommit - } - return errPrepare -} - From c94f4074bbe210d19c62b3ffb5c97015859912a1 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 16 May 2022 10:45:51 +0200 Subject: [PATCH 20/25] chore(helm): make ingress-nginx installation not mandatory. Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/Chart.yaml | 1 + scripts/helmcharts/openreplay/values.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/scripts/helmcharts/openreplay/Chart.yaml b/scripts/helmcharts/openreplay/Chart.yaml index c1875102e..a9a2f8e83 100644 --- a/scripts/helmcharts/openreplay/Chart.yaml +++ b/scripts/helmcharts/openreplay/Chart.yaml @@ -28,3 +28,4 @@ dependencies: - name: ingress-nginx version: "4.x.x" repository: "https://kubernetes.github.io/ingress-nginx" + condition: ingress-nginx.enabled diff --git a/scripts/helmcharts/openreplay/values.yaml b/scripts/helmcharts/openreplay/values.yaml index f702c1a49..3c063340c 100644 --- a/scripts/helmcharts/openreplay/values.yaml +++ b/scripts/helmcharts/openreplay/values.yaml @@ -114,3 +114,5 @@ assist: fullnameOverride: assist-openreplay peers: fullnameOverride: peers-openreplay +ingress-nginx: + enabled: true From 4175d98be8180c7252cd2ff5e2a3edf07b7ab0c8 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 11 May 2022 21:13:06 +0200 Subject: [PATCH 21/25] chore(helmcharts): adding clickhouse operator helm chart Signed-off-by: rjshrjndrn --- .../databases/charts/clickhouse/Chart.yaml | 7 +- .../altinity-clickhouse-operator/Chart.yaml | 13 + .../altinity-clickhouse-operator/README.md | 49 + ...installations.clickhouse.altinity.com.yaml | 1321 +++++++++++++++++ ...tiontemplates.clickhouse.altinity.com.yaml | 1321 +++++++++++++++++ ...onfigurations.clickhouse.altinity.com.yaml | 284 ++++ .../hacks/sync-yamls.sh | 58 + .../templates/_helpers.tpl | 73 + ...rRole-clickhouse-operator-kube-system.yaml | 163 ++ ...nding-clickhouse-operator-kube-system.yaml | 17 + ...p-etc-clickhouse-operator-confd-files.yaml | 18 + ...etc-clickhouse-operator-configd-files.yaml | 59 + ...nfigMap-etc-clickhouse-operator-files.yaml | 230 +++ ...-clickhouse-operator-templatesd-files.yaml | 102 ++ ...-etc-clickhouse-operator-usersd-files.yaml | 62 + .../Deployment-clickhouse-operator.yaml | 175 +++ .../Service-clickhouse-operator-metrics.yaml | 25 + .../ServiceAccount-clickhouse-operator.yaml | 21 + .../templates/servicemonitor.yaml | 18 + .../altinity-clickhouse-operator/values.yaml | 88 ++ .../templates/clickhouse-cluster.yaml | 93 ++ .../charts/clickhouse/templates/service.yaml | 19 - .../clickhouse/templates/serviceaccount.yaml | 12 - .../clickhouse/templates/statefulset.yaml | 69 - .../databases/charts/clickhouse/values.yaml | 58 +- 25 files changed, 4203 insertions(+), 152 deletions(-) create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/Chart.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/README.md create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/hacks/sync-yamls.sh create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/_helpers.tpl create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/Service-clickhouse-operator-metrics.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/servicemonitor.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/values.yaml create mode 100644 scripts/helmcharts/databases/charts/clickhouse/templates/clickhouse-cluster.yaml delete mode 100644 scripts/helmcharts/databases/charts/clickhouse/templates/service.yaml delete mode 100644 scripts/helmcharts/databases/charts/clickhouse/templates/serviceaccount.yaml delete mode 100644 scripts/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml diff --git a/scripts/helmcharts/databases/charts/clickhouse/Chart.yaml b/scripts/helmcharts/databases/charts/clickhouse/Chart.yaml index c7a0eb3d6..4026f2308 100644 --- a/scripts/helmcharts/databases/charts/clickhouse/Chart.yaml +++ b/scripts/helmcharts/databases/charts/clickhouse/Chart.yaml @@ -20,4 +20,9 @@ version: 0.1.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 1.16.0 +# It is recommended to use it with quotes. +appVersion: "1.16.0" + +dependencies: + - name: altinity-clickhouse-operator + version: 0.0.14 diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/Chart.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/Chart.yaml new file mode 100644 index 000000000..5556de4a7 --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +appVersion: 0.18.1 +description: Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator). The + ClickHouse Operator creates, configures and manages ClickHouse clusters running + on Kubernetes. Refer to operator repo for additional information. +home: https://github.com/slamdev/helm-charts/tree/master/charts/altinity-clickhouse-operator +icon: https://artifacthub.io/image/2d6aa29c-c74f-4bff-bede-ba7e6e0315a7@2x +maintainers: +- email: valentin.fedoskin@gmail.com + name: slamdev +name: altinity-clickhouse-operator +type: application +version: 0.0.14 diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/README.md b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/README.md new file mode 100644 index 000000000..a05913517 --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/README.md @@ -0,0 +1,49 @@ +# altinity-clickhouse-operator + +![Version: 0.0.13](https://img.shields.io/badge/Version-0.0.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.18.1](https://img.shields.io/badge/AppVersion-0.18.1-informational?style=flat-square) + +Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator). + +The ClickHouse Operator creates, configures and manages ClickHouse clusters running on Kubernetes. + +Refer to operator repo for additional information. + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| slamdev | valentin.fedoskin@gmail.com | | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | affinity for scheduler pod assignment | +| configs | object | `{"confdFiles":null,"configdFiles":null,"files":null,"templatesdFiles":null,"usersdFiles":null}` | overrides operator default configmaps | +| fullnameOverride | string | `""` | full name of the chart. | +| imagePullSecrets | list | `[]` | image pull secret for private images | +| metrics.env | list | `[]` | additional environment variables for the deployment | +| metrics.image.pullPolicy | string | `"IfNotPresent"` | image pull policy | +| metrics.image.repository | string | `"altinity/metrics-exporter"` | image repository | +| metrics.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) | +| metrics.resources | object | `{}` | custom resource configuration | +| metrics.command | list | `nil` | command for metrics-exporter container | +| metrics.args | list | `nil` | args for metrics-exporter container | +| nameOverride | string | `""` | override name of the chart | +| nodeSelector | object | `{}` | node for scheduler pod assignment | +| operator.env | list | `[]` | additional environment variables for the deployment | +| operator.image.pullPolicy | string | `"IfNotPresent"` | image pull policy | +| operator.image.repository | string | `"altinity/clickhouse-operator"` | image repository | +| operator.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) | +| operator.resources | object | `{}` | custom resource configuration | +| operator.command | list | `nil` | command for operator container | +| operator.args | list | `nil` | args for operator container | +| podAnnotations | object | `nil` | additional pod annotations | +| serviceAccount.annotations | object | `{}` | annotations to add to the service account | +| serviceAccount.create | bool | `true` | specifies whether a service account should be created | +| serviceAccount.name | string | `nil` | the name of the service account to use; if not set and create is true, a name is generated using the fullname template | +| serviceMonitor.additionalLabels | object | `{}` | additional labels for service monitor | +| serviceMonitor.enabled | bool | `false` | ServiceMonitor CRD is created for a prometheus operator | +| tolerations | list | `[]` | tolerations for scheduler pod assignment | diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml new file mode 100644 index 000000000..908961576 --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml @@ -0,0 +1,1321 @@ +# Template Parameters: +# +# KIND=ClickHouseInstallation +# SINGULAR=clickhouseinstallation +# PLURAL=clickhouseinstallations +# SHORT=chi +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.18.1 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallation + singular: clickhouseinstallation + plural: clickhouseinstallations + shortNames: + - chi + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + priority: 0 # show in standard view + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + priority: 0 # show in standard view + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: status + type: string + description: CHI status + priority: 0 # show in standard view + jsonPath: .status.status + - name: updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.updated + - name: added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.added + - name: deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.deleted + - name: delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.delete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + type: object + description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" + properties: + chop-version: + type: string + description: "ClickHouse operator version" + chop-commit: + type: string + description: "ClickHouse operator git commit SHA" + chop-date: + type: string + description: "ClickHouse operator build date" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + items: + type: string + updated: + type: integer + minimum: 0 + description: "Updated Hosts count" + added: + type: integer + minimum: 0 + description: "Added Hosts count" + deleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + delete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + items: + type: string + endpoint: + type: string + description: "Endpoint" + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized CHI" + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md" + properties: + taskID: + type: string + description: "Allow define custom taskID for named update and watch status of this update execution in .status.taskIDs field, by default every update of chi manifest will generate random taskID" + # Need to be StringBool + stop: + type: string + description: | + Allow stop all ClickHouse clusters described in current chi. + Stop mechanism works as follows: + - When `stop` is `1` then setup `Replicas: 0` in each related to current `chi` StatefulSet resource, all `Pods` and `Service` resources will desctroy, but PVCs still live + - When `stop` is `0` then `Pods` will created again and will attach retained PVCs and `Service` also will created again + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + restart: + type: string + description: "This is a 'soft restart' button. When set to 'RollingUpdate' operator will restart ClickHouse pods in a graceful way. Remove it after the use in order to avoid unneeded restarts" + enum: + - "" + - "RollingUpdate" + # Need to be StringBool + troubleshoot: + type: string + description: "allows troubleshoot Pods during CrashLoopBack state, when you apply wrong configuration, `clickhouse-server` wouldn't startup" + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + namespaceDomainPattern: + type: string + description: "custom domain suffix which will add to end of `Service` or `Pod` name, use it when you use custom cluster domain in your Kubernetes cluster" + templating: + type: object + # nullable: true + description: "optional, define policy for auto applying ClickHouseInstallationTemplate inside ClickHouseInstallation" + properties: + policy: + type: string + description: "when defined as `auto` inside ClickhouseInstallationTemplate, it will auto add into all ClickHouseInstallation, manual value is default" + enum: + - "auto" + - "manual" + reconciling: + type: object + description: "optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: DEPRECATED + configMapPropagationTimeout: + type: integer + description: | + timeout in seconds when `clickhouse-operator` will wait when applied `ConfigMap` during reconcile `ClickhouseInstallation` pods will updated from cache + see details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "optional, define behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: "what clickhouse-operator shall do when found Kubernetes resources which should be managed with clickhouse-operator, but not have `ownerReference` to any currently managed `ClickHouseInstallation` resource, default behavior is `Delete`" + # nullable: true + properties: + statefulSet: + type: string + description: "behavior policy for unknown StatefulSet, Delete by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + pvc: + type: string + description: "behavior policy for unknown PVC, Delete by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + configMap: + type: string + description: "behavior policy for unknown ConfigMap, Delete by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + service: + type: string + description: "behavior policy for unknown Service, Delete by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + reconcileFailedObjects: + type: object + description: "what clickhouse-operator shall do when reconciling Kubernetes resources are failed, default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + type: string + description: "behavior policy for failed StatefulSet reconciling, Retain by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + pvc: + type: string + description: "behavior policy for failed PVC reconciling, Retain by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + configMap: + type: string + description: "behavior policy for failed ConfigMap reconciling, Retain by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + service: + type: string + description: "behavior policy for failed Service reconciling, Retain by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + # Need to be StringBool + replicasUseFQDN: + type: string + description: | + define should replicas be specified by FQDN in ``, then "no" then will use short hostname and clickhouse-server will use kubernetes default suffixes for properly DNS lookup + "yes" by default + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + templates: + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + type: object + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + settings: + type: object + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: + type: object + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + # nullable: true + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure each `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod` only for one cluster" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one cluster" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one cluster" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one cluster" + serviceTemplate: + type: string + description: "optional, fully ignores for cluster-level" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters` only for one cluster" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one cluster" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters` only for one cluster" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + type: + type: string + description: "DEPRECATED - to be removed soon" + shardsCount: + type: integer + description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" + replicasCount: + type: integer + description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" + shards: + type: array + description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + # Need to be StringBool + internalReplication: + type: string + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + settings: + type: object + # nullable: true + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + x-kubernetes-preserve-unknown-fields: true + files: + type: object + # nullable: true + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure each `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod` only for one shard" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + serviceTemplate: + type: string + description: "optional, fully ignores for shard-level" + clusterServiceTemplate: + type: string + description: "optional, fully ignores for shard-level" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one shard" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one shard" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + type: object + # nullable: true + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + x-kubernetes-preserve-unknown-fields: true + files: + type: object + # nullable: true + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod` only for one replica" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one replica" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + serviceTemplate: + type: string + description: "optional, fully ignores for replica-level" + clusterServiceTemplate: + type: string + description: "optional, fully ignores for replica-level" + shardServiceTemplate: + type: string + description: "optional, fully ignores for replica-level" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one replica" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + type: object + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: + type: object + # nullable: true + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod` only for one replica" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one replica" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + serviceTemplate: + type: string + description: "optional, fully ignores for replica-level" + clusterServiceTemplate: + type: string + description: "optional, fully ignores for replica-level" + shardServiceTemplate: + type: string + description: "optional, fully ignores for replica-level" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one replica" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + type: object + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: + type: object + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + # nullable: true + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure each `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod` only for one shard" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + serviceTemplate: + type: string + description: "optional, fully ignores for shard-level" + clusterServiceTemplate: + type: string + description: "optional, fully ignores for shard-level" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one shard" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one shard" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + type: object + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: + type: object + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + # nullable: true + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: "be carefull, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + # nullable: true + properties: + hostTemplate: + type: string + podTemplate: + type: string + dataVolumeClaimTemplate: + type: string + logVolumeClaimTemplate: + type: string + serviceTemplate: + type: string + clusterServiceTemplate: + type: string + shardServiceTemplate: + type: string + replicaServiceTemplate: + type: string + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distibution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + type: string + reclaimPolicy: + type: string + description: "define behavior of `PVC` deletion policy during delete `Pod`, `Delete` by default, when `Retain` then `PVC` still alive even `Pod` will deleted" + enum: + - "" + - "Retain" + - "Delete" + metadata: + type: object + description: | + allows pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + enum: + # List useTypeXXX constants from model + - "" + - "merge" diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml new file mode 100644 index 000000000..7899b71de --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml @@ -0,0 +1,1321 @@ +# Template Parameters: +# +# KIND=ClickHouseInstallationTemplate +# SINGULAR=clickhouseinstallationtemplate +# PLURAL=clickhouseinstallationtemplates +# SHORT=chit +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallationtemplates.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.18.1 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallationTemplate + singular: clickhouseinstallationtemplate + plural: clickhouseinstallationtemplates + shortNames: + - chit + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + priority: 0 # show in standard view + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + priority: 0 # show in standard view + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: status + type: string + description: CHI status + priority: 0 # show in standard view + jsonPath: .status.status + - name: updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.updated + - name: added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.added + - name: deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.deleted + - name: delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.delete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + type: object + description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" + properties: + chop-version: + type: string + description: "ClickHouse operator version" + chop-commit: + type: string + description: "ClickHouse operator git commit SHA" + chop-date: + type: string + description: "ClickHouse operator build date" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + items: + type: string + updated: + type: integer + minimum: 0 + description: "Updated Hosts count" + added: + type: integer + minimum: 0 + description: "Added Hosts count" + deleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + delete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + items: + type: string + endpoint: + type: string + description: "Endpoint" + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized CHI" + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md" + properties: + taskID: + type: string + description: "Allow define custom taskID for named update and watch status of this update execution in .status.taskIDs field, by default every update of chi manifest will generate random taskID" + # Need to be StringBool + stop: + type: string + description: | + Allow stop all ClickHouse clusters described in current chi. + Stop mechanism works as follows: + - When `stop` is `1` then setup `Replicas: 0` in each related to current `chi` StatefulSet resource, all `Pods` and `Service` resources will desctroy, but PVCs still live + - When `stop` is `0` then `Pods` will created again and will attach retained PVCs and `Service` also will created again + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + restart: + type: string + description: "This is a 'soft restart' button. When set to 'RollingUpdate' operator will restart ClickHouse pods in a graceful way. Remove it after the use in order to avoid unneeded restarts" + enum: + - "" + - "RollingUpdate" + # Need to be StringBool + troubleshoot: + type: string + description: "allows troubleshoot Pods during CrashLoopBack state, when you apply wrong configuration, `clickhouse-server` wouldn't startup" + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + namespaceDomainPattern: + type: string + description: "custom domain suffix which will add to end of `Service` or `Pod` name, use it when you use custom cluster domain in your Kubernetes cluster" + templating: + type: object + # nullable: true + description: "optional, define policy for auto applying ClickHouseInstallationTemplate inside ClickHouseInstallation" + properties: + policy: + type: string + description: "when defined as `auto` inside ClickhouseInstallationTemplate, it will auto add into all ClickHouseInstallation, manual value is default" + enum: + - "auto" + - "manual" + reconciling: + type: object + description: "optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: DEPRECATED + configMapPropagationTimeout: + type: integer + description: | + timeout in seconds when `clickhouse-operator` will wait when applied `ConfigMap` during reconcile `ClickhouseInstallation` pods will updated from cache + see details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "optional, define behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: "what clickhouse-operator shall do when found Kubernetes resources which should be managed with clickhouse-operator, but not have `ownerReference` to any currently managed `ClickHouseInstallation` resource, default behavior is `Delete`" + # nullable: true + properties: + statefulSet: + type: string + description: "behavior policy for unknown StatefulSet, Delete by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + pvc: + type: string + description: "behavior policy for unknown PVC, Delete by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + configMap: + type: string + description: "behavior policy for unknown ConfigMap, Delete by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + service: + type: string + description: "behavior policy for unknown Service, Delete by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + reconcileFailedObjects: + type: object + description: "what clickhouse-operator shall do when reconciling Kubernetes resources are failed, default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + type: string + description: "behavior policy for failed StatefulSet reconciling, Retain by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + pvc: + type: string + description: "behavior policy for failed PVC reconciling, Retain by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + configMap: + type: string + description: "behavior policy for failed ConfigMap reconciling, Retain by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + service: + type: string + description: "behavior policy for failed Service reconciling, Retain by default" + enum: + # List ObjectsCleanupXXX constants from model + - "Retain" + - "Delete" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + # Need to be StringBool + replicasUseFQDN: + type: string + description: | + define should replicas be specified by FQDN in ``, then "no" then will use short hostname and clickhouse-server will use kubernetes default suffixes for properly DNS lookup + "yes" by default + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + templates: + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + type: object + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + settings: + type: object + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: + type: object + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + # nullable: true + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure each `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod` only for one cluster" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one cluster" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one cluster" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one cluster" + serviceTemplate: + type: string + description: "optional, fully ignores for cluster-level" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters` only for one cluster" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one cluster" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters` only for one cluster" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + type: + type: string + description: "DEPRECATED - to be removed soon" + shardsCount: + type: integer + description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" + replicasCount: + type: integer + description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" + shards: + type: array + description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + # Need to be StringBool + internalReplication: + type: string + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + settings: + type: object + # nullable: true + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + x-kubernetes-preserve-unknown-fields: true + files: + type: object + # nullable: true + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure each `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod` only for one shard" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + serviceTemplate: + type: string + description: "optional, fully ignores for shard-level" + clusterServiceTemplate: + type: string + description: "optional, fully ignores for shard-level" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one shard" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one shard" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + type: object + # nullable: true + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + x-kubernetes-preserve-unknown-fields: true + files: + type: object + # nullable: true + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod` only for one replica" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one replica" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + serviceTemplate: + type: string + description: "optional, fully ignores for replica-level" + clusterServiceTemplate: + type: string + description: "optional, fully ignores for replica-level" + shardServiceTemplate: + type: string + description: "optional, fully ignores for replica-level" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one replica" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + type: object + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: + type: object + # nullable: true + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod` only for one replica" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one replica" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + serviceTemplate: + type: string + description: "optional, fully ignores for replica-level" + clusterServiceTemplate: + type: string + description: "optional, fully ignores for replica-level" + shardServiceTemplate: + type: string + description: "optional, fully ignores for replica-level" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one replica" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + type: object + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: + type: object + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + # nullable: true + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure each `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod` only for one shard" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters` only for one shard" + serviceTemplate: + type: string + description: "optional, fully ignores for shard-level" + clusterServiceTemplate: + type: string + description: "optional, fully ignores for shard-level" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one shard" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside clickhouse cluster described in `chi.spec.configuration.clusters` only for one shard" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + type: object + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: + type: object + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + # nullable: true + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: "be carefull, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + # nullable: true + properties: + hostTemplate: + type: string + podTemplate: + type: string + dataVolumeClaimTemplate: + type: string + logVolumeClaimTemplate: + type: string + serviceTemplate: + type: string + clusterServiceTemplate: + type: string + shardServiceTemplate: + type: string + replicaServiceTemplate: + type: string + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distibution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + type: string + reclaimPolicy: + type: string + description: "define behavior of `PVC` deletion policy during delete `Pod`, `Delete` by default, when `Retain` then `PVC` still alive even `Pod` will deleted" + enum: + - "" + - "Retain" + - "Delete" + metadata: + type: object + description: | + allows pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + enum: + # List useTypeXXX constants from model + - "" + - "merge" diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml new file mode 100644 index 000000000..515a4c608 --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -0,0 +1,284 @@ +# Template Parameters: +# +# NONE +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseoperatorconfigurations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.18.1 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseOperatorConfiguration + singular: clickhouseoperatorconfiguration + plural: clickhouseoperatorconfigurations + shortNames: + - chopconf + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: namespaces + type: string + description: Watch namespaces + priority: 0 # show in standard view + jsonPath: .status + schema: + openAPIV3Schema: + type: object + description: "allows customize `clickhouse-operator` settings, need restart clickhouse-operator pod after adding, more details https://github.com/Altinity/clickhouse-operator/blob/master/docs/operator_configuration.md" + x-kubernetes-preserve-unknown-fields: true + properties: + status: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + Allows to define settings of the clickhouse-operator. + More info: https://github.com/Altinity/clickhouse-operator/blob/master/config/config.yaml + Check into etc-clickhouse-operator* ConfigMaps if you need more control + x-kubernetes-preserve-unknown-fields: true + properties: + watch: + type: object + properties: + namespaces: + type: array + description: "List of namespaces where clickhouse-operator watches for events." + items: + type: string + clickhouse: + type: object + properties: + configuration: + type: object + properties: + file: + type: object + properties: + path: + type: object + properties: + common: + type: string + description: "Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. Default - config.d" + host: + type: string + description: "Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. Default - conf.d" + user: + type: string + description: "Path to the folder where ClickHouse configuration files with users settings are located. Files are common for all instances within a CHI." + user: + type: object + properties: + default: + type: object + properties: + profile: + type: string + description: "ClickHouse server configuration `...` for any " + quota: + type: string + description: "ClickHouse server configuration `...` for any " + networksIP: + type: array + description: "ClickHouse server configuration `...` for any " + items: + type: string + password: + type: string + description: "ClickHouse server configuration `...` for any " + network: + type: object + properties: + hostRegexpTemplate: + type: string + description: "ClickHouse server configuration `...` for any " + access: + type: object + properties: + username: + type: string + description: "ClickHouse username to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + password: + type: string + description: "ClickHouse password to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + secret: + type: object + properties: + namespace: + type: string + description: "Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + name: + type: string + description: "Name of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + port: + type: integer + minimum: 1 + maximum: 65535 + description: "port to be used by operator to connect to ClickHouse instances" + template: + type: object + properties: + chi: + type: object + properties: + path: + type: string + description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." + reconcile: + type: object + properties: + runtime: + type: object + properties: + threadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile in parallel, 10 by default" + statefulSet: + type: object + properties: + create: + type: object + properties: + onFailure: + type: string + description: | + What to do in case created StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. delete - delete newly created problematic StatefulSet. + 3. ignore (default) - ignore error, pretend nothing happened and move on to the next StatefulSet. + update: + type: object + properties: + timeout: + type: integer + description: "How many seconds to wait for created/updated StatefulSet to be Ready" + pollInterval: + type: integer + description: "How many seconds to wait between checks for created/updated StatefulSet status" + onFailure: + type: string + description: | + What to do in case updated StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. rollback (default) - delete Pod and rollback StatefulSet to previous Generation. Pod would be recreated by StatefulSet based on rollback-ed configuration. + 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet. + host: + type: object + properties: + wait: + type: object + properties: + exclude: + type: boolean + include: + type: boolean + annotation: + type: object + properties: + include: + type: array + items: + type: string + exclude: + type: array + items: + type: string + label: + type: object + properties: + include: + type: array + items: + type: string + exclude: + type: array + items: + type: string + description: | + When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, + exclude labels from the following list + appendScope: + type: string + description: | + Whether to append *Scope* labels to StatefulSet and Pod + - "LabelShardScopeIndex" + - "LabelReplicaScopeIndex" + - "LabelCHIScopeIndex" + - "LabelCHIScopeCycleSize" + - "LabelCHIScopeCycleIndex" + - "LabelCHIScopeCycleOffset" + - "LabelClusterScopeIndex" + - "LabelClusterScopeCycleSize" + - "LabelClusterScopeCycleIndex" + - "LabelClusterScopeCycleOffset" + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + statefulSet: + type: object + properties: + revisionHistoryLimit: + type: integer + pod: + type: object + properties: + terminationGracePeriod: + type: integer + logger: + type: object + properties: + logtostderr: + type: string + description: "boolean, allows logs to stderr" + alsologtostderr: + type: string + description: "boolean allows logs to stderr and files both" + v: + type: string + description: "verbosity level of clickhouse-operator log, default - 1 max - 9" + stderrthreshold: + type: string + vmodule: + type: string + description: | + Comma-separated list of filename=N, where filename (can be a pattern) must have no .go ext, and N is a V level. + Ex.: file*=2 sets the 'V' to 2 in all files with names like file*. + log_backtrace_at: + type: string + description: | + It can be set to a file and line number with a logging line. + Ex.: file.go:123 + Each time when this line is being executed, a stack trace will be written to the Info log. diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/hacks/sync-yamls.sh b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/hacks/sync-yamls.sh new file mode 100644 index 000000000..12b38d39e --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/hacks/sync-yamls.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# +# Script downloads manifest from altinity repo, splits it to separate files +# and puts to the corresponding folders +# NOTE: yq ( https://mikefarah.gitbook.io/yq/ ) > v4.14.x is required +# +# Usage: ./sync.sh +# + +set -o errexit +set -o nounset +set -o pipefail + +readonly repo_url="https://raw.githubusercontent.com/Altinity/clickhouse-operator" +readonly crds_dir="../crds" +readonly templates_dir="../templates/generated" +readonly manifest_path="deploy/operator/clickhouse-operator-install-bundle.yaml" +readonly chart_def="../Chart.yaml" + +function main() { + readonly manifest_url="${repo_url}/$(detect_version)/${manifest_path}" + local tmpdir + tmpdir=$(mktemp -d) + + # shellcheck disable=SC2016 + (cd "${tmpdir}" && curl -s "${manifest_url}" 2>&1 | yq e --no-doc -s '$index') + + for f in "${tmpdir}"/*.yml; do + process "${f}" + done +} + +function process() { + local file="${1}" + + local kind + kind=$(yq e '.kind' "${file}") + + local name + name=$(yq e '.metadata.name' "${file}") + + local processed_file="${kind}-${name}.yaml" + + if [[ "${kind}" == "CustomResourceDefinition" ]]; then + processed_file="${crds_dir}/${processed_file}" + else + processed_file="${templates_dir}/${processed_file}" + fi + + mv "${file}" "${processed_file}" +} + +function detect_version() { + yq e '.appVersion' ${chart_def} +} + +main diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/_helpers.tpl b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/_helpers.tpl new file mode 100644 index 000000000..f1cd6318a --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/_helpers.tpl @@ -0,0 +1,73 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "altinity-clickhouse-operator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "altinity-clickhouse-operator.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "altinity-clickhouse-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "altinity-clickhouse-operator.labels" -}} +helm.sh/chart: {{ include "altinity-clickhouse-operator.chart" . }} +{{ include "altinity-clickhouse-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "altinity-clickhouse-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "altinity-clickhouse-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "altinity-clickhouse-operator.serviceAccountName" -}} + {{ default (include "altinity-clickhouse-operator.fullname" .) .Values.serviceAccount.name }} +{{- end -}} + +{{/* +Create the tag for the docker image to use +*/}} +{{- define "altinity-clickhouse-operator.operator.tag" -}} +{{- .Values.operator.image.tag | default .Chart.AppVersion -}} +{{- end -}} + +{{/* +Create the tag for the docker image to use +*/}} +{{- define "altinity-clickhouse-operator.metrics.tag" -}} +{{- .Values.metrics.image.tag | default .Chart.AppVersion -}} +{{- end -}} diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml new file mode 100644 index 000000000..3bb1ab7cc --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml @@ -0,0 +1,163 @@ +# Template Parameters: +# +# NAMESPACE=kube-system +# COMMENT=# +# ROLE_KIND=ClusterRole +# ROLE_NAME=clickhouse-operator-kube-system +# ROLE_BINDING_KIND=ClusterRoleBinding +# ROLE_BINDING_NAME=clickhouse-operator-kube-system +# +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + clickhouse.altinity.com/chop: 0.18.1 +rules: + - apiGroups: + - "" + resources: + - configmaps + - services + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - patch + - update + - watch + - delete + - apiGroups: + - "" + resources: + - persistentvolumes + - pods + verbs: + - get + - list + - patch + - update + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - patch + - update + - delete + - apiGroups: + - apps + resourceNames: + - clickhouse-operator + resources: + - deployments + verbs: + - get + - patch + - update + - delete + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations + verbs: + - get + - patch + - update + - delete + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations + - clickhouseinstallationtemplates + - clickhouseoperatorconfigurations + verbs: + - get + - list + - watch + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/finalizers + - clickhouseinstallationtemplates/finalizers + - clickhouseoperatorconfigurations/finalizers + verbs: + - update + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/status + - clickhouseinstallationtemplates/status + - clickhouseoperatorconfigurations/status + verbs: + - get + - update + - patch + - create + - delete + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml new file mode 100644 index 000000000..8c8ab1ff1 --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml @@ -0,0 +1,17 @@ +# Setup ClusterRoleBinding between ClusterRole and ServiceAccount. +# ClusterRoleBinding is namespace-less and must have unique name +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + clickhouse.altinity.com/chop: 0.18.1 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "altinity-clickhouse-operator.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "altinity-clickhouse-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml new file mode 100644 index 000000000..e8a9096f0 --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml @@ -0,0 +1,18 @@ +# Template Parameters: +# +# NAME=etc-clickhouse-operator-confd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-confd-files + namespace: {{ .Release.Namespace }} + labels: + clickhouse.altinity.com/chop: 0.18.1 + {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} +data: +{{- if .Values.configs.confdFiles }} + {{- toYaml .Values.configs.confdFiles | nindent 2 }} +{{ end }} diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml new file mode 100644 index 000000000..983602be4 --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml @@ -0,0 +1,59 @@ +# Template Parameters: +# +# NAME=etc-clickhouse-operator-configd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-configd-files + namespace: {{ .Release.Namespace }} + labels: + clickhouse.altinity.com/chop: 0.18.1 + {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} +data: +{{- if .Values.configs.configdFiles }} + {{- toYaml .Values.configs.configdFiles | nindent 2 }} +{{ else }} + 01-clickhouse-01-listen.xml: | + + + :: + 0.0.0.0 + 1 + + 01-clickhouse-02-logger.xml: | + + + + debug + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + 1 + + + 01-clickhouse-03-query_log.xml: | + + + system + query_log
+ Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day + 7500 + + + + 01-clickhouse-04-part_log.xml: | + + + system + part_log
+ Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day + 7500 + + + +{{ end }} diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml new file mode 100644 index 000000000..5ac2e1bbd --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml @@ -0,0 +1,230 @@ +# Template Parameters: +# +# NAME=etc-clickhouse-operator-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-files + namespace: {{ .Release.Namespace }} + labels: + clickhouse.altinity.com/chop: 0.18.1 + {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} +data: +{{- if .Values.configs.files }} + {{- toYaml .Values.configs.files | nindent 2 }} +{{ else }} + config.yaml: | + # IMPORTANT + # This file is auto-generated from deploy/builder/templates-config. + # It will be overwritten upon next sources build. + # + # Template parameters available: + # watchNamespaces + # chUsername + # chPassword + # password_sha256_hex + + ################################################ + ## + ## Watch Section + ## + ################################################ + watch: + # List of namespaces where clickhouse-operator watches for events. + # Concurrently running operators should watch on different namespaces + #namespaces: ["dev", "test"] + namespaces: [] + + clickhouse: + configuration: + ################################################ + ## + ## Configuration Files Section + ## + ################################################ + file: + path: + # Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. + common: config.d + # Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. + host: conf.d + # Path to the folder where ClickHouse configuration files with users settings are located. + # Files are common for all instances within a CHI. + user: users.d + ################################################ + ## + ## Configuration Users Section + ## + ################################################ + user: + default: + # Default values for ClickHouse user configuration + # 1. user/profile - string + # 2. user/quota - string + # 3. user/networks/ip - multiple strings + # 4. user/password - string + profile: default + quota: default + networksIP: + - "::1" + - "127.0.0.1" + password: "default" + ################################################ + ## + ## Configuration Network Section + ## + ################################################ + network: + # Default host_regexp to limit network connectivity from outside + hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$" + ################################################ + ## + ## Access to ClickHouse instances + ## + ################################################ + access: + # ClickHouse credentials (username, password and port) to be used by operator to connect to ClickHouse instances + # for: + # 1. Metrics requests + # 2. Schema maintenance + # 3. DROP DNS CACHE + # User with such credentials can be specified in additional ClickHouse .xml config files, + # located in `chUsersConfigsPath` folder + username: "clickhouse_operator" + password: "clickhouse_operator_password" + secret: + # Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances + # Can be used instead of explicitly specified username and password + namespace: "" + name: "" + # Port where to connect to ClickHouse instances to + port: 8123 + + ################################################ + ## + ## Templates Section + ## + ################################################ + template: + chi: + # Path to the folder where ClickHouseInstallation .yaml manifests are located. + # Manifests are applied in sorted alpha-numeric order. + path: templates.d + + ################################################ + ## + ## Reconcile Section + ## + ################################################ + reconcile: + runtime: + # Max number of concurrent reconciles in progress + threadsNumber: 10 + + statefulSet: + create: + # What to do in case created StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + # Possible options: + # 1. abort - do nothing, just break the process and wait for admin + # 2. delete - delete newly created problematic StatefulSet + # 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet + onFailure: ignore + + update: + # How many seconds to wait for created/updated StatefulSet to be Ready + timeout: 300 + # How many seconds to wait between checks for created/updated StatefulSet status + pollInterval: 5 + # What to do in case updated StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + # Possible options: + # 1. abort - do nothing, just break the process and wait for admin + # 2. rollback - delete Pod and rollback StatefulSet to previous Generation. + # Pod would be recreated by StatefulSet based on rollback-ed configuration + # 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet + onFailure: rollback + + host: + wait: + exclude: true + include: false + + ################################################ + ## + ## Annotations management + ## + ################################################ + annotation: + # Applied when: + # 1. Propagating annotations from the CHI's `metadata.annotations` to child objects' `metadata.annotations`, + # 2. Propagating annotations from the CHI Template's `metadata.annotations` to CHI's `metadata.annotations`, + # Include annotations from the following list: + # Applied only when not empty. Empty list means "include all, no selection" + include: [] + # Exclude annotations from the following list: + exclude: [] + + ################################################ + ## + ## Labels management + ## + ################################################ + label: + # Applied when: + # 1. Propagating labels from the CHI's `metadata.labels` to child objects' `metadata.labels`, + # 2. Propagating labels from the CHI Template's `metadata.labels` to CHI's `metadata.labels`, + # Include labels from the following list: + # Applied only when not empty. Empty list means "include all, no selection" + include: [] + # Exclude labels from the following list: + exclude: [] + # Whether to append *Scope* labels to StatefulSet and Pod. + # Full list of available *scope* labels check in labeler.go + # LabelShardScopeIndex + # LabelReplicaScopeIndex + # LabelCHIScopeIndex + # LabelCHIScopeCycleSize + # LabelCHIScopeCycleIndex + # LabelCHIScopeCycleOffset + # LabelClusterScopeIndex + # LabelClusterScopeCycleSize + # LabelClusterScopeCycleIndex + # LabelClusterScopeCycleOffset + appendScope: "no" + + ################################################ + ## + ## StatefulSet management + ## + ################################################ + statefulSet: + revisionHistoryLimit: 0 + + ################################################ + ## + ## Pod management + ## + ################################################ + pod: + # Grace period for Pod termination. + # How many seconds to wait between sending + # SIGTERM and SIGKILL during Pod termination process. + # Increase this number is case of slow shutdown. + terminationGracePeriod: 30 + + ################################################ + ## + ## Log parameters + ## + ################################################ + logger: + logtostderr: "true" + alsologtostderr: "false" + v: "1" + stderrthreshold: "" + vmodule: "" + log_backtrace_at: "" + +{{ end }} \ No newline at end of file diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml new file mode 100644 index 000000000..c6131c299 --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml @@ -0,0 +1,102 @@ +# Template Parameters: +# +# NAME=etc-clickhouse-operator-templatesd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-templatesd-files + namespace: {{ .Release.Namespace }} + labels: + clickhouse.altinity.com/chop: 0.18.1 + {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} +data: +{{- if .Values.configs.templatesdFiles }} + {{- toYaml .Values.configs.templatesdFiles | nindent 2 }} +{{ else }} + 001-templates.json.example: | + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallationTemplate", + "metadata": { + "name": "01-default-volumeclaimtemplate" + }, + "spec": { + "templates": { + "volumeClaimTemplates": [ + { + "name": "chi-default-volume-claim-template", + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "2Gi" + } + } + } + } + ], + "podTemplates": [ + { + "name": "chi-default-oneperhost-pod-template", + "distribution": "OnePerHost", + "spec": { + "containers" : [ + { + "name": "clickhouse", + "image": "yandex/clickhouse-server:21.3", + "ports": [ + { + "name": "http", + "containerPort": 8123 + }, + { + "name": "client", + "containerPort": 9000 + }, + { + "name": "interserver", + "containerPort": 9009 + } + ] + } + ] + } + } + ] + } + } + } + default-pod-template.yaml.example: | + apiVersion: "clickhouse.altinity.com/v1" + kind: "ClickHouseInstallationTemplate" + metadata: + name: "default-oneperhost-pod-template" + spec: + templates: + podTemplates: + - name: default-oneperhost-pod-template + distribution: "OnePerHost" + default-storage-template.yaml.example: | + apiVersion: "clickhouse.altinity.com/v1" + kind: "ClickHouseInstallationTemplate" + metadata: + name: "default-storage-template-2Gi" + spec: + templates: + volumeClaimTemplates: + - name: default-storage-template-2Gi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + readme: | + Templates in this folder are packaged with an operator and available via 'useTemplate' + +{{ end }} diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml new file mode 100644 index 000000000..3f6f8f5ce --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml @@ -0,0 +1,62 @@ +# Template Parameters: +# +# NAME=etc-clickhouse-operator-usersd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-usersd-files + namespace: {{ .Release.Namespace }} + labels: + clickhouse.altinity.com/chop: 0.18.1 + {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} +data: +{{- if .Values.configs.usersdFiles }} + {{- toYaml .Values.configs.usersdFiles | nindent 2 }} +{{ else }} + 01-clickhouse-user.xml: | + + + + + 127.0.0.1 + 0.0.0.0/0 + ::/0 + + 716b36073a90c6fe1d445ac1af85f4777c5b7a155cea359961826a030513e448 + clickhouse_operator + default + + + + + 0 + 1 + 10 + + + + 02-clickhouse-default-profile.xml: | + + + + 1 + 1000 + 1 + 1 + + + + 03-database-ordinary.xml: | + + + + + Ordinary + + + + +{{ end }} diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml new file mode 100644 index 000000000..e7109ce8c --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml @@ -0,0 +1,175 @@ +# Template Parameters: +# +# NAMESPACE=kube-system +# COMMENT= +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.18.1 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.18.1 +# +# Setup Deployment for clickhouse-operator +# Deployment would be created in kubectl-specified namespace +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + clickhouse.altinity.com/chop: 0.18.1 + {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "altinity-clickhouse-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "altinity-clickhouse-operator.selectorLabels" . | nindent 8 }} + annotations: + prometheus.io/port: '8888' + prometheus.io/scrape: 'true' + checksum/confd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml") . | sha256sum }} + checksum/configd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml") . | sha256sum }} + checksum/files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-files.yaml") . | sha256sum }} + checksum/templatesd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml") . | sha256sum }} + checksum/usersd-files: {{ include (print $.Template.BasePath "/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml") . | sha256sum }} + {{- if .Values.podAnnotations }} + {{- toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "altinity-clickhouse-operator.serviceAccountName" . }} + volumes: + - name: etc-clickhouse-operator-folder + configMap: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-files + - name: etc-clickhouse-operator-confd-folder + configMap: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-confd-files + - name: etc-clickhouse-operator-configd-folder + configMap: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-configd-files + - name: etc-clickhouse-operator-templatesd-folder + configMap: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-templatesd-files + - name: etc-clickhouse-operator-usersd-folder + configMap: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-usersd-files + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.operator.image.repository }}:{{ include "altinity-clickhouse-operator.operator.tag" . }}" + imagePullPolicy: {{ .Values.operator.image.pullPolicy }} + {{- if .Values.operator.command }} + command: + {{- range $value := .Values.operator.command }} + - {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.operator.args }} + args: + {{- range $value := .Values.operator.args }} + - {{ $value | quote }} + {{- end }} + {{- end }} + volumeMounts: + - name: etc-clickhouse-operator-folder + mountPath: /etc/clickhouse-operator + - name: etc-clickhouse-operator-confd-folder + mountPath: /etc/clickhouse-operator/conf.d + - name: etc-clickhouse-operator-configd-folder + mountPath: /etc/clickhouse-operator/config.d + - name: etc-clickhouse-operator-templatesd-folder + mountPath: /etc/clickhouse-operator/templates.d + - name: etc-clickhouse-operator-usersd-folder + mountPath: /etc/clickhouse-operator/users.d + env: + # Pod-specific + # spec.nodeName: ip-172-20-52-62.ec2.internal + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # metadata.name: clickhouse-operator-6f87589dbb-ftcsf + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # metadata.namespace: kube-system + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # status.podIP: 100.96.3.2 + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # spec.serviceAccount: clickhouse-operator + # spec.serviceAccountName: clickhouse-operator + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + # Container-specific + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: {{ .Chart.Name }} + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: {{ .Chart.Name }} + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: {{ .Chart.Name }} + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: {{ .Chart.Name }} + resource: limits.memory + {{- with .Values.operator.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.operator.resources | nindent 12 }} + - name: metrics-exporter + image: "{{ .Values.metrics.image.repository }}:{{ include "altinity-clickhouse-operator.metrics.tag" . }}" + imagePullPolicy: {{ .Values.metrics.image.pullPolicy }} + {{- if .Values.metrics.command }} + command: + {{- range $value := .Values.metrics.command }} + - {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.metrics.args }} + args: + {{- range $value := .Values.metrics.args }} + - {{ $value | quote }} + {{- end }} + {{- end }} + volumeMounts: + - name: etc-clickhouse-operator-folder + mountPath: /etc/clickhouse-operator + - name: etc-clickhouse-operator-confd-folder + mountPath: /etc/clickhouse-operator/conf.d + - name: etc-clickhouse-operator-configd-folder + mountPath: /etc/clickhouse-operator/config.d + - name: etc-clickhouse-operator-templatesd-folder + mountPath: /etc/clickhouse-operator/templates.d + - name: etc-clickhouse-operator-usersd-folder + mountPath: /etc/clickhouse-operator/users.d + ports: + - containerPort: 8888 + name: metrics + {{- with .Values.metrics.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.metrics.resources | nindent 12 }} diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/Service-clickhouse-operator-metrics.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/Service-clickhouse-operator-metrics.yaml new file mode 100644 index 000000000..3b03a6715 --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/Service-clickhouse-operator-metrics.yaml @@ -0,0 +1,25 @@ +# Template Parameters: +# +# NAMESPACE=kube-system +# COMMENT= +# +# Setup ClusterIP Service to provide monitoring metrics for Prometheus +# Service would be created in kubectl-specified namespace +# In order to get access outside of k8s it should be exposed as: +# kubectl --namespace prometheus port-forward service/prometheus 9090 +# and point browser to localhost:9090 +kind: Service +apiVersion: v1 +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }}-metrics + namespace: {{ .Release.Namespace }} + labels: + clickhouse.altinity.com/chop: 0.18.1 + {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} +spec: + ports: + - port: 8888 + name: clickhouse-operator-metrics + targetPort: metrics + selector: + {{- include "altinity-clickhouse-operator.selectorLabels" . | nindent 4 }} diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml new file mode 100644 index 000000000..d6986674d --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml @@ -0,0 +1,21 @@ +{{- if .Values.serviceAccount.create -}} +# Template Parameters: +# +# COMMENT= +# NAMESPACE=kube-system +# NAME=clickhouse-operator +# +# Setup ServiceAccount +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "altinity-clickhouse-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + clickhouse.altinity.com/chop: 0.18.1 + {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/servicemonitor.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/servicemonitor.yaml new file mode 100644 index 000000000..b96a4fe2f --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/templates/servicemonitor.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "altinity-clickhouse-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "altinity-clickhouse-operator.labels" . | nindent 4 }} + {{- if .Values.serviceMonitor.additionalLabels }} + {{- toYaml .Values.serviceMonitor.additionalLabels | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: clickhouse-operator-metrics + selector: + matchLabels: + {{- include "altinity-clickhouse-operator.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/values.yaml b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/values.yaml new file mode 100644 index 000000000..9ad166824 --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/charts/altinity-clickhouse-operator/values.yaml @@ -0,0 +1,88 @@ +operator: + image: + # operator.image.repository -- image repository + repository: altinity/clickhouse-operator + # operator.image.tag -- image tag (chart's appVersion value will be used if not set) + tag: "" + # operator.image.pullPolicy -- image pull policy + pullPolicy: IfNotPresent + # operator.resources -- custom resource configuration + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + # operator.env -- additional environment variables for the deployment + env: [ ] + # - name: SAMPLE + # value: text +metrics: + image: + # metrics.image.repository -- image repository + repository: altinity/metrics-exporter + # metrics.image.tag -- image tag (chart's appVersion value will be used if not set) + tag: "" + # metrics.image.pullPolicy -- image pull policy + pullPolicy: IfNotPresent + # metrics.resources -- custom resource configuration + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + # metrics.env -- additional environment variables for the deployment + env: [ ] + # - name: SAMPLE + # value: text + +# imagePullSecrets -- image pull secret for private images +imagePullSecrets: [] +# nameOverride -- override name of the chart +nameOverride: "" +# fullnameOverride -- full name of the chart. +fullnameOverride: "" + +serviceAccount: + # serviceAccount.create -- specifies whether a service account should be created + create: true + # serviceAccount.annotations -- annotations to add to the service account + annotations: {} + # serviceAccount.name -- the name of the service account to use; if not set and create is true, a name is generated using the fullname template + name: + +# nodeSelector -- node for scheduler pod assignment +nodeSelector: {} + +# tolerations -- tolerations for scheduler pod assignment +tolerations: [] + +# affinity -- affinity for scheduler pod assignment +affinity: {} + +serviceMonitor: + # serviceMonitor.enabled -- ServiceMonitor CRD is created for a prometheus operator + enabled: false + # serviceMonitor.additionalLabels -- additional labels for service monitor + additionalLabels: {} + +# configs -- overrides operator default configmaps +configs: + confdFiles: null +# some-file: | +# ... + configdFiles: null +# some-file: | +# ... + files: null +# some-file: | +# ... + templatesdFiles: null +# some-file: | +# ... + usersdFiles: null +# some-file: | +# ... diff --git a/scripts/helmcharts/databases/charts/clickhouse/templates/clickhouse-cluster.yaml b/scripts/helmcharts/databases/charts/clickhouse/templates/clickhouse-cluster.yaml new file mode 100644 index 000000000..f89bd5fed --- /dev/null +++ b/scripts/helmcharts/databases/charts/clickhouse/templates/clickhouse-cluster.yaml @@ -0,0 +1,93 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" + +metadata: + name: "openreplay-clickhouse" + +spec: + defaults: + templates: + dataVolumeClaimTemplate: default + podTemplate: clickhouse:19.6 + serviceTemplate: chi-service-template + + configuration: + zookeeper: + nodes: + - host: zookeeper + clusters: + - name: replicated + layout: + shardsCount: 1 + replicasCount: 2 + + templates: + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "{{ .Values.storage.size }}" + podTemplates: + - name: clickhouse:19.6 + spec: + containers: + - name: clickhouse-pod + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + resources: + {{ toYaml .Values.resources | nindent 16 }} + + # Ref: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates + serviceTemplates: + - name: chi-service-template + # generateName understands different sets of macroses, + # depending on the level of the object, for which Service is being created: + # + # For CHI-level Service: + # 1. {chi} - ClickHouseInstallation name + # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature) + # + # For Cluster-level Service: + # 1. {chi} - ClickHouseInstallation name + # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature) + # 3. {cluster} - cluster name + # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature) + # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature) + # + # For Shard-level Service: + # 1. {chi} - ClickHouseInstallation name + # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature) + # 3. {cluster} - cluster name + # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature) + # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature) + # 6. {shard} - shard name + # 7. {shardID} - short hashed shard name (BEWARE, this is an experimental feature) + # 8. {shardIndex} - 0-based index of the shard in the cluster (BEWARE, this is an experimental feature) + # + # For Replica-level Service: + # 1. {chi} - ClickHouseInstallation name + # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature) + # 3. {cluster} - cluster name + # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature) + # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature) + # 6. {shard} - shard name + # 7. {shardID} - short hashed shard name (BEWARE, this is an experimental feature) + # 8. {shardIndex} - 0-based index of the shard in the cluster (BEWARE, this is an experimental feature) + # 9. {replica} - replica name + # 10. {replicaID} - short hashed replica name (BEWARE, this is an experimental feature) + # 11. {replicaIndex} - 0-based index of the replica in the shard (BEWARE, this is an experimental feature) + generateName: "clickhouse-{chi}" + # type ObjectMeta struct from k8s.io/meta/v1 + metadata: + labels: + installation: "openreplay" + spec: + ports: + - name: http + port: 8123 + - name: client + port: 9000 + type: ClusterIP + diff --git a/scripts/helmcharts/databases/charts/clickhouse/templates/service.yaml b/scripts/helmcharts/databases/charts/clickhouse/templates/service.yaml deleted file mode 100644 index 4496f556c..000000000 --- a/scripts/helmcharts/databases/charts/clickhouse/templates/service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: clickhouse - labels: - {{- include "clickhouse.labels" . | nindent 4 }} -spec: - type: ClusterIP - ports: - - port: {{ .Values.service.webPort }} - targetPort: web - protocol: TCP - name: web - - port: {{ .Values.service.dataPort }} - targetPort: data - protocol: TCP - name: data - selector: - {{- include "clickhouse.selectorLabels" . | nindent 4 }} diff --git a/scripts/helmcharts/databases/charts/clickhouse/templates/serviceaccount.yaml b/scripts/helmcharts/databases/charts/clickhouse/templates/serviceaccount.yaml deleted file mode 100644 index 1f1183598..000000000 --- a/scripts/helmcharts/databases/charts/clickhouse/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "clickhouse.serviceAccountName" . }} - labels: - {{- include "clickhouse.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/scripts/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml b/scripts/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml deleted file mode 100644 index 26a11970f..000000000 --- a/scripts/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml +++ /dev/null @@ -1,69 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "clickhouse.fullname" . }} - labels: - {{- include "clickhouse.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - serviceName: {{ include "clickhouse.fullname" . }} - selector: - matchLabels: - {{- include "clickhouse.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "clickhouse.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "clickhouse.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - env: - {{- range $key, $value := .Values.env }} - - name: "{{ $key }}" - value: "{{ $value }}" - {{- end }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - containerPort: 9000 - name: web - - containerPort: 8123 - name: data - volumeMounts: - - name: ch-volume - mountPath: /var/lib/clickhouse - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumeClaimTemplates: - - metadata: - name: ch-volume - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: {{ .Values.storageSize }} diff --git a/scripts/helmcharts/databases/charts/clickhouse/values.yaml b/scripts/helmcharts/databases/charts/clickhouse/values.yaml index fc3d1ca10..00689b44d 100644 --- a/scripts/helmcharts/databases/charts/clickhouse/values.yaml +++ b/scripts/helmcharts/databases/charts/clickhouse/values.yaml @@ -1,58 +1,14 @@ -# Default values for clickhouse. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - image: - repository: yandex/clickhouse-server - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "20.9" + repository: clickhouse/clickhouse-server + tag: "22.3" -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -env: {} - -service: - webPort: 9000 - dataPort: 8123 +storage: + size: 200Gi resources: requests: cpu: 1 - memory: 4Gi + memory: 500Mi limits: - cpu: 2 - memory: 8Gi - -nodeSelector: {} - -tolerations: [] - -affinity: {} -storageSize: 100G + cpu: 1 + memory: 2000Mi From 24f64af95ab46fad270659985de1de92f119d8e4 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 16 May 2022 10:52:32 +0200 Subject: [PATCH 22/25] feat(backend/storage): service refactoring --- backend/build.sh | 2 +- backend/cmd/storage/main.go | 66 ++++++++++++ backend/internal/config/storage/config.go | 32 ++++++ .../{services => internal}/storage/clean.go | 10 +- .../{services => internal}/storage/gzip.go | 4 +- backend/internal/storage/storage.go | 58 ++++++++++ backend/services/storage/build_hack | 0 backend/services/storage/main.go | 100 ------------------ 8 files changed, 163 insertions(+), 109 deletions(-) create mode 100644 backend/cmd/storage/main.go create mode 100644 backend/internal/config/storage/config.go rename backend/{services => internal}/storage/clean.go (66%) rename backend/{services => internal}/storage/gzip.go (75%) create mode 100644 backend/internal/storage/storage.go create mode 100644 backend/services/storage/build_hack delete mode 100644 backend/services/storage/main.go diff --git a/backend/build.sh b/backend/build.sh index e67178c8c..97604dd0c 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -23,7 +23,7 @@ function build_service() { image="$1" echo "BUILDING $image" case "$image" in - http | db | sink | ender | heuristics) + http | db | sink | ender | heuristics | storage) echo build http docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --platform linux/amd64 --build-arg SERVICE_NAME=$image -f ./cmd/Dockerfile . [[ $PUSH_IMAGE -eq 1 ]] && { diff --git a/backend/cmd/storage/main.go b/backend/cmd/storage/main.go new file mode 100644 index 000000000..57fbd1776 --- /dev/null +++ b/backend/cmd/storage/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "log" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + config "openreplay/backend/internal/config/storage" + "openreplay/backend/internal/storage" + "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/queue" + "openreplay/backend/pkg/queue/types" + s3storage "openreplay/backend/pkg/storage" +) + +func main() { + log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) + + cfg := config.New() + + s3 := s3storage.NewS3(cfg.S3Region, cfg.S3Bucket) + srv, err := storage.New(cfg, s3) + if err != nil { + log.Printf("can't init storage service: %s", err) + return + } + + consumer := queue.NewMessageConsumer( + cfg.GroupStorage, + []string{ + cfg.TopicTrigger, + }, + func(sessionID uint64, msg messages.Message, meta *types.Meta) { + switch msg.(type) { + case *messages.SessionEnd: + srv.UploadKey(strconv.FormatUint(sessionID, 10), 5) + } + }, + true, + ) + + log.Printf("Storage service started\n") + + sigchan := make(chan os.Signal, 1) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + + cleanTick := time.Tick(time.Duration(cfg.FSCleanHRS) * time.Hour) + for { + select { + case sig := <-sigchan: + log.Printf("Caught signal %v: terminating\n", sig) + consumer.Close() + os.Exit(0) + case <-cleanTick: + go srv.CleanDir(cfg.FSDir) + default: + err := consumer.ConsumeNext() + if err != nil { + log.Fatalf("Error on consumption: %v", err) + } + } + } +} diff --git a/backend/internal/config/storage/config.go b/backend/internal/config/storage/config.go new file mode 100644 index 000000000..305c293c9 --- /dev/null +++ b/backend/internal/config/storage/config.go @@ -0,0 +1,32 @@ +package storage + +import ( + "openreplay/backend/pkg/env" + "time" +) + +type Config struct { + S3Region string + S3Bucket string + FSDir string + FSCleanHRS int + SessionFileSplitSize int + RetryTimeout time.Duration + GroupStorage string + TopicTrigger string + DeleteTimeout time.Duration +} + +func New() *Config { + return &Config{ + S3Region: env.String("AWS_REGION_WEB"), + S3Bucket: env.String("S3_BUCKET_WEB"), + FSDir: env.String("FS_DIR"), + FSCleanHRS: env.Int("FS_CLEAN_HRS"), + SessionFileSplitSize: 200000, // ~200 kB + RetryTimeout: 2 * time.Minute, + GroupStorage: env.String("GROUP_STORAGE"), + TopicTrigger: env.String("TOPIC_TRIGGER"), + DeleteTimeout: 48 * time.Hour, + } +} diff --git a/backend/services/storage/clean.go b/backend/internal/storage/clean.go similarity index 66% rename from backend/services/storage/clean.go rename to backend/internal/storage/clean.go index 72f5f359c..3b6e4d6eb 100644 --- a/backend/services/storage/clean.go +++ b/backend/internal/storage/clean.go @@ -1,4 +1,4 @@ -package main +package storage import ( "io/ioutil" @@ -10,9 +10,7 @@ import ( "openreplay/backend/pkg/flakeid" ) -const DELETE_TIMEOUT = 48 * time.Hour - -func cleanDir(dirname string) { +func (s *Storage) CleanDir(dirname string) { files, err := ioutil.ReadDir(dirname) if err != nil { log.Printf("Cannot read file directory. %v", err) @@ -27,8 +25,8 @@ func cleanDir(dirname string) { continue } ts := int64(flakeid.ExtractTimestamp(id)) - if time.UnixMilli(ts).Add(DELETE_TIMEOUT).Before(time.Now()) { - // returns a error. Don't log it sinse it can be race condition between worker instances + if time.UnixMilli(ts).Add(s.cfg.DeleteTimeout).Before(time.Now()) { + // returns an error. Don't log it since it can be race condition between worker instances os.Remove(dirname + "/" + name) } } diff --git a/backend/services/storage/gzip.go b/backend/internal/storage/gzip.go similarity index 75% rename from backend/services/storage/gzip.go rename to backend/internal/storage/gzip.go index 0e662efaa..ee47e5079 100644 --- a/backend/services/storage/gzip.go +++ b/backend/internal/storage/gzip.go @@ -1,11 +1,11 @@ -package main +package storage import ( gzip "github.com/klauspost/pgzip" "io" ) -func gzipFile(file io.Reader) io.Reader { +func (s *Storage) gzipFile(file io.Reader) io.Reader { reader, writer := io.Pipe() go func() { gw, _ := gzip.NewWriterLevel(writer, gzip.BestSpeed) diff --git a/backend/internal/storage/storage.go b/backend/internal/storage/storage.go new file mode 100644 index 000000000..0051fd5ea --- /dev/null +++ b/backend/internal/storage/storage.go @@ -0,0 +1,58 @@ +package storage + +import ( + "bytes" + "fmt" + "log" + config "openreplay/backend/internal/config/storage" + "openreplay/backend/pkg/storage" + "os" + "time" +) + +type Storage struct { + cfg *config.Config + s3 *storage.S3 +} + +func New(cfg *config.Config, s3 *storage.S3) (*Storage, error) { + switch { + case cfg == nil: + return nil, fmt.Errorf("config is empty") + case s3 == nil: + return nil, fmt.Errorf("s3 storage is empty") + } + return &Storage{s3: s3}, nil +} + +func (s *Storage) UploadKey(key string, retryCount int) { + if retryCount <= 0 { + return + } + + file, err := os.Open(s.cfg.FSDir + "/" + key) + if err != nil { + log.Printf("File error: %v; Will retry %v more time(s)\n", err, retryCount) + time.AfterFunc(s.cfg.RetryTimeout, func() { + s.UploadKey(key, retryCount-1) + }) + return + } + defer file.Close() + + startBytes := make([]byte, s.cfg.SessionFileSplitSize) + nRead, err := file.Read(startBytes) + if err != nil { + log.Printf("File read error: %f", err) + return + } + startReader := bytes.NewBuffer(startBytes) + if err := s.s3.Upload(s.gzipFile(startReader), key, "application/octet-stream", true); err != nil { + log.Fatalf("Storage: start upload failed. %v\n", err) + } + if nRead == s.cfg.SessionFileSplitSize { + if err := s.s3.Upload(s.gzipFile(file), key+"e", "application/octet-stream", true); err != nil { + log.Fatalf("Storage: end upload failed. %v\n", err) + } + } +} diff --git a/backend/services/storage/build_hack b/backend/services/storage/build_hack new file mode 100644 index 000000000..e69de29bb diff --git a/backend/services/storage/main.go b/backend/services/storage/main.go deleted file mode 100644 index 95c5c6d17..000000000 --- a/backend/services/storage/main.go +++ /dev/null @@ -1,100 +0,0 @@ -package main - -import ( - "log" - "os" - "strconv" - "time" - - "bytes" - - "os/signal" - "syscall" - - "openreplay/backend/pkg/env" - "openreplay/backend/pkg/messages" - "openreplay/backend/pkg/queue" - "openreplay/backend/pkg/queue/types" - "openreplay/backend/pkg/storage" -) - -const RetryTimeout = 2 * time.Minute - -const SESSION_FILE_SPLIT_SIZE = 200000 // ~200 kB - -func main() { - log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - - storage := storage.NewS3(env.String("AWS_REGION_WEB"), env.String("S3_BUCKET_WEB")) - FS_DIR := env.String("FS_DIR") - FS_CLEAN_HRS := env.Int("FS_CLEAN_HRS") - - var uploadKey func(string, int) - uploadKey = func(key string, retryCount int) { - if retryCount <= 0 { - return - } - - file, err := os.Open(FS_DIR + "/" + key) - if err != nil { - log.Printf("File error: %v; Will retry %v more time(s)\n", err, retryCount) - time.AfterFunc(RetryTimeout, func() { - uploadKey(key, retryCount-1) - }) - return - } - defer file.Close() - - startBytes := make([]byte, SESSION_FILE_SPLIT_SIZE) - nRead, err := file.Read(startBytes) - if err != nil { - log.Printf("File read error: %f", err) - return - } - startReader := bytes.NewBuffer(startBytes) - if err := storage.Upload(gzipFile(startReader), key, "application/octet-stream", true); err != nil { - log.Fatalf("Storage: start upload failed. %v\n", err) - } - if nRead == SESSION_FILE_SPLIT_SIZE { - if err := storage.Upload(gzipFile(file), key+"e", "application/octet-stream", true); err != nil { - log.Fatalf("Storage: end upload failed. %v\n", err) - } - } - } - - consumer := queue.NewMessageConsumer( - env.String("GROUP_STORAGE"), - []string{ - env.String("TOPIC_TRIGGER"), - }, - func(sessionID uint64, msg messages.Message, meta *types.Meta) { - switch msg.(type) { - case *messages.SessionEnd: - uploadKey(strconv.FormatUint(sessionID, 10), 5) - } - }, - true, - ) - - sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - - cleanTick := time.Tick(time.Duration(FS_CLEAN_HRS) * time.Hour) - - log.Printf("Storage service started\n") - for { - select { - case sig := <-sigchan: - log.Printf("Caught signal %v: terminating\n", sig) - consumer.Close() - os.Exit(0) - case <-cleanTick: - go cleanDir(FS_DIR) - default: - err := consumer.ConsumeNext() - if err != nil { - log.Fatalf("Error on consumption: %v", err) - } - } - } -} From d36d4862cf2d0e2a9f6c8b9568916b6bd02f5702 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 16 May 2022 14:12:16 +0200 Subject: [PATCH 23/25] fix(ui) - chart y axis numbers --- .../CustomMetriLineChart/CustomMetriLineChart.tsx | 5 +++-- .../BreakdownOfLoadedResources.tsx | 1 + .../PredefinedWidgets/ErrorsByOrigin/ErrorsByOrigin.tsx | 1 + .../Widgets/PredefinedWidgets/ErrorsByType/ErrorsByType.tsx | 1 + frontend/app/components/ui/Pagination/Pagination.tsx | 3 ++- 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/Dashboard/Widgets/CustomMetricsWidgets/CustomMetriLineChart/CustomMetriLineChart.tsx b/frontend/app/components/Dashboard/Widgets/CustomMetricsWidgets/CustomMetriLineChart/CustomMetriLineChart.tsx index 198afb088..4da7631fa 100644 --- a/frontend/app/components/Dashboard/Widgets/CustomMetricsWidgets/CustomMetriLineChart/CustomMetriLineChart.tsx +++ b/frontend/app/components/Dashboard/Widgets/CustomMetricsWidgets/CustomMetriLineChart/CustomMetriLineChart.tsx @@ -31,9 +31,10 @@ function CustomMetriLineChart(props: Props) { Styles.tickFormatter(val)} label={{ - ...Styles.axisLabelLeft, - value: "Number of Sessions" + ...Styles.axisLabelLeft, + value: "Number of Sessions" }} /> diff --git a/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/BreakdownOfLoadedResources/BreakdownOfLoadedResources.tsx b/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/BreakdownOfLoadedResources/BreakdownOfLoadedResources.tsx index fd38e2a55..73a8fd46a 100644 --- a/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/BreakdownOfLoadedResources/BreakdownOfLoadedResources.tsx +++ b/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/BreakdownOfLoadedResources/BreakdownOfLoadedResources.tsx @@ -33,6 +33,7 @@ function BreakdownOfLoadedResources(props: Props) { {...Styles.yaxis} allowDecimals={false} label={{ ...Styles.axisLabelLeft, value: "Number of Resources" }} + tickFormatter={val => Styles.tickFormatter(val)} /> diff --git a/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/ErrorsByOrigin/ErrorsByOrigin.tsx b/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/ErrorsByOrigin/ErrorsByOrigin.tsx index d7aefebd0..f50859051 100644 --- a/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/ErrorsByOrigin/ErrorsByOrigin.tsx +++ b/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/ErrorsByOrigin/ErrorsByOrigin.tsx @@ -33,6 +33,7 @@ function ErrorsByOrigin(props: Props) { /> Styles.tickFormatter(val)} label={{ ...Styles.axisLabelLeft, value: "Number of Errors" }} allowDecimals={false} /> diff --git a/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/ErrorsByType/ErrorsByType.tsx b/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/ErrorsByType/ErrorsByType.tsx index 23a6fda45..8d01941c8 100644 --- a/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/ErrorsByType/ErrorsByType.tsx +++ b/frontend/app/components/Dashboard/Widgets/PredefinedWidgets/ErrorsByType/ErrorsByType.tsx @@ -32,6 +32,7 @@ function ErrorsByType(props: Props) { /> Styles.tickFormatter(val)} label={{ ...Styles.axisLabelLeft, value: "Number of Errors" }} allowDecimals={false} /> diff --git a/frontend/app/components/ui/Pagination/Pagination.tsx b/frontend/app/components/ui/Pagination/Pagination.tsx index a915feee0..203213c90 100644 --- a/frontend/app/components/ui/Pagination/Pagination.tsx +++ b/frontend/app/components/ui/Pagination/Pagination.tsx @@ -4,6 +4,7 @@ import { Icon } from 'UI' import cn from 'classnames' import { debounce } from 'App/utils'; import { Tooltip } from 'react-tippy'; +import { numberWithCommas } from 'App/utils'; interface Props { page: number totalPages: number @@ -57,7 +58,7 @@ export default function Pagination(props: Props) { onChange={(e) => changePage(parseInt(e.target.value))} /> of - {totalPages} + {numberWithCommas(totalPages)} Date: Mon, 16 May 2022 14:12:37 +0200 Subject: [PATCH 24/25] feat(backend/assets): service refactoring --- backend/build.sh | 2 +- backend/{services => cmd}/assets/main.go | 29 +++++++++---------- .../assets/cacher/cacher.go | 0 .../assets/cacher/timeoutMap.go | 0 .../assets/jsexception.go | 4 +-- backend/internal/config/assets/config.go | 23 +++++++++++++++ backend/services/assets/build_hack | 0 7 files changed, 40 insertions(+), 18 deletions(-) rename backend/{services => cmd}/assets/main.go (77%) rename backend/{services => internal}/assets/cacher/cacher.go (100%) rename backend/{services => internal}/assets/cacher/timeoutMap.go (100%) rename backend/{services => internal}/assets/jsexception.go (87%) create mode 100644 backend/internal/config/assets/config.go create mode 100644 backend/services/assets/build_hack diff --git a/backend/build.sh b/backend/build.sh index 97604dd0c..58776c93b 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -23,7 +23,7 @@ function build_service() { image="$1" echo "BUILDING $image" case "$image" in - http | db | sink | ender | heuristics | storage) + http | db | sink | ender | heuristics | storage | assets) echo build http docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --platform linux/amd64 --build-arg SERVICE_NAME=$image -f ./cmd/Dockerfile . [[ $PUSH_IMAGE -eq 1 ]] && { diff --git a/backend/services/assets/main.go b/backend/cmd/assets/main.go similarity index 77% rename from backend/services/assets/main.go rename to backend/cmd/assets/main.go index 259918395..3d417d916 100644 --- a/backend/services/assets/main.go +++ b/backend/cmd/assets/main.go @@ -2,35 +2,34 @@ package main import ( "log" - "time" - "os" "os/signal" "syscall" + "time" - "openreplay/backend/pkg/env" + "openreplay/backend/internal/assets" + "openreplay/backend/internal/assets/cacher" + config "openreplay/backend/internal/config/assets" "openreplay/backend/pkg/messages" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/queue/types" - "openreplay/backend/services/assets/cacher" ) func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - GROUP_CACHE := env.String("GROUP_CACHE") - TOPIC_CACHE := env.String("TOPIC_CACHE") + cfg := config.New() cacher := cacher.NewCacher( - env.String("AWS_REGION"), - env.String("S3_BUCKET_ASSETS"), - env.String("ASSETS_ORIGIN"), - env.Int("ASSETS_SIZE_LIMIT"), + cfg.AWSRegion, + cfg.S3BucketAssets, + cfg.AssetsOrigin, + cfg.AssetsSizeLimit, ) consumer := queue.NewMessageConsumer( - GROUP_CACHE, - []string{TOPIC_CACHE}, + cfg.GroupCache, + []string{cfg.TopicCache}, func(sessionID uint64, message messages.Message, e *types.Meta) { switch msg := message.(type) { case *messages.AssetCache: @@ -39,7 +38,7 @@ func main() { if msg.Source != "js_exception" { return } - sourceList, err := extractJSExceptionSources(&msg.Payload) + sourceList, err := assets.ExtractJSExceptionSources(&msg.Payload) if err != nil { log.Printf("Error on source extraction: %v", err) return @@ -52,12 +51,12 @@ func main() { true, ) - tick := time.Tick(20 * time.Minute) + log.Printf("Cacher service started\n") sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - log.Printf("Cacher service started\n") + tick := time.Tick(20 * time.Minute) for { select { case sig := <-sigchan: diff --git a/backend/services/assets/cacher/cacher.go b/backend/internal/assets/cacher/cacher.go similarity index 100% rename from backend/services/assets/cacher/cacher.go rename to backend/internal/assets/cacher/cacher.go diff --git a/backend/services/assets/cacher/timeoutMap.go b/backend/internal/assets/cacher/timeoutMap.go similarity index 100% rename from backend/services/assets/cacher/timeoutMap.go rename to backend/internal/assets/cacher/timeoutMap.go diff --git a/backend/services/assets/jsexception.go b/backend/internal/assets/jsexception.go similarity index 87% rename from backend/services/assets/jsexception.go rename to backend/internal/assets/jsexception.go index c0b26e0db..180f24df1 100644 --- a/backend/services/assets/jsexception.go +++ b/backend/internal/assets/jsexception.go @@ -1,4 +1,4 @@ -package main +package assets import ( "encoding/json" @@ -9,7 +9,7 @@ type frame struct { FileName string `json:"fileName"` } -func extractJSExceptionSources(payload *string) ([]string, error) { +func ExtractJSExceptionSources(payload *string) ([]string, error) { var frameList []frame err := json.Unmarshal([]byte(*payload), &frameList) if err != nil { diff --git a/backend/internal/config/assets/config.go b/backend/internal/config/assets/config.go new file mode 100644 index 000000000..279a8283c --- /dev/null +++ b/backend/internal/config/assets/config.go @@ -0,0 +1,23 @@ +package assets + +import "openreplay/backend/pkg/env" + +type Config struct { + GroupCache string + TopicCache string + AWSRegion string + S3BucketAssets string + AssetsOrigin string + AssetsSizeLimit int +} + +func New() *Config { + return &Config{ + GroupCache: env.String("GROUP_CACHE"), + TopicCache: env.String("TOPIC_CACHE"), + AWSRegion: env.String("AWS_REGION"), + S3BucketAssets: env.String("S3_BUCKET_ASSETS"), + AssetsOrigin: env.String("ASSETS_ORIGIN"), + AssetsSizeLimit: env.Int("ASSETS_SIZE_LIMIT"), + } +} diff --git a/backend/services/assets/build_hack b/backend/services/assets/build_hack new file mode 100644 index 000000000..e69de29bb From ebbc9cc984e8816f6befa37e561136bf410f097f Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 16 May 2022 14:18:42 +0200 Subject: [PATCH 25/25] fix(ui) - alert form footer bg --- frontend/app/components/Alerts/AlertForm.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/Alerts/AlertForm.js b/frontend/app/components/Alerts/AlertForm.js index f4b9362c7..d3258d578 100644 --- a/frontend/app/components/Alerts/AlertForm.js +++ b/frontend/app/components/Alerts/AlertForm.js @@ -71,7 +71,7 @@ const AlertForm = props => { const isThreshold = instance.detectionMethod === 'threshold'; return ( -
props.onSubmit(instance)} id="alert-form"> + props.onSubmit(instance)} id="alert-form">
{
-
+