From 0cea30a501507ae8edd0054bc5a4aaa16aa14a8f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 25 Nov 2022 17:02:02 +0100 Subject: [PATCH 01/22] feat(DB): changed requests index --- ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql | 4 +++- ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql | 1 + scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql | 4 +++- scripts/schema/db/init_dbs/postgresql/init_schema.sql | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql index 6da0eebed..05878ddba 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql @@ -77,4 +77,6 @@ DROP INDEX IF EXISTS events_common.requests_url_gin_idx2; DROP INDEX IF EXISTS events.resources_url_gin_idx; DROP INDEX IF EXISTS events.resources_url_idx; -COMMIT; \ No newline at end of file +COMMIT; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_session_id_status_code_nn_idx ON events_common.requests (session_id, status_code) WHERE status_code IS NOT NULL; \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index 78026e245..e864f3664 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -1228,6 +1228,7 @@ $$ CREATE INDEX IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; CREATE INDEX IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; CREATE INDEX IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + CREATE INDEX IF NOT EXISTS requests_session_id_status_code_nn_idx ON events_common.requests (session_id, status_code) WHERE status_code IS NOT NULL; CREATE INDEX IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; CREATE INDEX IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; CREATE INDEX IF NOT EXISTS requests_path_nn_idx ON events_common.requests (path) WHERE path IS NOT NULL; diff --git a/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql b/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql index c4c146d9b..5850be8ba 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql @@ -67,4 +67,6 @@ DROP INDEX IF EXISTS events_common.requests_url_gin_idx2; DROP INDEX IF EXISTS events.resources_url_gin_idx; DROP INDEX IF EXISTS events.resources_url_idx; -COMMIT; \ No newline at end of file +COMMIT; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_session_id_status_code_nn_idx ON events_common.requests (session_id, status_code) WHERE status_code IS NOT NULL; \ No newline at end of file diff --git a/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/scripts/schema/db/init_dbs/postgresql/init_schema.sql index a57978965..79b0b649c 100644 --- a/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -603,6 +603,7 @@ $$ CREATE INDEX requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; CREATE INDEX requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; CREATE INDEX requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL; + CREATE INDEX requests_session_id_status_code_nn_idx ON events_common.requests (session_id, status_code) WHERE status_code IS NOT NULL; CREATE INDEX requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL; CREATE INDEX requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL; CREATE INDEX requests_path_nn_idx ON events_common.requests (path) WHERE path IS NOT NULL; From 3f423551e89d86906629553c4bae99e9fdfa5edf Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 25 Nov 2022 17:16:09 +0100 Subject: [PATCH 02/22] feat(chalice): optimized get projects list --- api/chalicelib/core/projects.py | 22 ++++++++++++---------- ee/api/chalicelib/core/projects.py | 24 +++++++++++++----------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/api/chalicelib/core/projects.py b/api/chalicelib/core/projects.py index 18e80944e..10d1e7aee 100644 --- a/api/chalicelib/core/projects.py +++ b/api/chalicelib/core/projects.py @@ -76,19 +76,21 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st rows = cur.fetchall() # if recorded is requested, check if it was saved or computed if recorded: - for r in rows: + u_values = [] + params = {} + for i, r in enumerate(rows): if r["first_recorded_session_at"] is None: - extra_update = "" - if r["recorded"]: - extra_update = ", first_recorded_session_at=to_timestamp(%(first_recorded)s/1000)" - query = cur.mogrify(f"""UPDATE public.projects - SET sessions_last_check_at=(now() at time zone 'utc') - {extra_update} - WHERE project_id=%(project_id)s""", - {"project_id": r["project_id"], "first_recorded": r["first_recorded"]}) - cur.execute(query) + u_values.append(f"(%(project_id_{i})s,to_timestamp(%(first_recorded_{i})s/1000))") + params[f"project_id_{i}"] = r["project_id"] + params[f"first_recorded_{i}"] = r["first_recorded"] if r["recorded"] else None r.pop("first_recorded_session_at") r.pop("first_recorded") + if len(u_values) > 0: + query = cur.mogrify(f"""UPDATE public.projects + SET sessions_last_check_at=(now() at time zone 'utc'), first_recorded_session_at=u.first_recorded + FROM (VALUES {",".join(u_values)}) AS u(project_id,first_recorded) + WHERE projects.project_id=u.project_id;""", params) + cur.execute(query) if recording_state and len(rows) > 0: project_ids = [f'({r["project_id"]})' for r in rows] diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index f2d7ebfdb..18d71914b 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -87,22 +87,23 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st {"tenant_id": tenant_id, "user_id": user_id, "now": TimeUTC.now()}) cur.execute(query) rows = cur.fetchall() - # if recorded is requested, check if it was saved or computed if recorded: - for r in rows: + u_values = [] + params = {} + for i, r in enumerate(rows): if r["first_recorded_session_at"] is None: - extra_update = "" - if r["recorded"]: - extra_update = ", first_recorded_session_at=to_timestamp(%(first_recorded)s/1000)" - query = cur.mogrify(f"""UPDATE public.projects - SET sessions_last_check_at=(now() at time zone 'utc') - {extra_update} - WHERE project_id=%(project_id)s""", - {"project_id": r["project_id"], "first_recorded": r["first_recorded"]}) - cur.execute(query) + u_values.append(f"(%(project_id_{i})s,to_timestamp(%(first_recorded_{i})s/1000))") + params[f"project_id_{i}"] = r["project_id"] + params[f"first_recorded_{i}"] = r["first_recorded"] if r["recorded"] else None r.pop("first_recorded_session_at") r.pop("first_recorded") + if len(u_values) > 0: + query = cur.mogrify(f"""UPDATE public.projects + SET sessions_last_check_at=(now() at time zone 'utc'), first_recorded_session_at=u.first_recorded + FROM (VALUES {",".join(u_values)}) AS u(project_id,first_recorded) + WHERE projects.project_id=u.project_id;""", params) + cur.execute(query) if recording_state and len(rows) > 0: project_ids = [f'({r["project_id"]})' for r in rows] @@ -112,6 +113,7 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st WHERE sessions.start_ts >= %(startDate)s AND sessions.start_ts <= %(endDate)s GROUP BY project_id;""", {"startDate": TimeUTC.now(delta_days=-3), "endDate": TimeUTC.now(delta_days=1)}) + cur.execute(query=query) status = cur.fetchall() for r in rows: From 9e319ed27c70f62d3ca907bab2a5d66e734d2922 Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 25 Nov 2022 17:25:55 +0100 Subject: [PATCH 03/22] [Sink] Improved files sync algo (#831) * feat(backend): use channel of changed sessions instead of sync.Map * feat(backend): avoid memory alloc for message body in message iterator * feat(backend): removed unnecessary locks in file syncer * feat(backend): sync.Map with prev updates * feat(backend): improved write algorith (added bufio.Writer) * feat(backend): session writer refactoring * feat(backend): removed unnecessary type definition * feat(backend): added write retrier to avoid data losing * feat(backend): refactoring * feat(backend): added session file implementation --- backend/cmd/sink/main.go | 26 +-- backend/internal/config/sink/config.go | 3 +- backend/internal/sink/sessionwriter/file.go | 57 +++++ backend/internal/sink/sessionwriter/meta.go | 56 +++++ .../internal/sink/sessionwriter/session.go | 101 +++++---- backend/internal/sink/sessionwriter/types.go | 8 - backend/internal/sink/sessionwriter/writer.go | 195 +++++++----------- backend/pkg/messages/iterator.go | 1 + backend/pkg/messages/primitives.go | 15 +- backend/pkg/messages/raw.go | 25 ++- 10 files changed, 281 insertions(+), 206 deletions(-) create mode 100644 backend/internal/sink/sessionwriter/file.go create mode 100644 backend/internal/sink/sessionwriter/meta.go delete mode 100644 backend/internal/sink/sessionwriter/types.go diff --git a/backend/cmd/sink/main.go b/backend/cmd/sink/main.go index 84520dd33..a7e2804c4 100644 --- a/backend/cmd/sink/main.go +++ b/backend/cmd/sink/main.go @@ -32,7 +32,7 @@ func main() { log.Fatalf("%v doesn't exist. %v", cfg.FsDir, err) } - writer := sessionwriter.NewWriter(cfg.FsUlimit, cfg.FsDir, cfg.DeadSessionTimeout) + writer := sessionwriter.NewWriter(cfg.FsUlimit, cfg.FsDir, cfg.FileBuffer, cfg.SyncTimeout) producer := queue.NewProducer(cfg.MessageSizeLimit, true) defer producer.Close(cfg.ProducerCloseTimeout) @@ -95,26 +95,20 @@ func main() { counter.Update(msg.SessionID(), time.UnixMilli(ts)) } - // Write encoded message with index to session file - data := msg.EncodeWithIndex() + // Try to encode message to avoid null data inserts + data := msg.Encode() if data == nil { return } // Write message to file - if messages.IsDOMType(msg.TypeID()) { - if err := writer.WriteDOM(msg.SessionID(), data); err != nil { - log.Printf("Writer error: %v\n", err) - } - } - if !messages.IsDOMType(msg.TypeID()) || msg.TypeID() == messages.MsgTimestamp { - if err := writer.WriteDEV(msg.SessionID(), data); err != nil { - log.Printf("Writer error: %v\n", err) - } + if err := writer.Write(msg); err != nil { + log.Printf("writer error: %s", err) + return } // [METRICS] Increase the number of written to the files messages and the message size - messageSize.Record(context.Background(), float64(len(data))) + messageSize.Record(context.Background(), float64(len(msg.Encode()))) savedMessages.Add(context.Background(), 1) } @@ -132,7 +126,8 @@ func main() { sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - tick := time.Tick(30 * time.Second) + tick := time.Tick(10 * time.Second) + tickInfo := time.Tick(30 * time.Second) for { select { case sig := <-sigchan: @@ -146,10 +141,11 @@ func main() { consumer.Close() os.Exit(0) case <-tick: - counter.Print() if err := consumer.Commit(); err != nil { log.Printf("can't commit messages: %s", err) } + case <-tickInfo: + counter.Print() log.Printf("writer: %s", writer.Info()) default: err := consumer.ConsumeNext() diff --git a/backend/internal/config/sink/config.go b/backend/internal/config/sink/config.go index a8703a596..1a2df142e 100644 --- a/backend/internal/config/sink/config.go +++ b/backend/internal/config/sink/config.go @@ -9,7 +9,8 @@ type Config struct { common.Config FsDir string `env:"FS_DIR,required"` FsUlimit uint16 `env:"FS_ULIMIT,required"` - DeadSessionTimeout int64 `env:"DEAD_SESSION_TIMEOUT,default=600"` + FileBuffer int `env:"FILE_BUFFER,default=32768"` + SyncTimeout int `env:"SYNC_TIMEOUT,default=5"` GroupSink string `env:"GROUP_SINK,required"` TopicRawWeb string `env:"TOPIC_RAW_WEB,required"` TopicRawIOS string `env:"TOPIC_RAW_IOS,required"` diff --git a/backend/internal/sink/sessionwriter/file.go b/backend/internal/sink/sessionwriter/file.go new file mode 100644 index 000000000..1ad076d72 --- /dev/null +++ b/backend/internal/sink/sessionwriter/file.go @@ -0,0 +1,57 @@ +package sessionwriter + +import ( + "bufio" + "os" +) + +type File struct { + file *os.File + buffer *bufio.Writer + updated bool +} + +func NewFile(path string, bufSize int) (*File, error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + return nil, err + } + return &File{ + file: file, + buffer: bufio.NewWriterSize(file, bufSize), + updated: false, + }, nil +} + +func (f *File) Write(data []byte) error { + leftToWrite := len(data) + for leftToWrite > 0 { + writtenDown, err := f.buffer.Write(data) + if err != nil { + return err + } + leftToWrite -= writtenDown + } + f.updated = true + return nil +} + +func (f *File) Sync() error { + if !f.updated { + return nil + } + if err := f.buffer.Flush(); err != nil { + return err + } + if err := f.file.Sync(); err != nil { + return err + } + f.updated = false + return nil +} + +func (f *File) Close() error { + _ = f.buffer.Flush() + _ = f.file.Sync() + return f.file.Close() +} diff --git a/backend/internal/sink/sessionwriter/meta.go b/backend/internal/sink/sessionwriter/meta.go new file mode 100644 index 000000000..4fac56e50 --- /dev/null +++ b/backend/internal/sink/sessionwriter/meta.go @@ -0,0 +1,56 @@ +package sessionwriter + +import ( + "math" + "sync" + "time" +) + +type Meta struct { + limit int + lock *sync.Mutex + meta map[uint64]int64 +} + +func NewMeta(limit int) *Meta { + return &Meta{ + limit: limit, + lock: &sync.Mutex{}, + meta: make(map[uint64]int64, limit), + } +} + +func (m *Meta) Add(sid uint64) { + m.lock.Lock() + m.meta[sid] = time.Now().Unix() + m.lock.Unlock() +} + +func (m *Meta) Count() int { + m.lock.Lock() + defer m.lock.Unlock() + return len(m.meta) +} + +func (m *Meta) Delete(sid uint64) { + m.lock.Lock() + delete(m.meta, sid) + m.lock.Unlock() +} + +func (m *Meta) GetExtra() uint64 { + m.lock.Lock() + defer m.lock.Unlock() + if len(m.meta) >= m.limit { + var extraSessID uint64 + var minTimestamp int64 = math.MaxInt64 + for sessID, timestamp := range m.meta { + if timestamp < minTimestamp { + extraSessID = sessID + minTimestamp = timestamp + } + } + return extraSessID + } + return 0 +} diff --git a/backend/internal/sink/sessionwriter/session.go b/backend/internal/sink/sessionwriter/session.go index f107c387b..8cf8881de 100644 --- a/backend/internal/sink/sessionwriter/session.go +++ b/backend/internal/sink/sessionwriter/session.go @@ -1,81 +1,96 @@ package sessionwriter import ( + "encoding/binary" "fmt" - "os" "strconv" "sync" - "time" + + "openreplay/backend/pkg/messages" ) type Session struct { - lock *sync.Mutex - dom *os.File - dev *os.File - lastUpdate time.Time + lock *sync.Mutex + dom *File + dev *File + index []byte + updated bool } -func NewSession(dir string, id uint64) (*Session, error) { - if id == 0 { +func NewSession(sessID uint64, workDir string, bufSize int) (*Session, error) { + if sessID == 0 { return nil, fmt.Errorf("wrong session id") } + filePath := workDir + strconv.FormatUint(sessID, 10) - filePath := dir + strconv.FormatUint(id, 10) - domFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + dom, err := NewFile(filePath, bufSize) if err != nil { return nil, err } - filePath += "devtools" - devFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + dev, err := NewFile(filePath+"devtools", bufSize) if err != nil { - domFile.Close() // should close first file descriptor + dom.Close() return nil, err } return &Session{ - lock: &sync.Mutex{}, - dom: domFile, - dev: devFile, - lastUpdate: time.Now(), + lock: &sync.Mutex{}, + dom: dom, + dev: dev, + index: make([]byte, 8), + updated: false, }, nil } -func (s *Session) Lock() { +func (s *Session) Write(msg messages.Message) error { s.lock.Lock() -} + defer s.lock.Unlock() -func (s *Session) Unlock() { - s.lock.Unlock() -} + // Encode message index + binary.LittleEndian.PutUint64(s.index, msg.Meta().Index) -func (s *Session) Write(mode FileType, data []byte) (err error) { - if mode == DOM { - _, err = s.dom.Write(data) - } else { - _, err = s.dev.Write(data) + // Write message to dom.mob file + if messages.IsDOMType(msg.TypeID()) { + // Write message index + if err := s.dom.Write(s.index); err != nil { + return err + } + // Write message body + if err := s.dom.Write(msg.Encode()); err != nil { + return err + } } - s.lastUpdate = time.Now() - return err -} - -func (s *Session) LastUpdate() time.Time { - return s.lastUpdate + s.updated = true + // Write message to dev.mob file + if !messages.IsDOMType(msg.TypeID()) || msg.TypeID() == messages.MsgTimestamp { + // Write message index + if err := s.dev.Write(s.index); err != nil { + return err + } + // Write message body + if err := s.dev.Write(msg.Encode()); err != nil { + return err + } + } + return nil } func (s *Session) Sync() error { - domErr := s.dom.Sync() - devErr := s.dev.Sync() - if domErr == nil && devErr == nil { - return nil + s.lock.Lock() + defer s.lock.Unlock() + + if err := s.dom.Sync(); err != nil { + return err } - return fmt.Errorf("dom: %s, dev: %s", domErr, devErr) + return s.dev.Sync() } func (s *Session) Close() error { - domErr := s.dom.Close() - devErr := s.dev.Close() - if domErr == nil && devErr == nil { - return nil + s.lock.Lock() + defer s.lock.Unlock() + + if err := s.dom.Close(); err != nil { + return err } - return fmt.Errorf("dom: %s, dev: %s", domErr, devErr) + return s.dev.Close() } diff --git a/backend/internal/sink/sessionwriter/types.go b/backend/internal/sink/sessionwriter/types.go deleted file mode 100644 index a20f61375..000000000 --- a/backend/internal/sink/sessionwriter/types.go +++ /dev/null @@ -1,8 +0,0 @@ -package sessionwriter - -type FileType int - -const ( - DOM FileType = 1 - DEV FileType = 2 -) diff --git a/backend/internal/sink/sessionwriter/writer.go b/backend/internal/sink/sessionwriter/writer.go index 94ff5dd66..f2eb052c7 100644 --- a/backend/internal/sink/sessionwriter/writer.go +++ b/backend/internal/sink/sessionwriter/writer.go @@ -3,47 +3,89 @@ package sessionwriter import ( "fmt" "log" - "math" "sync" "time" + + "openreplay/backend/pkg/messages" ) type SessionWriter struct { - ulimit int - dir string - zombieSessionTimeout float64 - lock *sync.Mutex - sessions *sync.Map - meta map[uint64]int64 - done chan struct{} - stopped chan struct{} + filesLimit int + workingDir string + fileBuffer int + syncTimeout time.Duration + meta *Meta + sessions *sync.Map + done chan struct{} + stopped chan struct{} } -func NewWriter(ulimit uint16, dir string, zombieSessionTimeout int64) *SessionWriter { +func NewWriter(filesLimit uint16, workingDir string, fileBuffer int, syncTimeout int) *SessionWriter { w := &SessionWriter{ - ulimit: int(ulimit) / 2, // should divide by 2 because each session has 2 files - dir: dir + "/", - zombieSessionTimeout: float64(zombieSessionTimeout), - lock: &sync.Mutex{}, - sessions: &sync.Map{}, - meta: make(map[uint64]int64, ulimit), - done: make(chan struct{}), - stopped: make(chan struct{}), + filesLimit: int(filesLimit) / 2, // should divide by 2 because each session has 2 files + workingDir: workingDir + "/", + fileBuffer: fileBuffer, + syncTimeout: time.Duration(syncTimeout) * time.Second, + meta: NewMeta(int(filesLimit)), + sessions: &sync.Map{}, + done: make(chan struct{}), + stopped: make(chan struct{}), } go w.synchronizer() return w } -func (w *SessionWriter) WriteDOM(sid uint64, data []byte) error { - return w.write(sid, DOM, data) +func (w *SessionWriter) Write(msg messages.Message) (err error) { + var ( + sess *Session + sid = msg.SessionID() + ) + + // Load session + sessObj, ok := w.sessions.Load(sid) + if !ok { + // Create new session + sess, err = NewSession(sid, w.workingDir, w.fileBuffer) + if err != nil { + return fmt.Errorf("can't create session: %d, err: %s", sid, err) + } + + // Check opened sessions limit and close extra session if you need to + if extraSessID := w.meta.GetExtra(); extraSessID != 0 { + if err := w.Close(extraSessID); err != nil { + log.Printf("can't close session: %s", err) + } + } + + // Add created session + w.sessions.Store(sid, sess) + w.meta.Add(sid) + } else { + sess = sessObj.(*Session) + } + + // Write data to session + return sess.Write(msg) } -func (w *SessionWriter) WriteDEV(sid uint64, data []byte) error { - return w.write(sid, DEV, data) +func (w *SessionWriter) sync(sid uint64) error { + sessObj, ok := w.sessions.Load(sid) + if !ok { + return fmt.Errorf("session: %d not found", sid) + } + sess := sessObj.(*Session) + return sess.Sync() } -func (w *SessionWriter) Close(sid uint64) { - w.close(sid) +func (w *SessionWriter) Close(sid uint64) error { + sessObj, ok := w.sessions.LoadAndDelete(sid) + if !ok { + return fmt.Errorf("session: %d not found", sid) + } + sess := sessObj.(*Session) + err := sess.Close() + w.meta.Delete(sid) + return err } func (w *SessionWriter) Stop() { @@ -52,110 +94,11 @@ func (w *SessionWriter) Stop() { } func (w *SessionWriter) Info() string { - return fmt.Sprintf("%d sessions", w.numberOfSessions()) -} - -func (w *SessionWriter) addSession(sid uint64) { - w.lock.Lock() - w.meta[sid] = time.Now().Unix() - w.lock.Unlock() -} - -func (w *SessionWriter) deleteSession(sid uint64) { - w.lock.Lock() - delete(w.meta, sid) - w.lock.Unlock() -} - -func (w *SessionWriter) numberOfSessions() int { - w.lock.Lock() - defer w.lock.Unlock() - return len(w.meta) -} - -func (w *SessionWriter) write(sid uint64, mode FileType, data []byte) error { - var ( - sess *Session - err error - ) - - sessObj, ok := w.sessions.Load(sid) - if !ok { - sess, err = NewSession(w.dir, sid) - if err != nil { - return fmt.Errorf("can't write to session: %d, err: %s", sid, err) - } - sess.Lock() - defer sess.Unlock() - - // Check opened files limit - if len(w.meta) >= w.ulimit { - var oldSessID uint64 - var minTimestamp int64 = math.MaxInt64 - for sessID, timestamp := range w.meta { - if timestamp < minTimestamp { - oldSessID = sessID - minTimestamp = timestamp - } - } - if err := w.close(oldSessID); err != nil { - log.Printf("can't close session: %s", err) - } - } - - // Add new session to manager - w.sessions.Store(sid, sess) - w.addSession(sid) - } else { - sess = sessObj.(*Session) - sess.Lock() - defer sess.Unlock() - } - - // Write data to session - return sess.Write(mode, data) -} - -func (w *SessionWriter) sync(sid uint64) error { - sessObj, ok := w.sessions.Load(sid) - if !ok { - return fmt.Errorf("can't sync, session: %d not found", sid) - } - sess := sessObj.(*Session) - sess.Lock() - defer sess.Unlock() - - err := sess.Sync() - if time.Now().Sub(sess.LastUpdate()).Seconds() > w.zombieSessionTimeout { - if err != nil { - log.Printf("can't sync session: %d, err: %s", sid, err) - } - // Close "zombie" session - err = sess.Close() - w.deleteSession(sid) - } - return err -} - -func (w *SessionWriter) close(sid uint64) error { - sessObj, ok := w.sessions.LoadAndDelete(sid) - if !ok { - return fmt.Errorf("can't close, session: %d not found", sid) - } - sess := sessObj.(*Session) - sess.Lock() - defer sess.Unlock() - - if err := sess.Sync(); err != nil { - log.Printf("can't sync session: %d, err: %s", sid, err) - } - err := sess.Close() - w.deleteSession(sid) - return err + return fmt.Sprintf("%d sessions", w.meta.Count()) } func (w *SessionWriter) synchronizer() { - tick := time.Tick(2 * time.Second) + tick := time.Tick(w.syncTimeout) for { select { case <-tick: @@ -167,7 +110,7 @@ func (w *SessionWriter) synchronizer() { }) case <-w.done: w.sessions.Range(func(sid, lockObj any) bool { - if err := w.close(sid.(uint64)); err != nil { + if err := w.Close(sid.(uint64)); err != nil { log.Printf("can't close file descriptor: %s", err) } return true diff --git a/backend/pkg/messages/iterator.go b/backend/pkg/messages/iterator.go index 8b23cb97e..7b7991b19 100644 --- a/backend/pkg/messages/iterator.go +++ b/backend/pkg/messages/iterator.go @@ -100,6 +100,7 @@ func (i *messageIteratorImpl) Iterate(batchData []byte, batchInfo *BatchInfo) { tp: msgType, size: i.size, reader: reader, + raw: batchData, skipped: &i.canSkip, broken: &i.broken, meta: i.messageInfo, diff --git a/backend/pkg/messages/primitives.go b/backend/pkg/messages/primitives.go index eb65ae7b1..1d3d2410b 100644 --- a/backend/pkg/messages/primitives.go +++ b/backend/pkg/messages/primitives.go @@ -8,13 +8,17 @@ import ( "log" ) +var ( + one = []byte{0} + three = []byte{0, 0, 0} +) + func ReadByte(reader io.Reader) (byte, error) { - p := make([]byte, 1) - _, err := io.ReadFull(reader, p) + _, err := io.ReadFull(reader, one) if err != nil { return 0, err } - return p[0], nil + return one[0], nil } func ReadData(reader io.Reader) ([]byte, error) { @@ -156,8 +160,7 @@ func WriteSize(size uint64, buf []byte, p int) { } func ReadSize(reader io.Reader) (uint64, error) { - buf := make([]byte, 3) - n, err := io.ReadFull(reader, buf) + n, err := io.ReadFull(reader, three) if err != nil { return 0, err } @@ -165,7 +168,7 @@ func ReadSize(reader io.Reader) (uint64, error) { return 0, fmt.Errorf("read only %d of 3 size bytes", n) } var size uint64 - for i, b := range buf { + for i, b := range three { size += uint64(b) << (8 * i) } return size, nil diff --git a/backend/pkg/messages/raw.go b/backend/pkg/messages/raw.go index 33419d115..dbc71f4e6 100644 --- a/backend/pkg/messages/raw.go +++ b/backend/pkg/messages/raw.go @@ -13,6 +13,7 @@ type RawMessage struct { size uint64 data []byte reader *bytes.Reader + raw []byte meta *message encoded bool skipped *bool @@ -23,15 +24,25 @@ func (m *RawMessage) Encode() []byte { if m.encoded { return m.data } - m.data = make([]byte, m.size+1) - m.data[0] = uint8(m.tp) - m.encoded = true - *m.skipped = false - _, err := io.ReadFull(m.reader, m.data[1:]) - if err != nil { - log.Printf("message encode err: %s, type: %d, sess: %d", err, m.tp, m.SessionID()) + // Try to avoid EOF error + if m.reader.Len() < int(m.size) { return nil } + // Get current batch position + currPos, err := m.reader.Seek(0, io.SeekCurrent) + if err != nil { + log.Printf("can't get current batch position: %s", err) + return nil + } + // "Move" message type + if currPos == 0 { + log.Printf("can't move message type, curr position = %d", currPos) + return nil + } + // Dirty hack to avoid extra memory allocation + m.raw[currPos-1] = uint8(m.tp) + m.data = m.raw[currPos-1 : currPos+int64(m.size)] + m.encoded = true return m.data } From 159794ce72254a551327cfa68a5723c77816faab Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 25 Nov 2022 17:26:41 +0100 Subject: [PATCH 04/22] feat(chalice): changed funnel limits --- api/chalicelib/core/significance.py | 9 ++++++--- ee/api/chalicelib/core/significance.py | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index 52a22aee8..c3ac7077f 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -191,7 +191,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: GROUP BY main.session_id) AS T{i + 1} {"ON (TRUE)" if i > 0 else ""} """) - n_stages=len(n_stages_query) + n_stages = len(n_stages_query) if n_stages == 0: return [] n_stages_query = " LEFT JOIN LATERAL ".join(n_stages_query) @@ -215,7 +215,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: AND ISE.session_id = stages_t.session_id AND ISS.type!='custom' -- ignore custom issues because they are massive {"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""} - LIMIT 50 -- remove the limit to get exact stats + LIMIT 10 -- remove the limit to get exact stats ) AS issues_t ON (TRUE) ) AS stages_and_issues_t INNER JOIN sessions USING(session_id); """ @@ -348,7 +348,7 @@ def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues, if error_id not in errors: errors[error_id] = [] ic = 0 - row_issue_id=row['issue_id'] + row_issue_id = row['issue_id'] if row_issue_id is not None: if last_ts is None or (first_ts < row['issue_timestamp'] < last_ts): if error_id == row_issue_id: @@ -533,6 +533,9 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False) if is_sign: n_critical_issues += n_issues_dict[issue_id] + # To limit the number of returned issues to the frontend + issues_dict["significant"] = issues_dict["significant"][:50] + issues_dict["insignificant"] = issues_dict["insignificant"][:50] return n_critical_issues, issues_dict, total_drop_due_to_issues diff --git a/ee/api/chalicelib/core/significance.py b/ee/api/chalicelib/core/significance.py index d2ad650b4..b46adb589 100644 --- a/ee/api/chalicelib/core/significance.py +++ b/ee/api/chalicelib/core/significance.py @@ -198,7 +198,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: GROUP BY main.session_id) AS T{i + 1} {"ON (TRUE)" if i > 0 else ""} """) - n_stages=len(n_stages_query) + n_stages = len(n_stages_query) if n_stages == 0: return [] n_stages_query = " LEFT JOIN LATERAL ".join(n_stages_query) @@ -222,7 +222,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: AND ISE.session_id = stages_t.session_id AND ISS.type!='custom' -- ignore custom issues because they are massive {"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""} - LIMIT 50 -- remove the limit to get exact stats + LIMIT 10 -- remove the limit to get exact stats ) AS issues_t ON (TRUE) ) AS stages_and_issues_t INNER JOIN sessions USING(session_id); """ @@ -355,7 +355,7 @@ def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues, if error_id not in errors: errors[error_id] = [] ic = 0 - row_issue_id=row['issue_id'] + row_issue_id = row['issue_id'] if row_issue_id is not None: if last_ts is None or (first_ts < row['issue_timestamp'] < last_ts): if error_id == row_issue_id: @@ -540,6 +540,9 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False) if is_sign: n_critical_issues += n_issues_dict[issue_id] + # To limit the number of returned issues to the frontend + issues_dict["significant"] = issues_dict["significant"][:50] + issues_dict["insignificant"] = issues_dict["insignificant"][:50] return n_critical_issues, issues_dict, total_drop_due_to_issues From c1e6ba87200d6a5e5dc9dda88b6901d6baadc761 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 25 Nov 2022 17:31:04 +0100 Subject: [PATCH 05/22] feat(chalice): changed funnel limits --- api/chalicelib/core/significance.py | 4 ++-- ee/api/chalicelib/core/significance.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index c3ac7077f..c4a4fcaac 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -534,8 +534,8 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False) if is_sign: n_critical_issues += n_issues_dict[issue_id] # To limit the number of returned issues to the frontend - issues_dict["significant"] = issues_dict["significant"][:50] - issues_dict["insignificant"] = issues_dict["insignificant"][:50] + issues_dict["significant"] = issues_dict["significant"][:20] + issues_dict["insignificant"] = issues_dict["insignificant"][:20] return n_critical_issues, issues_dict, total_drop_due_to_issues diff --git a/ee/api/chalicelib/core/significance.py b/ee/api/chalicelib/core/significance.py index b46adb589..e3d6cc735 100644 --- a/ee/api/chalicelib/core/significance.py +++ b/ee/api/chalicelib/core/significance.py @@ -541,8 +541,8 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False) if is_sign: n_critical_issues += n_issues_dict[issue_id] # To limit the number of returned issues to the frontend - issues_dict["significant"] = issues_dict["significant"][:50] - issues_dict["insignificant"] = issues_dict["insignificant"][:50] + issues_dict["significant"] = issues_dict["significant"][:20] + issues_dict["insignificant"] = issues_dict["insignificant"][:20] return n_critical_issues, issues_dict, total_drop_due_to_issues From a0d7188910df334498c43c0dbe6fb1c423ace538 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 25 Nov 2022 17:32:17 +0100 Subject: [PATCH 06/22] fix(ui) - safari width fix --- .../app/components/shared/Filters/FilterValue/FilterValue.tsx | 3 +-- .../components/LatestSessionsMessage/LatestSessionsMessage.tsx | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/shared/Filters/FilterValue/FilterValue.tsx b/frontend/app/components/shared/Filters/FilterValue/FilterValue.tsx index f9e9dea2e..7daab8acb 100644 --- a/frontend/app/components/shared/Filters/FilterValue/FilterValue.tsx +++ b/frontend/app/components/shared/Filters/FilterValue/FilterValue.tsx @@ -175,8 +175,7 @@ function FilterValue(props: Props) { }; return ( - // -
+
{filter.type === FilterType.DURATION ? renderValueFiled(filter.value, 0) : filter.value && diff --git a/frontend/app/components/shared/SessionListContainer/components/LatestSessionsMessage/LatestSessionsMessage.tsx b/frontend/app/components/shared/SessionListContainer/components/LatestSessionsMessage/LatestSessionsMessage.tsx index c1283d357..68052dd3c 100644 --- a/frontend/app/components/shared/SessionListContainer/components/LatestSessionsMessage/LatestSessionsMessage.tsx +++ b/frontend/app/components/shared/SessionListContainer/components/LatestSessionsMessage/LatestSessionsMessage.tsx @@ -16,7 +16,7 @@ function LatestSessionsMessage(props: Props) { style={{ backgroundColor: 'rgb(255 251 235)' }} onClick={() => props.updateCurrentPage(1)} > - Show {numberWithCommas(count)} new Sessions + Show {numberWithCommas(count)} New Sessions
) : ( <> From 30014d0b6dfba1e7e8517a30dd22fa983cdaf1f2 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Fri, 25 Nov 2022 18:09:44 +0100 Subject: [PATCH 07/22] fix(tracker):tracker-version 4.1.9 --- tracker/tracker/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 8e16b6c54..6cf707862 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "4.1.8", + "version": "4.1.9", "keywords": [ "logging", "replay" From 3d8ad1a6253cf6f6f49c9f328508d4dfb8c1ae6d Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 25 Nov 2022 18:33:33 +0100 Subject: [PATCH 08/22] fix(ui) - issue form fetch the projects --- frontend/app/components/Session_/Issues/IssueForm.js | 2 -- frontend/app/components/Session_/Issues/Issues.js | 1 + frontend/app/components/ui/Popover/Popover.tsx | 11 +++++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/frontend/app/components/Session_/Issues/IssueForm.js b/frontend/app/components/Session_/Issues/IssueForm.js index 757f5d206..4b6585985 100644 --- a/frontend/app/components/Session_/Issues/IssueForm.js +++ b/frontend/app/components/Session_/Issues/IssueForm.js @@ -76,8 +76,6 @@ class IssueForm extends React.PureComponent { const selectedIssueType = issueTypes.filter((issue) => issue.id == instance.issueType)[0]; - console.log('instance', instance); - return (
diff --git a/frontend/app/components/Session_/Issues/Issues.js b/frontend/app/components/Session_/Issues/Issues.js index a7a0c6528..3c037a019 100644 --- a/frontend/app/components/Session_/Issues/Issues.js +++ b/frontend/app/components/Session_/Issues/Issues.js @@ -70,6 +70,7 @@ class Issues extends React.Component {
(
void; labelId: string; descriptionId: string }) => React.ReactNode; placement?: Placement; children: JSX.Element; + onOpen?: () => void; } -const Popover = ({ children, render, placement }: Props) => { +const Popover = ({ children, render, placement, onOpen = () => {} }: Props) => { const [open, setOpen] = useState(false); + useEffect(() => { + if (open) { + onOpen(); + } + }, [open]); + const { x, y, reference, floating, strategy, context } = useFloating({ open, onOpenChange: setOpen, From 5d77f8439c4df02b3a2576e6bb88134ed32f824f Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Fri, 25 Nov 2022 18:45:20 +0100 Subject: [PATCH 09/22] fix(tracker-assist):4.1.3:agents reconnection on new page(no agentInfo) --- tracker/tracker-assist/src/Assist.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index e5b3c61cf..a742c4fbd 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -49,7 +49,7 @@ type OptionalCallback = (()=>Record) | void type Agent = { onDisconnect?: OptionalCallback, onControlReleased?: OptionalCallback, - agentInfo: Record + agentInfo: Record | undefined // } @@ -229,9 +229,10 @@ export default class Assist { }) socket.on('AGENTS_CONNECTED', (ids: string[]) => { ids.forEach(id =>{ + const agentInfo = this.agents[id]?.agentInfo this.agents[id] = { - ...this.agents[id], - onDisconnect: this.options.onAgentConnect?.( this.agents[id].agentInfo), + agentInfo, + onDisconnect: this.options.onAgentConnect?.(agentInfo), } }) this.assistDemandedRestart = true From 4a4cfc20892ab88a6f729465f362257b7d81a82b Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 25 Nov 2022 18:44:36 +0100 Subject: [PATCH 10/22] change(ui) - more from over to click --- .../Session_/EventsBlock/UserCard/UserCard.js | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/frontend/app/components/Session_/EventsBlock/UserCard/UserCard.js b/frontend/app/components/Session_/EventsBlock/UserCard/UserCard.js index 3d3f2b550..2e4e4daa1 100644 --- a/frontend/app/components/Session_/EventsBlock/UserCard/UserCard.js +++ b/frontend/app/components/Session_/EventsBlock/UserCard/UserCard.js @@ -5,7 +5,7 @@ import { countries } from 'App/constants'; import { useStore } from 'App/mstore'; import { browserIcon, osIcon, deviceTypeIcon } from 'App/iconNames'; import { formatTimeOrDate } from 'App/date'; -import { Avatar, TextEllipsis, CountryFlag, Icon, Tooltip } from 'UI'; +import { Avatar, TextEllipsis, CountryFlag, Icon, Tooltip, Popover } from 'UI'; import cn from 'classnames'; import { withRequest } from 'HOCs'; import SessionInfoItem from '../../SessionInfoItem'; @@ -75,15 +75,9 @@ function UserCard({ className, request, session, width, height, similarSessions, {userBrowser}, {userOs}, {userDevice} · - + ( +
} label={countries[userCountry]} @@ -99,14 +93,10 @@ function UserCard({ className, request, session, width, height, similarSessions, /> {revId && }
- } - position="bottom" - // hoverable - // disabled={false} - on="hover" + )} > - More - + More +
From 3d816053ec0c57abbbde9dddafc8d5fdc2474f63 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 25 Nov 2022 19:11:42 +0100 Subject: [PATCH 11/22] change(ui) - metric config while updating (moving back from funnel to other) --- frontend/app/mstore/types/widget.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/mstore/types/widget.ts b/frontend/app/mstore/types/widget.ts index 68e2813f4..62371fde3 100644 --- a/frontend/app/mstore/types/widget.ts +++ b/frontend/app/mstore/types/widget.ts @@ -131,7 +131,7 @@ export default class Widget { series: this.series.map((series: any) => series.toJson()), config: { ...this.config, - col: this.metricType === 'funnel' || this.metricOf === FilterKey.ERRORS || this.metricOf === FilterKey.SESSIONS ? 4 : this.config.col + col: (this.metricType === 'funnel' || this.metricOf === FilterKey.ERRORS || this.metricOf === FilterKey.SESSIONS) ? 4 : 2 }, } } From e0a88666af96c80a1b4c8468e3f09c50d5a0a376 Mon Sep 17 00:00:00 2001 From: Jorgen Evens Date: Wed, 10 Aug 2022 14:47:59 +0200 Subject: [PATCH 12/22] feat(redis): add suppport for credentials and TLS By using `ParseURL` it is now possible to - Supply login credentials for the redis instance - Use an encrypted TLS connection by setting the scheme to `rediss://` --- backend/pkg/redisstream/redis.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/backend/pkg/redisstream/redis.go b/backend/pkg/redisstream/redis.go index 7dba0b537..434099879 100644 --- a/backend/pkg/redisstream/redis.go +++ b/backend/pkg/redisstream/redis.go @@ -2,6 +2,7 @@ package redisstream import ( "log" + "regexp" "github.com/go-redis/redis" @@ -14,9 +15,20 @@ func getRedisClient() *redis.Client { if redisClient != nil { return redisClient } - redisClient = redis.NewClient(&redis.Options{ - Addr: env.String("REDIS_STRING"), - }) + + connectionString := env.String("REDIS_STRING") + + match, _ := regexp.MatchString("^[^:]+://", connectionString) + if !match { + connectionString = "redis://" + connectionString + } + + options, err := redis.ParseURL(connectionString) + if err != nil { + log.Fatalln(err) + } + + redisClient = redis.NewClient(options) if _, err := redisClient.Ping().Result(); err != nil { log.Fatalln(err) } From 65ef10447a3eb2b1042df1f1bed505808278a210 Mon Sep 17 00:00:00 2001 From: Jorgen Evens Date: Wed, 10 Aug 2022 15:23:08 +0200 Subject: [PATCH 13/22] feat(redis): provide credentials using secrets - Compatible with `redis.existingSecret` in the redis chart. - Uses dependent environment variables for `REDIS_STRING` connection strings --- .../charts/assets/templates/deployment.yaml | 16 +++++++++++++++- .../charts/db/templates/deployment.yaml | 16 +++++++++++++++- .../charts/ender/templates/deployment.yaml | 16 +++++++++++++++- .../charts/frontend/templates/deployment.yaml | 16 +++++++++++++++- .../charts/heuristics/templates/deployment.yaml | 16 +++++++++++++++- .../charts/http/templates/deployment.yaml | 16 +++++++++++++++- .../integrations/templates/deployment.yaml | 16 +++++++++++++++- .../charts/sink/templates/deployment.yaml | 16 +++++++++++++++- .../charts/storage/templates/deployment.yaml | 16 +++++++++++++++- 9 files changed, 135 insertions(+), 9 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml index 350054599..d722997b1 100644 --- a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml @@ -69,8 +69,22 @@ spec: value: '{{ .Values.global.s3.endpoint }}' - name: AWS_REGION value: '{{ .Values.global.s3.region }}' + {{- if .Values.global.redis.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: redis-password + {{- else if .Values.global.redis.redisPassword }} + - name: REDIS_PASSWORD + value: {{ .Values.global.redis.redisPassword }} + {{- end}} - name: REDIS_STRING - value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- else }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL diff --git a/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml index 02831fa73..5cd90ec09 100644 --- a/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml @@ -47,8 +47,22 @@ spec: value: '{{ .Values.global.clickhouse.chHost }}:{{.Values.global.clickhouse.service.webPort}}/{{.Values.env.ch_db}}' - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' + {{- if .Values.global.redis.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: redis-password + {{- else if .Values.global.redis.redisPassword }} + - name: REDIS_PASSWORD + value: {{ .Values.global.redis.redisPassword }} + {{- end}} - name: REDIS_STRING - value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- else }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL diff --git a/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml index 22713eee8..a2c101cd9 100644 --- a/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml @@ -45,8 +45,22 @@ spec: env: - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' + {{- if .Values.global.redis.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: redis-password + {{- else if .Values.global.redis.redisPassword }} + - name: REDIS_PASSWORD + value: {{ .Values.global.redis.redisPassword }} + {{- end}} - name: REDIS_STRING - value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- else }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL diff --git a/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml index c41dc1313..aa782c023 100644 --- a/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml @@ -65,8 +65,22 @@ spec: value: '{{ .Values.global.s3.region }}' - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' + {{- if .Values.global.redis.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: redis-password + {{- else if .Values.global.redis.redisPassword }} + - name: REDIS_PASSWORD + value: {{ .Values.global.redis.redisPassword }} + {{- end}} - name: REDIS_STRING - value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- else }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL diff --git a/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml index 4e1b78605..fc33efe24 100644 --- a/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml @@ -45,8 +45,22 @@ spec: env: - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' + {{- if .Values.global.redis.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: redis-password + {{- else if .Values.global.redis.redisPassword }} + - name: REDIS_PASSWORD + value: {{ .Values.global.redis.redisPassword }} + {{- end}} - name: REDIS_STRING - value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- else }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL diff --git a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml index 6322ab754..81a144003 100644 --- a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml @@ -65,8 +65,22 @@ spec: value: '{{ .Values.global.s3.region }}' - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' + {{- if .Values.global.redis.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: redis-password + {{- else if .Values.global.redis.redisPassword }} + - name: REDIS_PASSWORD + value: {{ .Values.global.redis.redisPassword }} + {{- end}} - name: REDIS_STRING - value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- else }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL diff --git a/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml index 618d32f47..b80e246f9 100644 --- a/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml @@ -45,8 +45,22 @@ spec: env: - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' + {{- if .Values.global.redis.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: redis-password + {{- else if .Values.global.redis.redisPassword }} + - name: REDIS_PASSWORD + value: {{ .Values.global.redis.redisPassword }} + {{- end}} - name: REDIS_STRING - value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- else }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL diff --git a/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml index 20806f4a1..d8c2e406b 100644 --- a/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml @@ -45,8 +45,22 @@ spec: env: - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' + {{- if .Values.global.redis.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: redis-password + {{- else if .Values.global.redis.redisPassword }} + - name: REDIS_PASSWORD + value: {{ .Values.global.redis.redisPassword }} + {{- end}} - name: REDIS_STRING - value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- else }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL diff --git a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml index 6a60e3ab3..46a1b9700 100644 --- a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml @@ -73,8 +73,22 @@ spec: value: {{ .Values.global.s3.recordingsBucket }} - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' + {{- if .Values.global.redis.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.redis.existingSecret }} + key: redis-password + {{- else if .Values.global.redis.redisPassword }} + - name: REDIS_PASSWORD + value: {{ .Values.global.redis.redisPassword }} + {{- end}} - name: REDIS_STRING - value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- else }} + value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL From c91c97f1c7bb3d1416913ae6abca4756913d94b1 Mon Sep 17 00:00:00 2001 From: Jorgen Evens Date: Wed, 10 Aug 2022 16:46:05 +0200 Subject: [PATCH 14/22] feat(redis): cleanup REDIS_STRING generation --- .../charts/assets/templates/deployment.yaml | 17 +------------- .../charts/db/templates/deployment.yaml | 17 +------------- .../charts/ender/templates/deployment.yaml | 17 +------------- .../charts/frontend/templates/deployment.yaml | 17 +------------- .../heuristics/templates/deployment.yaml | 17 +------------- .../charts/http/templates/deployment.yaml | 17 +------------- .../integrations/templates/deployment.yaml | 17 +------------- .../charts/sink/templates/deployment.yaml | 17 +------------- .../charts/storage/templates/deployment.yaml | 17 +------------- .../openreplay/templates/_helpers.tpl | 23 +++++++++++++++++++ 10 files changed, 32 insertions(+), 144 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml index d722997b1..348a166c9 100644 --- a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml @@ -69,22 +69,6 @@ spec: value: '{{ .Values.global.s3.endpoint }}' - name: AWS_REGION value: '{{ .Values.global.s3.region }}' - {{- if .Values.global.redis.existingSecret }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.global.redis.existingSecret }} - key: redis-password - {{- else if .Values.global.redis.redisPassword }} - - name: REDIS_PASSWORD - value: {{ .Values.global.redis.redisPassword }} - {{- end}} - - name: REDIS_STRING - {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- else }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL @@ -109,6 +93,7 @@ spec: # S3 compatible storage value: '{{ .Values.global.s3.endpoint }}/{{.Values.global.s3.assetsBucket}}' {{- end }} + {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml index 5cd90ec09..ba1620a6d 100644 --- a/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml @@ -47,22 +47,6 @@ spec: value: '{{ .Values.global.clickhouse.chHost }}:{{.Values.global.clickhouse.service.webPort}}/{{.Values.env.ch_db}}' - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' - {{- if .Values.global.redis.existingSecret }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.global.redis.existingSecret }} - key: redis-password - {{- else if .Values.global.redis.redisPassword }} - - name: REDIS_PASSWORD - value: {{ .Values.global.redis.redisPassword }} - {{- end}} - - name: REDIS_STRING - {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- else }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL @@ -80,6 +64,7 @@ spec: value: '{{ .Values.global.quickwit.enabled }}' - name: POSTGRES_STRING value: 'postgres://{{ .Values.global.postgresql.postgresqlUser }}:$(pg_password)@{{ .Values.global.postgresql.postgresqlHost }}:{{ .Values.global.postgresql.postgresqlPort }}/{{ .Values.global.postgresql.postgresqlDatabase }}' + {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml index a2c101cd9..05487bd75 100644 --- a/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml @@ -45,22 +45,6 @@ spec: env: - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' - {{- if .Values.global.redis.existingSecret }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.global.redis.existingSecret }} - key: redis-password - {{- else if .Values.global.redis.redisPassword }} - - name: REDIS_PASSWORD - value: {{ .Values.global.redis.redisPassword }} - {{- end}} - - name: REDIS_STRING - {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- else }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL @@ -76,6 +60,7 @@ spec: {{- end}} - name: POSTGRES_STRING value: 'postgres://{{ .Values.global.postgresql.postgresqlUser }}:$(pg_password)@{{ .Values.global.postgresql.postgresqlHost }}:{{ .Values.global.postgresql.postgresqlPort }}/{{ .Values.global.postgresql.postgresqlDatabase }}' + {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml index aa782c023..4c4f8dbb9 100644 --- a/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml @@ -65,22 +65,6 @@ spec: value: '{{ .Values.global.s3.region }}' - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' - {{- if .Values.global.redis.existingSecret }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.global.redis.existingSecret }} - key: redis-password - {{- else if .Values.global.redis.redisPassword }} - - name: REDIS_PASSWORD - value: {{ .Values.global.redis.redisPassword }} - {{- end}} - - name: REDIS_STRING - {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- else }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL @@ -116,6 +100,7 @@ spec: # S3 compatible storage value: '{{ .Values.global.s3.endpoint }}/{{.Values.global.s3.assetsBucket}}' {{- end }} + {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml index fc33efe24..1d3d1cd8c 100644 --- a/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml @@ -45,26 +45,11 @@ spec: env: - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' - {{- if .Values.global.redis.existingSecret }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.global.redis.existingSecret }} - key: redis-password - {{- else if .Values.global.redis.redisPassword }} - - name: REDIS_PASSWORD - value: {{ .Values.global.redis.redisPassword }} - {{- end}} - - name: REDIS_STRING - {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- else }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL value: '{{ .Values.global.kafka.kafkaUseSsl }}' + {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml index 81a144003..2bc61d3cc 100644 --- a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml @@ -65,22 +65,6 @@ spec: value: '{{ .Values.global.s3.region }}' - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' - {{- if .Values.global.redis.existingSecret }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.global.redis.existingSecret }} - key: redis-password - {{- else if .Values.global.redis.redisPassword }} - - name: REDIS_PASSWORD - value: {{ .Values.global.redis.redisPassword }} - {{- end}} - - name: REDIS_STRING - {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- else }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL @@ -116,6 +100,7 @@ spec: # S3 compatible storage value: '{{ .Values.global.s3.endpoint }}/{{.Values.global.s3.assetsBucket}}' {{- end }} + {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml index b80e246f9..1e0ff879f 100644 --- a/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml @@ -45,22 +45,6 @@ spec: env: - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' - {{- if .Values.global.redis.existingSecret }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.global.redis.existingSecret }} - key: redis-password - {{- else if .Values.global.redis.redisPassword }} - - name: REDIS_PASSWORD - value: {{ .Values.global.redis.redisPassword }} - {{- end}} - - name: REDIS_STRING - {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- else }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL @@ -76,6 +60,7 @@ spec: {{- end}} - name: POSTGRES_STRING value: 'postgres://{{ .Values.global.postgresql.postgresqlUser }}:$(pg_password)@{{ .Values.global.postgresql.postgresqlHost }}:{{ .Values.global.postgresql.postgresqlPort }}/{{ .Values.global.postgresql.postgresqlDatabase }}' + {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml index d8c2e406b..44494f434 100644 --- a/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml @@ -45,22 +45,6 @@ spec: env: - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' - {{- if .Values.global.redis.existingSecret }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.global.redis.existingSecret }} - key: redis-password - {{- else if .Values.global.redis.redisPassword }} - - name: REDIS_PASSWORD - value: {{ .Values.global.redis.redisPassword }} - {{- end}} - - name: REDIS_STRING - {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- else }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL @@ -85,6 +69,7 @@ spec: # S3 compatible storage value: '{{ .Values.global.s3.endpoint }}/{{.Values.global.s3.assetsBucket}}' {{- end }} + {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml index 46a1b9700..8cf8784ef 100644 --- a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml @@ -73,26 +73,11 @@ spec: value: {{ .Values.global.s3.recordingsBucket }} - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' - {{- if .Values.global.redis.existingSecret }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.global.redis.existingSecret }} - key: redis-password - {{- else if .Values.global.redis.redisPassword }} - - name: REDIS_PASSWORD - value: {{ .Values.global.redis.redisPassword }} - {{- end}} - - name: REDIS_STRING - {{- if or .Values.global.redis.existingSecret .Values.global.redis.redisPassword }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisUsername }}:$(REDIS_PASSWORD)@{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- else }} - value: '{{ ternary "rediss" "redis" .Values.global.redis.tls.enabled }}://{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - {{- end}} - name: KAFKA_SERVERS value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - name: KAFKA_USE_SSL value: '{{ .Values.global.kafka.kafkaUseSsl }}' + {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/templates/_helpers.tpl b/scripts/helmcharts/openreplay/templates/_helpers.tpl index ea4e67add..87ad3fd5c 100644 --- a/scripts/helmcharts/openreplay/templates/_helpers.tpl +++ b/scripts/helmcharts/openreplay/templates/_helpers.tpl @@ -60,3 +60,26 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Create the environment configuration for REDIS_STRING +*/}} +{{- define "openreplay.env.redis_string" -}} +{{- $scheme := (eq .tls.enabled true) | ternary "rediss" "redis" -}} +{{- $auth := "" -}} +{{- if or .existingSecret .redisPassword -}} + {{- $auth = printf "%s:$(REDIS_PASSWORD)@" (.redisUsername | default "") -}} +{{- end -}} +{{- if .existingSecret -}} +- name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .existingSecret }} + key: redis-password +{{- else if .redisPassword }} +- name: REDIS_PASSWORD + value: {{ .redisPassword }} +{{- end}} +- name: REDIS_STRING + value: '{{ $scheme }}://{{ $auth }}{{ .redisHost }}:{{ .redisPort }}' +{{- end }} From 0cbbf43890af3f71260cfb7b607dfafe1f10d25d Mon Sep 17 00:00:00 2001 From: Jorgen Evens Date: Wed, 10 Aug 2022 17:33:28 +0200 Subject: [PATCH 15/22] feat(redis): add support for custom TLS CA certificates --- .../charts/assets/templates/deployment.yaml | 6 ++++-- .../charts/db/templates/deployment.yaml | 6 ++++-- .../charts/ender/templates/deployment.yaml | 6 ++++-- .../charts/frontend/templates/deployment.yaml | 10 ++++++++++ .../openreplay/charts/frontend/values.yaml | 2 ++ .../heuristics/templates/deployment.yaml | 6 ++++-- .../charts/http/templates/deployment.yaml | 6 ++++-- .../integrations/templates/deployment.yaml | 6 ++++-- .../charts/sink/templates/deployment.yaml | 3 +++ .../charts/storage/templates/deployment.yaml | 2 ++ .../openreplay/templates/_helpers.tpl | 19 +++++++++++++++++++ 11 files changed, 60 insertions(+), 12 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml index 348a166c9..f66479475 100644 --- a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml @@ -104,14 +104,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} - {{- with .Values.persistence.mounts }} volumeMounts: + {{- include "openreplay.volume.redis_ca_certificate.mount" .Values.global.redis | nindent 12 }} + {{- with .Values.persistence.mounts }} {{- toYaml . | nindent 12 }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.persistence.volumes }} volumes: + {{- include "openreplay.volume.redis_ca_certificate" .Values.global.redis | nindent 8 }} + {{- with .Values.persistence.volumes }} {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.nodeSelector }} diff --git a/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml index ba1620a6d..039e889a1 100644 --- a/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml @@ -75,16 +75,18 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} - {{- with .Values.persistence.mounts }} volumeMounts: + {{- include "openreplay.volume.redis_ca_certificate.mount" .Values.global.redis | nindent 12 }} + {{- with .Values.persistence.mounts }} {{- toYaml . | nindent 12 }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.persistence.volumes }} volumes: + {{- with .Values.persistence.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{- include "openreplay.volume.redis_ca_certificate" .Values.global.redis | nindent 8 }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml index 05487bd75..e5b0a946b 100644 --- a/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml @@ -71,16 +71,18 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} - {{- with .Values.persistence.mounts }} volumeMounts: + {{- include "openreplay.volume.redis_ca_certificate.mount" .Values.global.redis | nindent 12 }} + {{- with .Values.persistence.mounts }} {{- toYaml . | nindent 12 }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.persistence.volumes }} volumes: + {{- with .Values.persistence.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{- include "openreplay.volume.redis_ca_certificate" .Values.global.redis | nindent 8 }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml index 4c4f8dbb9..e5eb29441 100644 --- a/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml @@ -111,6 +111,11 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + volumeMounts: + {{- include "openreplay.volume.redis_ca_certificate.mount" .Values.global.redis | nindent 12 }} + {{- with .Values.persistence.mounts }} + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} @@ -125,3 +130,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + volumes: + {{- with .Values.persistence.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "openreplay.volume.redis_ca_certificate" .Values.global.redis | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/frontend/values.yaml b/scripts/helmcharts/openreplay/charts/frontend/values.yaml index 1171b284d..a34841fb2 100644 --- a/scripts/helmcharts/openreplay/charts/frontend/values.yaml +++ b/scripts/helmcharts/openreplay/charts/frontend/values.yaml @@ -99,3 +99,5 @@ nodeSelector: {} tolerations: [] affinity: {} + +persistence: {} diff --git a/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml index 1d3d1cd8c..6d88fec7a 100644 --- a/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml @@ -60,16 +60,18 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} - {{- with .Values.persistence.mounts }} volumeMounts: + {{- include "openreplay.volume.redis_ca_certificate.mount" .Values.global.redis | nindent 12 }} + {{- with .Values.persistence.mounts }} {{- toYaml . | nindent 12 }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.persistence.volumes }} volumes: + {{- with .Values.persistence.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{- include "openreplay.volume.redis_ca_certificate" .Values.global.redis | nindent 8 }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml index 2bc61d3cc..9f7d407bb 100644 --- a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml @@ -111,14 +111,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} - {{- with .Values.persistence.mounts }} volumeMounts: + {{- include "openreplay.volume.redis_ca_certificate.mount" .Values.global.redis | nindent 12 }} + {{- with .Values.persistence.mounts }} {{- toYaml . | nindent 12 }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.persistence.volumes }} volumes: + {{- include "openreplay.volume.redis_ca_certificate" .Values.global.redis | nindent 8 }} + {{- with .Values.persistence.volumes }} {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.nodeSelector }} diff --git a/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml index 1e0ff879f..0f9ead73c 100644 --- a/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml @@ -71,14 +71,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} - {{- with .Values.persistence.mounts }} volumeMounts: + {{- include "openreplay.volume.redis_ca_certificate.mount" .Values.global.redis | nindent 12 }} + {{- with .Values.persistence.mounts }} {{- toYaml . | nindent 12 }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.persistence.volumes }} volumes: + {{- include "openreplay.volume.redis_ca_certificate" .Values.global.redis | nindent 8 }} + {{- with .Values.persistence.volumes }} {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.nodeSelector }} diff --git a/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml index 44494f434..257af0d1a 100644 --- a/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml @@ -85,6 +85,7 @@ spec: volumeMounts: - name: datadir mountPath: /mnt/efs + {{- include "openreplay.volume.redis_ca_certificate.mount" .Values.global.redis | nindent 10 }} {{- with .Values.persistence.mounts }} {{- toYaml . | nindent 10 }} {{- end }} @@ -95,6 +96,7 @@ spec: # Ensure the file directory is created. path: {{ .Values.pvc.hostMountPath }} type: DirectoryOrCreate + {{- include "openreplay.volume.redis_ca_certificate" .Values.global.redis | nindent 6 }} {{- with .Values.persistence.volumes }} {{- toYaml . | nindent 6 }} {{- end }} @@ -103,6 +105,7 @@ spec: - name: datadir persistentVolumeClaim: claimName: {{ .Values.pvc.name }} + {{- include "openreplay.volume.redis_ca_certificate" .Values.global.redis | nindent 6 }} {{- with .Values.persistence.volumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml index 8cf8784ef..d20059fdc 100644 --- a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml @@ -93,6 +93,7 @@ spec: volumeMounts: - name: datadir mountPath: /mnt/efs + {{- include "openreplay.volume.redis_ca_certificate.mount" .Values.global.redis | nindent 10 }} {{- with .Values.persistence.mounts }} {{- toYaml . | nindent 10 }} {{- end }} @@ -115,6 +116,7 @@ spec: persistentVolumeClaim: claimName: {{ .Values.pvc.name }} {{- end }} + {{- include "openreplay.volume.redis_ca_certificate" .Values.global.redis | nindent 6 }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/templates/_helpers.tpl b/scripts/helmcharts/openreplay/templates/_helpers.tpl index 87ad3fd5c..a884509d8 100644 --- a/scripts/helmcharts/openreplay/templates/_helpers.tpl +++ b/scripts/helmcharts/openreplay/templates/_helpers.tpl @@ -83,3 +83,22 @@ Create the environment configuration for REDIS_STRING - name: REDIS_STRING value: '{{ $scheme }}://{{ $auth }}{{ .redisHost }}:{{ .redisPort }}' {{- end }} + +{{/* +Create the volume mount config for redis TLS certificates +*/}} +{{- define "openreplay.volume.redis_ca_certificate" -}} +{{- if and (.tls.enabled) (.tls.certificatesSecret) (.tls.certCAFilename) -}} +- name: redis-ca-certificate + secret: + secretName: {{ .tls.certificatesSecret }} +{{- end }} +{{- end }} + +{{- define "openreplay.volume.redis_ca_certificate.mount" -}} +{{- if and (.tls.enabled) (.tls.certificatesSecret) (.tls.certCAFilename) -}} +- name: redis-ca-certificate + mountPath: /etc/ssl/certs/redis-ca-certificate.pem + subPath: {{ .tls.certCAFilename }} +{{- end }} +{{- end }} From c63ce29759621dce1be7a3fead58c04d6585078f Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Fri, 25 Nov 2022 15:17:37 +0000 Subject: [PATCH 16/22] chore(helm): Adding default value to redis tls dict As redis.tls is not defained in vars.yaml, helm will throw nil point exception. Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/templates/_helpers.tpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/helmcharts/openreplay/templates/_helpers.tpl b/scripts/helmcharts/openreplay/templates/_helpers.tpl index a884509d8..400ff0366 100644 --- a/scripts/helmcharts/openreplay/templates/_helpers.tpl +++ b/scripts/helmcharts/openreplay/templates/_helpers.tpl @@ -65,7 +65,7 @@ Create the name of the service account to use Create the environment configuration for REDIS_STRING */}} {{- define "openreplay.env.redis_string" -}} -{{- $scheme := (eq .tls.enabled true) | ternary "rediss" "redis" -}} +{{- $scheme := (eq (.tls | default dict).enabled true) | ternary "rediss" "redis" -}} {{- $auth := "" -}} {{- if or .existingSecret .redisPassword -}} {{- $auth = printf "%s:$(REDIS_PASSWORD)@" (.redisUsername | default "") -}} @@ -88,7 +88,7 @@ Create the environment configuration for REDIS_STRING Create the volume mount config for redis TLS certificates */}} {{- define "openreplay.volume.redis_ca_certificate" -}} -{{- if and (.tls.enabled) (.tls.certificatesSecret) (.tls.certCAFilename) -}} +{{- if and ((.tls | default dict).enabled) (.tls.certificatesSecret) (.tls.certCAFilename) -}} - name: redis-ca-certificate secret: secretName: {{ .tls.certificatesSecret }} @@ -96,7 +96,7 @@ Create the volume mount config for redis TLS certificates {{- end }} {{- define "openreplay.volume.redis_ca_certificate.mount" -}} -{{- if and (.tls.enabled) (.tls.certificatesSecret) (.tls.certCAFilename) -}} +{{- if and ((.tls |default dict).enabled) (.tls.certificatesSecret) (.tls.certCAFilename) -}} - name: redis-ca-certificate mountPath: /etc/ssl/certs/redis-ca-certificate.pem subPath: {{ .tls.certCAFilename }} From 0581ea7a63b4707c7317b315c3ded4994a73407e Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 25 Nov 2022 19:22:49 +0100 Subject: [PATCH 17/22] change(ui) - latest session message checking for more than 1 --- .../components/LatestSessionsMessage/LatestSessionsMessage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/shared/SessionListContainer/components/LatestSessionsMessage/LatestSessionsMessage.tsx b/frontend/app/components/shared/SessionListContainer/components/LatestSessionsMessage/LatestSessionsMessage.tsx index 68052dd3c..39ed264bf 100644 --- a/frontend/app/components/shared/SessionListContainer/components/LatestSessionsMessage/LatestSessionsMessage.tsx +++ b/frontend/app/components/shared/SessionListContainer/components/LatestSessionsMessage/LatestSessionsMessage.tsx @@ -16,7 +16,7 @@ function LatestSessionsMessage(props: Props) { style={{ backgroundColor: 'rgb(255 251 235)' }} onClick={() => props.updateCurrentPage(1)} > - Show {numberWithCommas(count)} New Sessions + Show {numberWithCommas(count)} New {count > 1 ? 'Sessions' : 'Session'}
) : ( <> From fdaa60d16cdd73f1fc453572116946b375f4ff74 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 25 Nov 2022 19:27:25 +0100 Subject: [PATCH 18/22] feat(chalice): support changing metric's config feat(chalice): merge widget and metric's config --- api/chalicelib/core/custom_metrics.py | 6 ++++-- api/chalicelib/core/dashboards.py | 2 ++ api/schemas.py | 6 +++--- ee/api/chalicelib/core/custom_metrics.py | 6 ++++-- ee/api/chalicelib/core/dashboards.py | 2 ++ 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 29c4b6fa9..743ca41e5 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -266,7 +266,8 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche params = {"metric_id": metric_id, "is_public": data.is_public, "name": data.name, "user_id": user_id, "project_id": project_id, "view_type": data.view_type, "metric_type": data.metric_type, "metric_of": data.metric_of, - "metric_value": data.metric_value, "metric_format": data.metric_format} + "metric_value": data.metric_value, "metric_format": data.metric_format, + "config": json.dumps(data.config.dict())} for i, s in enumerate(data.series): prefix = "u_" if s.index is None: @@ -316,7 +317,8 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche view_type= %(view_type)s, metric_type= %(metric_type)s, metric_of= %(metric_of)s, metric_value= %(metric_value)s, metric_format= %(metric_format)s, - edited_at = timezone('utc'::text, now()) + edited_at = timezone('utc'::text, now()), + default_config = %(config)s WHERE metric_id = %(metric_id)s AND project_id = %(project_id)s AND (user_id = %(user_id)s OR is_public) diff --git a/api/chalicelib/core/dashboards.py b/api/chalicelib/core/dashboards.py index 9d1dc4c81..ac98b44e7 100644 --- a/api/chalicelib/core/dashboards.py +++ b/api/chalicelib/core/dashboards.py @@ -111,6 +111,8 @@ def get_dashboard(project_id, user_id, dashboard_id): for w in row["widgets"]: w["created_at"] = TimeUTC.datetime_to_timestamp(w["created_at"]) w["edited_at"] = TimeUTC.datetime_to_timestamp(w["edited_at"]) + w["config"]["col"] = w["default_config"]["col"] + w["config"]["row"] = w["default_config"]["row"] for s in w["series"]: s["created_at"] = TimeUTC.datetime_to_timestamp(s["created_at"]) return helper.dict_to_camel_case(row) diff --git a/api/schemas.py b/api/schemas.py index f1f3d9cb7..960b89e76 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -874,14 +874,14 @@ class TryCustomMetricsPayloadSchema(CustomMetricChartPayloadSchema): class CustomMetricsConfigSchema(BaseModel): - col: Optional[int] = Field(default=2) - row: Optional[int] = Field(default=2) + col: Optional[int] = Field(...) + row: Optional[int] = Field(...) position: Optional[int] = Field(default=0) class CreateCustomMetricsSchema(TryCustomMetricsPayloadSchema): series: List[CustomMetricCreateSeriesSchema] = Field(..., min_items=1) - config: CustomMetricsConfigSchema = Field(default=CustomMetricsConfigSchema()) + config: CustomMetricsConfigSchema = Field(...) @root_validator(pre=True) def transform_series(cls, values): diff --git a/ee/api/chalicelib/core/custom_metrics.py b/ee/api/chalicelib/core/custom_metrics.py index b925429f6..e871a5646 100644 --- a/ee/api/chalicelib/core/custom_metrics.py +++ b/ee/api/chalicelib/core/custom_metrics.py @@ -279,7 +279,8 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche params = {"metric_id": metric_id, "is_public": data.is_public, "name": data.name, "user_id": user_id, "project_id": project_id, "view_type": data.view_type, "metric_type": data.metric_type, "metric_of": data.metric_of, - "metric_value": data.metric_value, "metric_format": data.metric_format} + "metric_value": data.metric_value, "metric_format": data.metric_format, + "config": json.dumps(data.config.dict())} for i, s in enumerate(data.series): prefix = "u_" if s.index is None: @@ -329,7 +330,8 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche view_type= %(view_type)s, metric_type= %(metric_type)s, metric_of= %(metric_of)s, metric_value= %(metric_value)s, metric_format= %(metric_format)s, - edited_at = timezone('utc'::text, now()) + edited_at = timezone('utc'::text, now()), + default_config = %(config)s WHERE metric_id = %(metric_id)s AND project_id = %(project_id)s AND (user_id = %(user_id)s OR is_public) diff --git a/ee/api/chalicelib/core/dashboards.py b/ee/api/chalicelib/core/dashboards.py index d96356df1..25b1551d3 100644 --- a/ee/api/chalicelib/core/dashboards.py +++ b/ee/api/chalicelib/core/dashboards.py @@ -118,6 +118,8 @@ def get_dashboard(project_id, user_id, dashboard_id): for w in row["widgets"]: w["created_at"] = TimeUTC.datetime_to_timestamp(w["created_at"]) w["edited_at"] = TimeUTC.datetime_to_timestamp(w["edited_at"]) + w["config"]["col"] = w["default_config"]["col"] + w["config"]["row"] = w["default_config"]["row"] for s in w["series"]: s["created_at"] = TimeUTC.datetime_to_timestamp(s["created_at"]) return helper.dict_to_camel_case(row) From 910dd90a274f1bb2ca8724c3569bb446bfb33376 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 25 Nov 2022 20:16:21 +0100 Subject: [PATCH 19/22] feat(chalice): changed default metric's config row --- api/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/schemas.py b/api/schemas.py index 960b89e76..7e990bcb8 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -875,7 +875,7 @@ class TryCustomMetricsPayloadSchema(CustomMetricChartPayloadSchema): class CustomMetricsConfigSchema(BaseModel): col: Optional[int] = Field(...) - row: Optional[int] = Field(...) + row: Optional[int] = Field(default=2) position: Optional[int] = Field(default=0) From bf3916353cba55306ff490f5000e2545cf03bcad Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 25 Nov 2022 20:51:42 +0100 Subject: [PATCH 20/22] fix(ui) - activeSite id from path --- frontend/app/Router.js | 3 ++- frontend/app/duck/site.js | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/frontend/app/Router.js b/frontend/app/Router.js index 1e4f4b4b7..6a4aea446 100644 --- a/frontend/app/Router.js +++ b/frontend/app/Router.js @@ -126,8 +126,9 @@ class Router extends React.Component { } fetchInitialData = async () => { + const siteIdFromPath = parseInt(window.location.pathname.split("/")[1]) await this.props.fetchUserInfo() - await this.props.fetchSiteList() + await this.props.fetchSiteList(siteIdFromPath) const { mstore } = this.props; mstore.initClient(); }; diff --git a/frontend/app/duck/site.js b/frontend/app/duck/site.js index ad451f1fb..e1a4b9ea6 100644 --- a/frontend/app/duck/site.js +++ b/frontend/app/duck/site.js @@ -65,7 +65,9 @@ const reducer = (state = initialState, action = {}) => { case FETCH_LIST_SUCCESS: let siteId = state.get("siteId"); const siteExists = action.data.map(s => s.projectId).includes(siteId); - if (!siteId || !siteExists) { + if (action.siteIdFromPath) { + siteId = action.siteIdFromPath; + } else if (!siteId || !siteExists) { siteId = !!action.data.find(s => s.projectId === parseInt(storedSiteId)) ? storedSiteId : action.data[0].projectId; @@ -83,7 +85,7 @@ const reducer = (state = initialState, action = {}) => { .set('active', list.find(s => s.id === parseInt(siteId))); case SET_SITE_ID: localStorage.setItem(SITE_ID_STORAGE_KEY, action.siteId) - const site = state.get('list').find(s => s.id === action.siteId); + const site = state.get('list').find(s => parseInt(s.id) == action.siteId); return state.set('siteId', action.siteId).set('active', site); } return state; @@ -110,10 +112,11 @@ export function saveGDPR(siteId, gdpr) { }; } -export function fetchList() { +export function fetchList(siteId) { return { types: array(FETCH_LIST), call: client => client.get('/projects'), + siteIdFromPath: siteId }; } From ad11ba4d066a593374fa8cb0e3602145947b1121 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Fri, 25 Nov 2022 23:18:22 +0100 Subject: [PATCH 21/22] refactor(frontend):remove unnecessary hook abstraction --- .../components/SessionList/SessionList.tsx | 14 +++++++------ frontend/app/hooks/useTimeout.ts | 21 ------------------- 2 files changed, 8 insertions(+), 27 deletions(-) delete mode 100644 frontend/app/hooks/useTimeout.ts diff --git a/frontend/app/components/shared/SessionListContainer/components/SessionList/SessionList.tsx b/frontend/app/components/shared/SessionListContainer/components/SessionList/SessionList.tsx index 97c02d4ca..5f279c394 100644 --- a/frontend/app/components/shared/SessionListContainer/components/SessionList/SessionList.tsx +++ b/frontend/app/components/shared/SessionListContainer/components/SessionList/SessionList.tsx @@ -11,7 +11,6 @@ import { setScrollPosition, checkForLatestSessions, } from 'Duck/search'; -import useTimeout from 'App/hooks/useTimeout'; import { numberWithCommas } from 'App/utils'; import { fetchListActive as fetchMetadata } from 'Duck/customField'; @@ -82,11 +81,14 @@ function SessionList(props: Props) { }; }, [isBookmark, isVault, activeTab]); - useTimeout(() => { - if (!document.hidden) { - props.checkForLatestSessions(); - } - }, AUTOREFRESH_INTERVAL); + useEffect(() => { + const id = setInterval(() => { + if (!document.hidden) { + props.checkForLatestSessions() + } + }, AUTOREFRESH_INTERVAL) + return () => clearInterval(id) + }, []) useEffect(() => { // handle scroll position diff --git a/frontend/app/hooks/useTimeout.ts b/frontend/app/hooks/useTimeout.ts deleted file mode 100644 index ea6dd2ba3..000000000 --- a/frontend/app/hooks/useTimeout.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { useRef, useEffect } from 'react'; - -const useTimeout = (callback: () => void, delay: number) => { - const savedCallback = useRef<() => void>(); - - useEffect(() => { - savedCallback.current = callback; - }, [callback]); - - useEffect(() => { - function tick() { - savedCallback.current && savedCallback.current(); - } - if (delay !== null) { - const id = setInterval(tick, delay); - return () => clearInterval(id); - } - }, [delay]); -}; - -export default useTimeout; From 9d36aefc7c7d2d09c43fefe94c09a0734dd2754a Mon Sep 17 00:00:00 2001 From: sylenien Date: Mon, 28 Nov 2022 12:56:45 +0100 Subject: [PATCH 22/22] fix(ui): fix filterlist method --- frontend/app/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/utils.ts b/frontend/app/utils.ts index e3cd50134..34e6d896e 100644 --- a/frontend/app/utils.ts +++ b/frontend/app/utils.ts @@ -72,7 +72,7 @@ export const filterList = >( if (searchQuery === '') return list; const filterRE = getRE(searchQuery, 'i'); let _list = list.filter((listItem: T) => { - return testKeys.some((key) => filterRE.test(listItem[key]) || searchCb?.(listItem, filterRE)); + return testKeys.some((key) => filterRE.test(listItem[key])) || searchCb?.(listItem, filterRE); }); return _list; }