diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index be37a7abe..6b11383a2 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -37,9 +37,9 @@ func main() { messages.MsgUserID, messages.MsgUserAnonymousID, messages.MsgIntegrationEvent, messages.MsgPerformanceTrackAggr, messages.MsgJSException, messages.MsgResourceTiming, messages.MsgCustomEvent, messages.MsgCustomIssue, messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL, messages.MsgStateAction, - messages.MsgSetInputTarget, messages.MsgSetInputValue, messages.MsgCreateDocument, messages.MsgMouseClick, + messages.MsgCreateDocument, messages.MsgMouseClick, messages.MsgSetPageLocation, messages.MsgPageLoadTiming, messages.MsgPageRenderTiming, - messages.MsgInputEvent, messages.MsgPageEvent, messages.MsgMouseThrashing, messages.MsgInputChange, + messages.MsgPageEvent, messages.MsgMouseThrashing, messages.MsgInputChange, messages.MsgUnbindNodes} // Init consumer diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index 2b5f7ca8c..7d806c10e 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -26,7 +26,6 @@ func main() { // HandlersFabric returns the list of message handlers we want to be applied to each incoming message. handlersFabric := func() []handlers.MessageProcessor { return []handlers.MessageProcessor{ - custom.NewInputEventBuilder(), custom.NewPageEventBuilder(), web.NewDeadClickDetector(), &web.ClickRageDetector{}, diff --git a/backend/internal/config/storage/config.go b/backend/internal/config/storage/config.go index ca4ff8028..a11be1c06 100644 --- a/backend/internal/config/storage/config.go +++ b/backend/internal/config/storage/config.go @@ -23,6 +23,7 @@ type Config struct { MaxFileSize int64 `env:"MAX_FILE_SIZE,default=524288000"` UseSort bool `env:"USE_SESSION_SORT,default=true"` UseProfiler bool `env:"PROFILER_ENABLED,default=false"` + UseBrotli bool `env:"USE_BROTLI,default=false"` } func New() *Config { diff --git a/backend/internal/db/datasaver/saver.go b/backend/internal/db/datasaver/saver.go index 1a017fa6f..6a542dc58 100644 --- a/backend/internal/db/datasaver/saver.go +++ b/backend/internal/db/datasaver/saver.go @@ -65,8 +65,6 @@ func (s *saverImpl) handleMessage(msg Message) error { return s.pg.InsertWebCustomEvent(m) case *MouseClick: return s.pg.InsertWebClickEvent(m) - case *InputEvent: - return s.pg.InsertWebInputEvent(m) case *PageEvent: return s.pg.InsertWebPageEvent(m) case *NetworkRequest: diff --git a/backend/internal/storage/storage.go b/backend/internal/storage/storage.go index 3e86d403f..13bce3db2 100644 --- a/backend/internal/storage/storage.go +++ b/backend/internal/storage/storage.go @@ -198,7 +198,7 @@ func (s *Storage) prepareSession(path string, tp FileType, task *Task) error { func (s *Storage) packSession(task *Task, tp FileType) { // If encryption key is empty, pack session using better algorithm - if task.key == "" { + if task.key == "" && s.cfg.UseBrotli { s.packSessionBetter(task, tp) return } diff --git a/backend/pkg/db/cache/messages-web.go b/backend/pkg/db/cache/messages-web.go index 58c703318..ae4e3db8a 100644 --- a/backend/pkg/db/cache/messages-web.go +++ b/backend/pkg/db/cache/messages-web.go @@ -172,15 +172,6 @@ func (c *PGCache) InsertWebClickEvent(e *MouseClick) error { return c.Conn.InsertWebClickEvent(sessionID, session.ProjectID, e) } -func (c *PGCache) InsertWebInputEvent(e *InputEvent) error { - sessionID := e.SessionID() - session, err := c.Cache.GetSession(sessionID) - if err != nil { - return err - } - return c.Conn.InsertWebInputEvent(sessionID, session.ProjectID, e) -} - func (c *PGCache) InsertWebInputDuration(e *InputChange) error { sessionID := e.SessionID() session, err := c.Cache.GetSession(sessionID) diff --git a/backend/pkg/db/postgres/bulks.go b/backend/pkg/db/postgres/bulks.go index e58299acd..61526021a 100644 --- a/backend/pkg/db/postgres/bulks.go +++ b/backend/pkg/db/postgres/bulks.go @@ -18,7 +18,6 @@ type BulkSet struct { requests Bulk customEvents Bulk webPageEvents Bulk - webInputEvents Bulk webInputDurations Bulk webGraphQL Bulk webErrors Bulk @@ -56,8 +55,6 @@ func (conn *BulkSet) Get(name string) Bulk { return conn.customEvents case "webPageEvents": return conn.webPageEvents - case "webInputEvents": - return conn.webInputEvents case "webInputDurations": return conn.webInputDurations case "webGraphQL": @@ -121,14 +118,6 @@ func (conn *BulkSet) initBulks() { if err != nil { log.Fatalf("can't create webPageEvents bulk: %s", err) } - conn.webInputEvents, err = NewBulk(conn.c, - "events.inputs", - "(session_id, message_id, timestamp, label)", - "($%d, $%d, $%d, NULLIF(LEFT($%d, 2000),''))", - 4, 200) - if err != nil { - log.Fatalf("can't create webPageEvents bulk: %s", err) - } conn.webInputDurations, err = NewBulk(conn.c, "events.inputs", "(session_id, message_id, timestamp, label, hesitation, duration)", @@ -219,7 +208,6 @@ func (conn *BulkSet) Send() { newTask.bulks = append(newTask.bulks, conn.requests) newTask.bulks = append(newTask.bulks, conn.customEvents) newTask.bulks = append(newTask.bulks, conn.webPageEvents) - newTask.bulks = append(newTask.bulks, conn.webInputEvents) newTask.bulks = append(newTask.bulks, conn.webInputDurations) newTask.bulks = append(newTask.bulks, conn.webGraphQL) newTask.bulks = append(newTask.bulks, conn.webErrors) diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index 3f93ae35d..df77888da 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -76,18 +76,6 @@ func (conn *Conn) InsertWebClickEvent(sessionID uint64, projectID uint32, e *Mou return nil } -func (conn *Conn) InsertWebInputEvent(sessionID uint64, projectID uint32, e *InputEvent) error { - if e.Label == "" { - return nil - } - if err := conn.bulks.Get("webInputEvents").Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.Label); err != nil { - log.Printf("insert web input event err: %s", err) - } - conn.updateSessionEvents(sessionID, 1, 0) - conn.insertAutocompleteValue(sessionID, projectID, "INPUT", e.Label) - return nil -} - func (conn *Conn) InsertWebInputDuration(sessionID uint64, projectID uint32, e *InputChange) error { if e.Label == "" { return nil diff --git a/backend/pkg/handlers/custom/inputEventBuilder.go b/backend/pkg/handlers/custom/inputEventBuilder.go deleted file mode 100644 index d057db3e3..000000000 --- a/backend/pkg/handlers/custom/inputEventBuilder.go +++ /dev/null @@ -1,77 +0,0 @@ -package custom - -import ( - . "openreplay/backend/pkg/messages" -) - -const InputEventTimeout = 1 * 60 * 1000 - -type inputLabels map[uint64]string - -type inputEventBuilder struct { - inputEvent *InputEvent - inputLabels inputLabels - inputID uint64 -} - -func NewInputEventBuilder() *inputEventBuilder { - ieBuilder := &inputEventBuilder{} - ieBuilder.clearLabels() - return ieBuilder -} - -func (b *inputEventBuilder) clearLabels() { - b.inputLabels = make(inputLabels) -} - -func (b *inputEventBuilder) Handle(message Message, timestamp uint64) Message { - var inputEvent Message = nil - switch msg := message.(type) { - case *SetInputTarget: - if b.inputID != msg.ID { - inputEvent = b.Build() - b.inputID = msg.ID - } - b.inputLabels[msg.ID] = msg.Label - return inputEvent - case *SetInputValue: - if b.inputID != msg.ID { - inputEvent = b.Build() - b.inputID = msg.ID - } - if b.inputEvent == nil { - b.inputEvent = &InputEvent{ - MessageID: message.MsgID(), - Timestamp: timestamp, - Value: msg.Value, - ValueMasked: msg.Mask > 0, - } - } else { - b.inputEvent.Value = msg.Value - b.inputEvent.ValueMasked = msg.Mask > 0 - } - return inputEvent - case *CreateDocument: - inputEvent = b.Build() - b.clearLabels() - return inputEvent - case *MouseClick: - return b.Build() - } - - if b.inputEvent != nil && b.inputEvent.Timestamp+InputEventTimeout < timestamp { - return b.Build() - } - return nil -} - -func (b *inputEventBuilder) Build() Message { - if b.inputEvent == nil { - return nil - } - inputEvent := b.inputEvent - inputEvent.Label = b.inputLabels[b.inputID] // might be empty string - - b.inputEvent = nil - return inputEvent -} diff --git a/ee/backend/internal/db/datasaver/methods.go b/ee/backend/internal/db/datasaver/methods.go index 09653bde9..65bb68bb3 100644 --- a/ee/backend/internal/db/datasaver/methods.go +++ b/ee/backend/internal/db/datasaver/methods.go @@ -8,7 +8,6 @@ import ( "openreplay/backend/pkg/db/clickhouse" "openreplay/backend/pkg/db/types" "openreplay/backend/pkg/env" - . "openreplay/backend/pkg/messages" "openreplay/backend/pkg/queue" ) @@ -52,8 +51,6 @@ func (s *saverImpl) handleExtraMessage(msg Message) error { return s.ch.InsertWebPerformanceTrackAggr(session, m) case *MouseClick: return s.ch.InsertWebClickEvent(session, m) - case *InputEvent: - return s.ch.InsertWebInputEvent(session, m) // Unique for Web case *PageEvent: return s.ch.InsertWebPageEvent(session, m) diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index 12ed1fb3f..98f1a0ba7 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -3,7 +3,6 @@ package clickhouse import ( "errors" "fmt" - "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "log" "openreplay/backend/pkg/db/types" @@ -25,7 +24,6 @@ type Connector interface { InsertWebResourceEvent(session *types.Session, msg *messages.ResourceTiming) error InsertWebPageEvent(session *types.Session, msg *messages.PageEvent) error InsertWebClickEvent(session *types.Session, msg *messages.MouseClick) error - InsertWebInputEvent(session *types.Session, msg *messages.InputEvent) error InsertWebErrorEvent(session *types.Session, msg *types.ErrorEvent) error InsertWebPerformanceTrackAggr(session *types.Session, msg *messages.PerformanceTrackAggr) error InsertAutocomplete(session *types.Session, msgType, msgValue string) error @@ -373,26 +371,6 @@ func (c *connectorImpl) InsertWebClickEvent(session *types.Session, msg *message return nil } -func (c *connectorImpl) InsertWebInputEvent(session *types.Session, msg *messages.InputEvent) error { - if msg.Label == "" { - return nil - } - if err := c.batches["inputs"].Append( - session.SessionID, - uint16(session.ProjectID), - msg.MessageID, - datetime(msg.Timestamp), - msg.Label, - "INPUT", - nil, - nil, - ); err != nil { - c.checkError("inputs", err) - return fmt.Errorf("can't append to inputs batch: %s", err) - } - return nil -} - func (c *connectorImpl) InsertWebErrorEvent(session *types.Session, msg *types.ErrorEvent) error { keys, values := make([]string, 0, len(msg.Tags)), make([]*string, 0, len(msg.Tags)) for k, v := range msg.Tags {