diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index ac7b34730..7ce4c059b 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -56,7 +56,7 @@ func main() { messages.MsgUserID, messages.MsgUserAnonymousID, messages.MsgIntegrationEvent, messages.MsgPerformanceTrackAggr, messages.MsgJSException, messages.MsgResourceTiming, messages.MsgCustomEvent, messages.MsgCustomIssue, messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL, messages.MsgStateAction, - messages.MsgSetInputTarget, messages.MsgSetInputValue, messages.MsgCreateDocument, messages.MsgMouseClick, + messages.MsgCreateDocument, messages.MsgMouseClick, messages.MsgSetPageLocation, messages.MsgPageLoadTiming, messages.MsgPageRenderTiming, messages.MsgInputEvent, messages.MsgPageEvent, messages.MsgMouseThrashing, messages.MsgInputChange, messages.MsgUnbindNodes} diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index c40a2fe2a..f5ff67fe6 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -26,7 +26,6 @@ func main() { // HandlersFabric returns the list of message handlers we want to be applied to each incoming message. handlersFabric := func() []handlers.MessageProcessor { return []handlers.MessageProcessor{ - custom.NewInputEventBuilder(), custom.NewPageEventBuilder(), web.NewDeadClickDetector(), &web.ClickRageDetector{}, diff --git a/backend/internal/config/storage/config.go b/backend/internal/config/storage/config.go index facd01153..b8e594a3b 100644 --- a/backend/internal/config/storage/config.go +++ b/backend/internal/config/storage/config.go @@ -23,6 +23,7 @@ type Config struct { MaxFileSize int64 `env:"MAX_FILE_SIZE,default=524288000"` UseSort bool `env:"USE_SESSION_SORT,default=true"` UseProfiler bool `env:"PROFILER_ENABLED,default=false"` + UseBrotli bool `env:"USE_BROTLI,default=false"` } func New() *Config { diff --git a/backend/internal/db/datasaver/saver.go b/backend/internal/db/datasaver/saver.go index 12bca3b83..31482611e 100644 --- a/backend/internal/db/datasaver/saver.go +++ b/backend/internal/db/datasaver/saver.go @@ -1,7 +1,6 @@ package datasaver import ( - "errors" "log" "openreplay/backend/internal/config/db" @@ -103,14 +102,6 @@ func (s *saverImpl) handleMessage(msg Message) error { return err } return s.sessions.UpdateEventsStats(session.SessionID, 1, 0) - case *InputEvent: - if err = s.pg.InsertWebInputEvent(session, m); err != nil { - if errors.Is(err, postgres.EmptyLabel) { - return nil - } - return err - } - return s.sessions.UpdateEventsStats(session.SessionID, 1, 0) case *PageEvent: if err = s.pg.InsertWebPageEvent(session, m); err != nil { return err diff --git a/backend/internal/storage/storage.go b/backend/internal/storage/storage.go index 9504a2f73..b75d12ee0 100644 --- a/backend/internal/storage/storage.go +++ b/backend/internal/storage/storage.go @@ -197,7 +197,7 @@ func (s *Storage) prepareSession(path string, tp FileType, task *Task) error { func (s *Storage) packSession(task *Task, tp FileType) { // If encryption key is empty, pack session using better algorithm - if task.key == "" { + if task.key == "" && s.cfg.UseBrotli { s.packSessionBetter(task, tp) return } diff --git a/backend/pkg/db/postgres/bulks.go b/backend/pkg/db/postgres/bulks.go index cd896cf58..3e40e0c9e 100644 --- a/backend/pkg/db/postgres/bulks.go +++ b/backend/pkg/db/postgres/bulks.go @@ -19,7 +19,6 @@ type BulkSet struct { requests Bulk customEvents Bulk webPageEvents Bulk - webInputEvents Bulk webInputDurations Bulk webGraphQL Bulk webErrors Bulk @@ -57,8 +56,6 @@ func (conn *BulkSet) Get(name string) Bulk { return conn.customEvents case "webPageEvents": return conn.webPageEvents - case "webInputEvents": - return conn.webInputEvents case "webInputDurations": return conn.webInputDurations case "webGraphQL": @@ -122,14 +119,6 @@ func (conn *BulkSet) initBulks() { if err != nil { log.Fatalf("can't create webPageEvents bulk: %s", err) } - conn.webInputEvents, err = NewBulk(conn.c, - "events.inputs", - "(session_id, message_id, timestamp, label)", - "($%d, $%d, $%d, NULLIF(LEFT($%d, 2000),''))", - 4, 200) - if err != nil { - log.Fatalf("can't create webPageEvents bulk: %s", err) - } conn.webInputDurations, err = NewBulk(conn.c, "events.inputs", "(session_id, message_id, timestamp, label, hesitation, duration)", @@ -220,7 +209,6 @@ func (conn *BulkSet) Send() { newTask.bulks = append(newTask.bulks, conn.requests) newTask.bulks = append(newTask.bulks, conn.customEvents) newTask.bulks = append(newTask.bulks, conn.webPageEvents) - newTask.bulks = append(newTask.bulks, conn.webInputEvents) newTask.bulks = append(newTask.bulks, conn.webInputDurations) newTask.bulks = append(newTask.bulks, conn.webGraphQL) newTask.bulks = append(newTask.bulks, conn.webErrors) diff --git a/backend/pkg/db/postgres/events.go b/backend/pkg/db/postgres/events.go index 0cb073996..892f90298 100644 --- a/backend/pkg/db/postgres/events.go +++ b/backend/pkg/db/postgres/events.go @@ -1,7 +1,6 @@ package postgres import ( - "errors" "fmt" "log" "strings" @@ -14,8 +13,6 @@ import ( "openreplay/backend/pkg/url" ) -var EmptyLabel = errors.New("empty label") - func getAutocompleteType(baseType string, platform string) string { if platform == "web" { return baseType @@ -141,17 +138,6 @@ func (conn *Conn) InsertWebClickEvent(sess *sessions.Session, e *messages.MouseC return nil } -func (conn *Conn) InsertWebInputEvent(sess *sessions.Session, e *messages.InputEvent) error { - if e.Label == "" { - return EmptyLabel - } - if err := conn.bulks.Get("webInputEvents").Append(sess.SessionID, truncSqIdx(e.MessageID), e.Timestamp, e.Label); err != nil { - log.Printf("insert web input event err: %s", err) - } - conn.InsertAutocompleteValue(sess.SessionID, sess.ProjectID, "INPUT", e.Label) - return nil -} - func (conn *Conn) InsertInputChangeEvent(sess *sessions.Session, e *messages.InputChange) error { if e.Label == "" { return nil diff --git a/backend/pkg/handlers/custom/inputEventBuilder.go b/backend/pkg/handlers/custom/inputEventBuilder.go deleted file mode 100644 index d057db3e3..000000000 --- a/backend/pkg/handlers/custom/inputEventBuilder.go +++ /dev/null @@ -1,77 +0,0 @@ -package custom - -import ( - . "openreplay/backend/pkg/messages" -) - -const InputEventTimeout = 1 * 60 * 1000 - -type inputLabels map[uint64]string - -type inputEventBuilder struct { - inputEvent *InputEvent - inputLabels inputLabels - inputID uint64 -} - -func NewInputEventBuilder() *inputEventBuilder { - ieBuilder := &inputEventBuilder{} - ieBuilder.clearLabels() - return ieBuilder -} - -func (b *inputEventBuilder) clearLabels() { - b.inputLabels = make(inputLabels) -} - -func (b *inputEventBuilder) Handle(message Message, timestamp uint64) Message { - var inputEvent Message = nil - switch msg := message.(type) { - case *SetInputTarget: - if b.inputID != msg.ID { - inputEvent = b.Build() - b.inputID = msg.ID - } - b.inputLabels[msg.ID] = msg.Label - return inputEvent - case *SetInputValue: - if b.inputID != msg.ID { - inputEvent = b.Build() - b.inputID = msg.ID - } - if b.inputEvent == nil { - b.inputEvent = &InputEvent{ - MessageID: message.MsgID(), - Timestamp: timestamp, - Value: msg.Value, - ValueMasked: msg.Mask > 0, - } - } else { - b.inputEvent.Value = msg.Value - b.inputEvent.ValueMasked = msg.Mask > 0 - } - return inputEvent - case *CreateDocument: - inputEvent = b.Build() - b.clearLabels() - return inputEvent - case *MouseClick: - return b.Build() - } - - if b.inputEvent != nil && b.inputEvent.Timestamp+InputEventTimeout < timestamp { - return b.Build() - } - return nil -} - -func (b *inputEventBuilder) Build() Message { - if b.inputEvent == nil { - return nil - } - inputEvent := b.inputEvent - inputEvent.Label = b.inputLabels[b.inputID] // might be empty string - - b.inputEvent = nil - return inputEvent -} diff --git a/ee/backend/internal/db/datasaver/methods.go b/ee/backend/internal/db/datasaver/methods.go index 89fa49d71..29f2e1c81 100644 --- a/ee/backend/internal/db/datasaver/methods.go +++ b/ee/backend/internal/db/datasaver/methods.go @@ -50,8 +50,6 @@ func (s *saverImpl) handleExtraMessage(msg messages.Message) error { return s.ch.InsertWebPerformanceTrackAggr(session, m) case *messages.MouseClick: return s.ch.InsertWebClickEvent(session, m) - case *messages.InputEvent: - return s.ch.InsertWebInputEvent(session, m) // Unique for Web case *messages.PageEvent: return s.ch.InsertWebPageEvent(session, m) diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index 858fa930e..c72253fed 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -3,7 +3,6 @@ package clickhouse import ( "errors" "fmt" - "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "log" "openreplay/backend/pkg/db/types" @@ -26,7 +25,6 @@ type Connector interface { InsertWebResourceEvent(session *sessions.Session, msg *messages.ResourceTiming) error InsertWebPageEvent(session *sessions.Session, msg *messages.PageEvent) error InsertWebClickEvent(session *sessions.Session, msg *messages.MouseClick) error - InsertWebInputEvent(session *sessions.Session, msg *messages.InputEvent) error InsertWebErrorEvent(session *sessions.Session, msg *types.ErrorEvent) error InsertWebPerformanceTrackAggr(session *sessions.Session, msg *messages.PerformanceTrackAggr) error InsertAutocomplete(session *sessions.Session, msgType, msgValue string) error @@ -377,26 +375,6 @@ func (c *connectorImpl) InsertWebClickEvent(session *sessions.Session, msg *mess return nil } -func (c *connectorImpl) InsertWebInputEvent(session *sessions.Session, msg *messages.InputEvent) error { - if msg.Label == "" { - return nil - } - if err := c.batches["inputs"].Append( - session.SessionID, - uint16(session.ProjectID), - msg.MessageID, - datetime(msg.Timestamp), - msg.Label, - "INPUT", - nil, - nil, - ); err != nil { - c.checkError("inputs", err) - return fmt.Errorf("can't append to inputs batch: %s", err) - } - return nil -} - func (c *connectorImpl) InsertWebErrorEvent(session *sessions.Session, msg *types.ErrorEvent) error { keys, values := make([]string, 0, len(msg.Tags)), make([]*string, 0, len(msg.Tags)) for k, v := range msg.Tags {