diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index a062df95a..ebba00168 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -1,12 +1,9 @@ package postgres import ( - "encoding/hex" - "hash/fnv" "log" - "strconv" - "openreplay/backend/pkg/db/types" + "openreplay/backend/pkg/hashid" . "openreplay/backend/pkg/messages" "openreplay/backend/pkg/url" ) @@ -171,7 +168,7 @@ func (conn *Conn) InsertMouseThrashing(sessionID uint64, projectID uint32, e *Mo // Debug log log.Printf("new MouseThrashing event: %v", e) // - issueID := mouseThrashingID(projectID, sessionID, e.Timestamp) + issueID := hashid.MouseThrashingID(projectID, sessionID, e.Timestamp) if err := conn.bulks.Get("webIssues").Append(projectID, issueID, "mouse_thrashing", e.Url); err != nil { log.Printf("insert web issue err: %s", err) } @@ -181,11 +178,3 @@ func (conn *Conn) InsertMouseThrashing(sessionID uint64, projectID uint32, e *Mo conn.updateSessionIssues(sessionID, 0, 50) return nil } - -func mouseThrashingID(projectID uint32, sessID, ts uint64) string { - hash := fnv.New128a() - hash.Write([]byte("mouse_trashing")) - hash.Write([]byte(strconv.FormatUint(sessID, 10))) - hash.Write([]byte(strconv.FormatUint(ts, 10))) - return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) -} diff --git a/backend/pkg/hashid/hashid.go b/backend/pkg/hashid/hashid.go index 25ce11369..5bcb23578 100644 --- a/backend/pkg/hashid/hashid.go +++ b/backend/pkg/hashid/hashid.go @@ -23,3 +23,11 @@ func IOSCrashID(projectID uint32, crash *messages.IOSCrash) string { hash.Write([]byte(crash.Stacktrace)) return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) } + +func MouseThrashingID(projectID uint32, sessID, ts uint64) string { + hash := fnv.New128a() + hash.Write([]byte("mouse_trashing")) + hash.Write([]byte(strconv.FormatUint(sessID, 10))) + hash.Write([]byte(strconv.FormatUint(ts, 10))) + return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) +} diff --git a/ee/backend/internal/db/datasaver/methods.go b/ee/backend/internal/db/datasaver/methods.go index 277fd8906..ac0a8b88d 100644 --- a/ee/backend/internal/db/datasaver/methods.go +++ b/ee/backend/internal/db/datasaver/methods.go @@ -78,6 +78,10 @@ func (s *saverImpl) handleExtraMessage(msg Message) error { } case *GraphQL: return s.ch.InsertGraphQL(session, m) + case *InputChange: + return s.ch.InsertWebInputDuration(session, m) + case *MouseThrashing: + return s.ch.InsertMouseThrashing(session, m) } return nil } diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index 489411550..ad08bd6e4 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -32,6 +32,8 @@ type Connector interface { InsertCustom(session *types.Session, msg *messages.CustomEvent) error InsertGraphQL(session *types.Session, msg *messages.GraphQL) error InsertIssue(session *types.Session, msg *messages.IssueEvent) error + InsertWebInputDuration(session *types.Session, msg *messages.InputChange) error + InsertMouseThrashing(session *types.Session, msg *messages.MouseThrashing) error } type task struct { @@ -97,7 +99,7 @@ var batches = map[string]string{ "autocompletes": "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, ?)", "pages": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint_time, speed_index, visually_complete, time_to_interactive, event_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "clicks": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, hesitation_time, event_type) VALUES (?, ?, ?, ?, ?, ?, ?)", - "inputs": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, event_type) VALUES (?, ?, ?, ?, ?, ?)", + "inputs": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, event_type, duration, hesitation_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "errors": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, source, name, message, error_id, event_type, error_tags_keys, error_tags_values) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "performance": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "requests": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_body, response_body, status, method, duration, success, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?)", @@ -164,11 +166,59 @@ func (c *connectorImpl) checkError(name string, err error) { } } +func (c *connectorImpl) InsertWebInputDuration(session *types.Session, msg *messages.InputChange) error { + if msg.Label == "" { + return nil + } + if err := c.batches["inputs"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.MessageID, + datetime(msg.Timestamp), + msg.Label, + "INPUT", + nullableUint16(uint16(msg.Duration)), + nullableUint32(uint32(msg.HesitationTime)), + ); err != nil { + c.checkError("inputs", err) + return fmt.Errorf("can't append to inputs batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertMouseThrashing(session *types.Session, msg *messages.MouseThrashing) error { + issueID := hashid.MouseThrashingID(session.projectID, session.sessionID, msg.Timestamp) + // Insert issue event to batches + if err := c.batches["issuesEvents"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.MsgID(), + datetime(msg.Timestamp), + issueID, + "mouse_thrashing", + "ISSUE", + msg.Url, + ); err != nil { + c.checkError("issuesEvents", err) + return fmt.Errorf("can't append to issuesEvents batch: %s", err) + } + if err := c.batches["issues"].Append( + uint16(session.ProjectID), + issueID, + "mouse_thrashing", + msg.Url, + ); err != nil { + c.checkError("issues", err) + return fmt.Errorf("can't append to issues batch: %s", err) + } + return nil +} + func (c *connectorImpl) InsertIssue(session *types.Session, msg *messages.IssueEvent) error { issueID := hashid.IssueID(session.ProjectID, msg) // Check issue type before insert to avoid panic from clickhouse lib switch msg.Type { - case "click_rage", "dead_click", "excessive_scrolling", "bad_request", "missing_resource", "memory", "cpu", "slow_resource", "slow_page_load", "crash", "ml_cpu", "ml_memory", "ml_dead_click", "ml_click_rage", "ml_mouse_thrashing", "ml_excessive_scrolling", "ml_slow_resources", "custom", "js_exception": + case "click_rage", "dead_click", "excessive_scrolling", "bad_request", "missing_resource", "memory", "cpu", "slow_resource", "slow_page_load", "crash", "ml_cpu", "ml_memory", "ml_dead_click", "ml_click_rage", "ml_mouse_thrashing", "ml_excessive_scrolling", "ml_slow_resources", "custom", "js_exception", "mouse_thrashing": default: return fmt.Errorf("unknown issueType: %s", msg.Type) } @@ -323,6 +373,8 @@ func (c *connectorImpl) InsertWebInputEvent(session *types.Session, msg *message datetime(msg.Timestamp), msg.Label, "INPUT", + nil, + nil, ); err != nil { c.checkError("inputs", err) return fmt.Errorf("can't append to inputs batch: %s", err)