diff --git a/backend/pkg/db/postgres/helpers.go b/backend/pkg/db/postgres/helpers.go index 2d4c96e28..f3b96134b 100644 --- a/backend/pkg/db/postgres/helpers.go +++ b/backend/pkg/db/postgres/helpers.go @@ -1,6 +1,8 @@ package postgres import ( + "math" + "openreplay/backend/pkg/messages" ) @@ -33,3 +35,8 @@ func calcResponseTime(pe *messages.PageEvent) uint64 { } return 0 } + +// TODO: review message indexing (it is better to have lower values in db for faster search (int4/int2)) +func truncSqIdx(messageID uint64) uint32 { + return uint32(messageID % math.MaxInt32) +} diff --git a/backend/pkg/db/postgres/messages-common.go b/backend/pkg/db/postgres/messages-common.go index e69d9508c..1742c18e6 100644 --- a/backend/pkg/db/postgres/messages-common.go +++ b/backend/pkg/db/postgres/messages-common.go @@ -102,14 +102,14 @@ func (conn *Conn) HandleSessionEnd(sessionID uint64) error { return conn.c.Exec(sqlRequest, sessionID) } -func (conn *Conn) InsertRequest(sessionID uint64, timestamp uint64, index uint64, url string, duration uint64, success bool) error { +func (conn *Conn) InsertRequest(sessionID uint64, timestamp uint64, index uint32, url string, duration uint64, success bool) error { if err := conn.requests.Append(sessionID, timestamp, index, url, duration, success); err != nil { return fmt.Errorf("insert request in bulk err: %s", err) } return nil } -func (conn *Conn) InsertCustomEvent(sessionID uint64, timestamp uint64, index uint64, name string, payload string) error { +func (conn *Conn) InsertCustomEvent(sessionID uint64, timestamp uint64, index uint32, name string, payload string) error { if err := conn.customEvents.Append(sessionID, timestamp, index, name, payload); err != nil { return fmt.Errorf("insert custom event in bulk err: %s", err) } @@ -184,7 +184,7 @@ func (conn *Conn) InsertIssueEvent(sessionID uint64, projectID uint32, e *messag $1, $2, $3, $4, CAST($5 AS jsonb) )`, sessionID, issueID, e.Timestamp, - e.MessageID, + truncSqIdx(e.MessageID), payload, ); err != nil { return err @@ -204,7 +204,7 @@ func (conn *Conn) InsertIssueEvent(sessionID uint64, projectID uint32, e *messag VALUES ($1, $2, $3, left($4, 2700), $5, 'error') `, - sessionID, e.MessageID, e.Timestamp, e.ContextString, e.Payload, + sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.ContextString, e.Payload, ); err != nil { return err } diff --git a/backend/pkg/db/postgres/messages-ios.go b/backend/pkg/db/postgres/messages-ios.go index d7b2f58f3..027cfc968 100644 --- a/backend/pkg/db/postgres/messages-ios.go +++ b/backend/pkg/db/postgres/messages-ios.go @@ -7,7 +7,7 @@ import ( ) func (conn *Conn) InsertIOSCustomEvent(sessionID uint64, e *messages.IOSCustomEvent) error { - err := conn.InsertCustomEvent(sessionID, e.Timestamp, e.Index, e.Name, e.Payload) + err := conn.InsertCustomEvent(sessionID, e.Timestamp, truncSqIdx(e.Index), e.Name, e.Payload) if err == nil { conn.insertAutocompleteValue(sessionID, 0, "CUSTOM_IOS", e.Name) } @@ -31,7 +31,7 @@ func (conn *Conn) InsertIOSUserAnonymousID(sessionID uint64, userAnonymousID *me } func (conn *Conn) InsertIOSNetworkCall(sessionID uint64, e *messages.IOSNetworkCall) error { - err := conn.InsertRequest(sessionID, e.Timestamp, e.Index, e.URL, e.Duration, e.Success) + err := conn.InsertRequest(sessionID, e.Timestamp, truncSqIdx(e.Index), e.URL, e.Duration, e.Success) if err == nil { conn.insertAutocompleteValue(sessionID, 0, "REQUEST_IOS", url.DiscardURLQuery(e.URL)) } diff --git a/backend/pkg/db/postgres/messages-web-stats.go b/backend/pkg/db/postgres/messages-web-stats.go index ba7db4da4..719fe0d61 100644 --- a/backend/pkg/db/postgres/messages-web-stats.go +++ b/backend/pkg/db/postgres/messages-web-stats.go @@ -60,7 +60,7 @@ func (conn *Conn) InsertWebStatsResourceEvent(sessionID uint64, e *ResourceEvent urlQuery := url.DiscardURLQuery(e.URL) urlMethod := url.EnsureMethod(e.Method) conn.batchQueue(sessionID, sqlRequest, - sessionID, e.Timestamp, e.MessageID, + sessionID, e.Timestamp, truncSqIdx(e.MessageID), e.Type, e.URL, host, urlQuery, e.Success, e.Status, diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index f57c3a8de..10cfad409 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -10,7 +10,7 @@ import ( func (conn *Conn) InsertWebCustomEvent(sessionID uint64, projectID uint32, e *CustomEvent) error { err := conn.InsertCustomEvent(sessionID, e.Timestamp, - e.MessageID, + truncSqIdx(e.MessageID), e.Name, e.Payload) if err == nil { conn.insertAutocompleteValue(sessionID, projectID, "CUSTOM", e.Name) @@ -40,7 +40,7 @@ func (conn *Conn) InsertWebPageEvent(sessionID uint64, projectID uint32, e *Page return err } // base_path is deprecated - if err = conn.webPageEvents.Append(sessionID, e.MessageID, e.Timestamp, e.Referrer, url.DiscardURLQuery(e.Referrer), + if err = conn.webPageEvents.Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.Referrer, url.DiscardURLQuery(e.Referrer), host, path, query, e.DomContentLoadedEventEnd, e.LoadEventEnd, e.ResponseEnd, e.FirstPaint, e.FirstContentfulPaint, e.SpeedIndex, e.VisuallyComplete, e.TimeToInteractive, calcResponseTime(e), calcDomBuildingTime(e)); err != nil { log.Printf("insert web page event in bulk err: %s", err) @@ -63,7 +63,7 @@ func (conn *Conn) InsertWebClickEvent(sessionID uint64, projectID uint32, e *Cli WHERE session_id = $1 AND timestamp <= $3 ORDER BY timestamp DESC LIMIT 1 ) ` - conn.batchQueue(sessionID, sqlRequest, sessionID, e.MessageID, e.Timestamp, e.Label, e.Selector) + conn.batchQueue(sessionID, sqlRequest, sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.Label, e.Selector) // Accumulate session updates and exec inside batch with another sql commands conn.updateSessionEvents(sessionID, 1, 0) // Add new value set to autocomplete bulk @@ -79,7 +79,7 @@ func (conn *Conn) InsertWebInputEvent(sessionID uint64, projectID uint32, e *Inp if e.ValueMasked { value = nil } - if err := conn.webInputEvents.Append(sessionID, e.MessageID, e.Timestamp, value, e.Label); err != nil { + if err := conn.webInputEvents.Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, value, e.Label); err != nil { log.Printf("insert web input event err: %s", err) } conn.updateSessionEvents(sessionID, 1, 0) @@ -117,7 +117,7 @@ func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *typ VALUES ($1, $2, $3, $4) `, - sessionID, e.MessageID, e.Timestamp, errorID, + sessionID, truncSqIdx(e.MessageID), e.Timestamp, errorID, ); err != nil { return err } @@ -138,7 +138,7 @@ func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *typ $1, $2, $3, $4, $5 ) ON CONFLICT DO NOTHING` for key, value := range e.Tags { - conn.batchQueue(sessionID, sqlRequest, sessionID, e.MessageID, errorID, key, value) + conn.batchQueue(sessionID, sqlRequest, sessionID, truncSqIdx(e.MessageID), errorID, key, value) } return @@ -169,7 +169,7 @@ func (conn *Conn) InsertWebFetchEvent(sessionID uint64, projectID uint32, savePa $12, $13 ) ON CONFLICT DO NOTHING` conn.batchQueue(sessionID, sqlRequest, - sessionID, e.Timestamp, e.MessageID, + sessionID, e.Timestamp, truncSqIdx(e.MessageID), e.URL, host, path, query, request, response, e.Status, url.EnsureMethod(e.Method), e.Duration, e.Status < 400, @@ -187,7 +187,7 @@ func (conn *Conn) InsertWebGraphQLEvent(sessionID uint64, projectID uint32, save request = &e.Variables response = &e.Response } - if err := conn.webGraphQLEvents.Append(sessionID, e.Timestamp, e.MessageID, e.OperationName, request, response); err != nil { + if err := conn.webGraphQLEvents.Append(sessionID, e.Timestamp, truncSqIdx(e.MessageID), e.OperationName, request, response); err != nil { log.Printf("insert web graphQL event err: %s", err) } conn.insertAutocompleteValue(sessionID, projectID, "GRAPHQL", e.OperationName)