jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(backend/db): truncate messageIndex to Int32 before insertion

Browse source
This commit is contained in:
Alex Kaminskii 2022-10-31 11:12:27 +01:00
parent 2ec73b47c5
commit e1cd349cb9
5 changed files with 22 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
backend/pkg/db/postgres/helpers.go
View file

@ -1,6 +1,8 @@
package postgres package postgres
import ( import (
"math"
"openreplay/backend/pkg/messages" "openreplay/backend/pkg/messages"
) )
@ -33,3 +35,8 @@ func calcResponseTime(pe *messages.PageEvent) uint64 {
} }
return 0 return 0
} }
// TODO: review message indexing (it is better to have lower values in db for faster search (int4/int2))
func truncSqIdx(messageID uint64) uint32 {
return uint32(messageID % math.MaxInt32)
}

8
backend/pkg/db/postgres/messages-common.go
View file

@ -102,14 +102,14 @@ func (conn *Conn) HandleSessionEnd(sessionID uint64) error {
return conn.c.Exec(sqlRequest, sessionID) return conn.c.Exec(sqlRequest, sessionID)
} }
func (conn *Conn) InsertRequest(sessionID uint64, timestamp uint64, index uint64, url string, duration uint64, success bool) error { func (conn *Conn) InsertRequest(sessionID uint64, timestamp uint64, index uint32, url string, duration uint64, success bool) error {
if err := conn.requests.Append(sessionID, timestamp, index, url, duration, success); err != nil { if err := conn.requests.Append(sessionID, timestamp, index, url, duration, success); err != nil {
return fmt.Errorf("insert request in bulk err: %s", err) return fmt.Errorf("insert request in bulk err: %s", err)
} }
return nil return nil
} }
func (conn *Conn) InsertCustomEvent(sessionID uint64, timestamp uint64, index uint64, name string, payload string) error { func (conn *Conn) InsertCustomEvent(sessionID uint64, timestamp uint64, index uint32, name string, payload string) error {
if err := conn.customEvents.Append(sessionID, timestamp, index, name, payload); err != nil { if err := conn.customEvents.Append(sessionID, timestamp, index, name, payload); err != nil {
return fmt.Errorf("insert custom event in bulk err: %s", err) return fmt.Errorf("insert custom event in bulk err: %s", err)
} }
@ -184,7 +184,7 @@ func (conn *Conn) InsertIssueEvent(sessionID uint64, projectID uint32, e *messag
$1, $2, $3, $4, CAST($5 AS jsonb) $1, $2, $3, $4, CAST($5 AS jsonb)
)`, )`,
sessionID, issueID, e.Timestamp, sessionID, issueID, e.Timestamp,
e.MessageID, truncSqIdx(e.MessageID),
payload, payload,
); err != nil { ); err != nil {
return err return err
@ -204,7 +204,7 @@ func (conn *Conn) InsertIssueEvent(sessionID uint64, projectID uint32, e *messag
VALUES VALUES
($1, $2, $3, left($4, 2700), $5, 'error') ($1, $2, $3, left($4, 2700), $5, 'error')
`, `,
sessionID, e.MessageID, e.Timestamp, e.ContextString, e.Payload, sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.ContextString, e.Payload,
); err != nil { ); err != nil {
return err return err
} }

4
backend/pkg/db/postgres/messages-ios.go
View file

@ -7,7 +7,7 @@ import (
) )
func (conn *Conn) InsertIOSCustomEvent(sessionID uint64, e *messages.IOSCustomEvent) error { func (conn *Conn) InsertIOSCustomEvent(sessionID uint64, e *messages.IOSCustomEvent) error {
err := conn.InsertCustomEvent(sessionID, e.Timestamp, e.Index, e.Name, e.Payload) err := conn.InsertCustomEvent(sessionID, e.Timestamp, truncSqIdx(e.Index), e.Name, e.Payload)
if err == nil { if err == nil {
conn.insertAutocompleteValue(sessionID, 0, "CUSTOM_IOS", e.Name) conn.insertAutocompleteValue(sessionID, 0, "CUSTOM_IOS", e.Name)
} }
@ -31,7 +31,7 @@ func (conn *Conn) InsertIOSUserAnonymousID(sessionID uint64, userAnonymousID *me
} }
func (conn *Conn) InsertIOSNetworkCall(sessionID uint64, e *messages.IOSNetworkCall) error { func (conn *Conn) InsertIOSNetworkCall(sessionID uint64, e *messages.IOSNetworkCall) error {
err := conn.InsertRequest(sessionID, e.Timestamp, e.Index, e.URL, e.Duration, e.Success) err := conn.InsertRequest(sessionID, e.Timestamp, truncSqIdx(e.Index), e.URL, e.Duration, e.Success)
if err == nil { if err == nil {
conn.insertAutocompleteValue(sessionID, 0, "REQUEST_IOS", url.DiscardURLQuery(e.URL)) conn.insertAutocompleteValue(sessionID, 0, "REQUEST_IOS", url.DiscardURLQuery(e.URL))
} }

2
backend/pkg/db/postgres/messages-web-stats.go
View file

@ -60,7 +60,7 @@ func (conn *Conn) InsertWebStatsResourceEvent(sessionID uint64, e *ResourceEvent
urlQuery := url.DiscardURLQuery(e.URL) urlQuery := url.DiscardURLQuery(e.URL)
urlMethod := url.EnsureMethod(e.Method) urlMethod := url.EnsureMethod(e.Method)
conn.batchQueue(sessionID, sqlRequest, conn.batchQueue(sessionID, sqlRequest,
sessionID, e.Timestamp, e.MessageID, sessionID, e.Timestamp, truncSqIdx(e.MessageID),
e.Type, e.Type,
e.URL, host, urlQuery, e.URL, host, urlQuery,
e.Success, e.Status, e.Success, e.Status,

16
backend/pkg/db/postgres/messages-web.go
View file

@ -10,7 +10,7 @@ import (
func (conn *Conn) InsertWebCustomEvent(sessionID uint64, projectID uint32, e *CustomEvent) error { func (conn *Conn) InsertWebCustomEvent(sessionID uint64, projectID uint32, e *CustomEvent) error {
err := conn.InsertCustomEvent(sessionID, e.Timestamp, err := conn.InsertCustomEvent(sessionID, e.Timestamp,
e.MessageID, truncSqIdx(e.MessageID),
e.Name, e.Payload) e.Name, e.Payload)
if err == nil { if err == nil {
conn.insertAutocompleteValue(sessionID, projectID, "CUSTOM", e.Name) conn.insertAutocompleteValue(sessionID, projectID, "CUSTOM", e.Name)
@ -40,7 +40,7 @@ func (conn *Conn) InsertWebPageEvent(sessionID uint64, projectID uint32, e *Page
return err return err
} }
// base_path is deprecated // base_path is deprecated
if err = conn.webPageEvents.Append(sessionID, e.MessageID, e.Timestamp, e.Referrer, url.DiscardURLQuery(e.Referrer), if err = conn.webPageEvents.Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.Referrer, url.DiscardURLQuery(e.Referrer),
host, path, query, e.DomContentLoadedEventEnd, e.LoadEventEnd, e.ResponseEnd, e.FirstPaint, e.FirstContentfulPaint, host, path, query, e.DomContentLoadedEventEnd, e.LoadEventEnd, e.ResponseEnd, e.FirstPaint, e.FirstContentfulPaint,
e.SpeedIndex, e.VisuallyComplete, e.TimeToInteractive, calcResponseTime(e), calcDomBuildingTime(e)); err != nil { e.SpeedIndex, e.VisuallyComplete, e.TimeToInteractive, calcResponseTime(e), calcDomBuildingTime(e)); err != nil {
log.Printf("insert web page event in bulk err: %s", err) log.Printf("insert web page event in bulk err: %s", err)
@ -63,7 +63,7 @@ func (conn *Conn) InsertWebClickEvent(sessionID uint64, projectID uint32, e *Cli
WHERE session_id = $1 AND timestamp <= $3 ORDER BY timestamp DESC LIMIT 1 WHERE session_id = $1 AND timestamp <= $3 ORDER BY timestamp DESC LIMIT 1
) )
` `
conn.batchQueue(sessionID, sqlRequest, sessionID, e.MessageID, e.Timestamp, e.Label, e.Selector) conn.batchQueue(sessionID, sqlRequest, sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.Label, e.Selector)
// Accumulate session updates and exec inside batch with another sql commands // Accumulate session updates and exec inside batch with another sql commands
conn.updateSessionEvents(sessionID, 1, 0) conn.updateSessionEvents(sessionID, 1, 0)
// Add new value set to autocomplete bulk // Add new value set to autocomplete bulk
@ -79,7 +79,7 @@ func (conn *Conn) InsertWebInputEvent(sessionID uint64, projectID uint32, e *Inp
if e.ValueMasked { if e.ValueMasked {
value = nil value = nil
} }
if err := conn.webInputEvents.Append(sessionID, e.MessageID, e.Timestamp, value, e.Label); err != nil { if err := conn.webInputEvents.Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, value, e.Label); err != nil {
log.Printf("insert web input event err: %s", err) log.Printf("insert web input event err: %s", err)
} }
conn.updateSessionEvents(sessionID, 1, 0) conn.updateSessionEvents(sessionID, 1, 0)
@ -117,7 +117,7 @@ func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *typ
VALUES VALUES
($1, $2, $3, $4) ($1, $2, $3, $4)
`, `,
sessionID, e.MessageID, e.Timestamp, errorID, sessionID, truncSqIdx(e.MessageID), e.Timestamp, errorID,
); err != nil { ); err != nil {
return err return err
} }
@ -138,7 +138,7 @@ func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *typ
$1, $2, $3, $4, $5 $1, $2, $3, $4, $5
) ON CONFLICT DO NOTHING` ) ON CONFLICT DO NOTHING`
for key, value := range e.Tags { for key, value := range e.Tags {
conn.batchQueue(sessionID, sqlRequest, sessionID, e.MessageID, errorID, key, value) conn.batchQueue(sessionID, sqlRequest, sessionID, truncSqIdx(e.MessageID), errorID, key, value)
} }
return return
@ -169,7 +169,7 @@ func (conn *Conn) InsertWebFetchEvent(sessionID uint64, projectID uint32, savePa
$12, $13 $12, $13
) ON CONFLICT DO NOTHING` ) ON CONFLICT DO NOTHING`
conn.batchQueue(sessionID, sqlRequest, conn.batchQueue(sessionID, sqlRequest,
sessionID, e.Timestamp, e.MessageID, sessionID, e.Timestamp, truncSqIdx(e.MessageID),
e.URL, host, path, query, e.URL, host, path, query,
request, response, e.Status, url.EnsureMethod(e.Method), request, response, e.Status, url.EnsureMethod(e.Method),
e.Duration, e.Status < 400, e.Duration, e.Status < 400,
@ -187,7 +187,7 @@ func (conn *Conn) InsertWebGraphQLEvent(sessionID uint64, projectID uint32, save
request = &e.Variables request = &e.Variables
response = &e.Response response = &e.Response
} }
if err := conn.webGraphQLEvents.Append(sessionID, e.Timestamp, e.MessageID, e.OperationName, request, response); err != nil { if err := conn.webGraphQLEvents.Append(sessionID, e.Timestamp, truncSqIdx(e.MessageID), e.OperationName, request, response); err != nil {
log.Printf("insert web graphQL event err: %s", err) log.Printf("insert web graphQL event err: %s", err)
} }
conn.insertAutocompleteValue(sessionID, projectID, "GRAPHQL", e.OperationName) conn.insertAutocompleteValue(sessionID, projectID, "GRAPHQL", e.OperationName)