jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(backend/db): truncate messageIndex to Int32 before insertion

Browse source
This commit is contained in:
Alex Kaminskii 2022-10-31 11:12:27 +01:00
parent 2ec73b47c5
commit e1cd349cb9
5 changed files with 22 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
backend/pkg/db/postgres/helpers.go
View file

@ -1,6 +1,8 @@
package postgres
import (
"math"
"openreplay/backend/pkg/messages"
)
@ -33,3 +35,8 @@ func calcResponseTime(pe *messages.PageEvent) uint64 {
}
return 0
}
// TODO: review message indexing (it is better to have lower values in db for faster search (int4/int2))
func truncSqIdx(messageID uint64) uint32 {
return uint32(messageID % math.MaxInt32)
}

8
backend/pkg/db/postgres/messages-common.go
View file

@ -102,14 +102,14 @@ func (conn *Conn) HandleSessionEnd(sessionID uint64) error {
return conn.c.Exec(sqlRequest, sessionID)
}
func (conn *Conn) InsertRequest(sessionID uint64, timestamp uint64, index uint64, url string, duration uint64, success bool) error {
func (conn *Conn) InsertRequest(sessionID uint64, timestamp uint64, index uint32, url string, duration uint64, success bool) error {
if err := conn.requests.Append(sessionID, timestamp, index, url, duration, success); err != nil {
return fmt.Errorf("insert request in bulk err: %s", err)
}
return nil
}
func (conn *Conn) InsertCustomEvent(sessionID uint64, timestamp uint64, index uint64, name string, payload string) error {
func (conn *Conn) InsertCustomEvent(sessionID uint64, timestamp uint64, index uint32, name string, payload string) error {
if err := conn.customEvents.Append(sessionID, timestamp, index, name, payload); err != nil {
return fmt.Errorf("insert custom event in bulk err: %s", err)
}
@ -184,7 +184,7 @@ func (conn *Conn) InsertIssueEvent(sessionID uint64, projectID uint32, e *messag
$1, $2, $3, $4, CAST($5 AS jsonb)
)`,
sessionID, issueID, e.Timestamp,
e.MessageID,
truncSqIdx(e.MessageID),
payload,
); err != nil {
return err
@ -204,7 +204,7 @@ func (conn *Conn) InsertIssueEvent(sessionID uint64, projectID uint32, e *messag
VALUES
($1, $2, $3, left($4, 2700), $5, 'error')
`,
sessionID, e.MessageID, e.Timestamp, e.ContextString, e.Payload,
sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.ContextString, e.Payload,
); err != nil {
return err
}

4
backend/pkg/db/postgres/messages-ios.go
View file

@ -7,7 +7,7 @@ import (
)
func (conn *Conn) InsertIOSCustomEvent(sessionID uint64, e *messages.IOSCustomEvent) error {
err := conn.InsertCustomEvent(sessionID, e.Timestamp, e.Index, e.Name, e.Payload)
err := conn.InsertCustomEvent(sessionID, e.Timestamp, truncSqIdx(e.Index), e.Name, e.Payload)
if err == nil {
conn.insertAutocompleteValue(sessionID, 0, "CUSTOM_IOS", e.Name)
}
@ -31,7 +31,7 @@ func (conn *Conn) InsertIOSUserAnonymousID(sessionID uint64, userAnonymousID *me
}
func (conn *Conn) InsertIOSNetworkCall(sessionID uint64, e *messages.IOSNetworkCall) error {
err := conn.InsertRequest(sessionID, e.Timestamp, e.Index, e.URL, e.Duration, e.Success)
err := conn.InsertRequest(sessionID, e.Timestamp, truncSqIdx(e.Index), e.URL, e.Duration, e.Success)
if err == nil {
conn.insertAutocompleteValue(sessionID, 0, "REQUEST_IOS", url.DiscardURLQuery(e.URL))
}

2
backend/pkg/db/postgres/messages-web-stats.go
View file

@ -60,7 +60,7 @@ func (conn *Conn) InsertWebStatsResourceEvent(sessionID uint64, e *ResourceEvent
urlQuery := url.DiscardURLQuery(e.URL)
urlMethod := url.EnsureMethod(e.Method)
conn.batchQueue(sessionID, sqlRequest,
sessionID, e.Timestamp, e.MessageID,
sessionID, e.Timestamp, truncSqIdx(e.MessageID),
e.Type,
e.URL, host, urlQuery,
e.Success, e.Status,

16
backend/pkg/db/postgres/messages-web.go
View file

@ -10,7 +10,7 @@ import (
func (conn *Conn) InsertWebCustomEvent(sessionID uint64, projectID uint32, e *CustomEvent) error {
err := conn.InsertCustomEvent(sessionID, e.Timestamp,
e.MessageID,
truncSqIdx(e.MessageID),
e.Name, e.Payload)
if err == nil {
conn.insertAutocompleteValue(sessionID, projectID, "CUSTOM", e.Name)
@ -40,7 +40,7 @@ func (conn *Conn) InsertWebPageEvent(sessionID uint64, projectID uint32, e *Page
return err
}
// base_path is deprecated
if err = conn.webPageEvents.Append(sessionID, e.MessageID, e.Timestamp, e.Referrer, url.DiscardURLQuery(e.Referrer),
if err = conn.webPageEvents.Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.Referrer, url.DiscardURLQuery(e.Referrer),
host, path, query, e.DomContentLoadedEventEnd, e.LoadEventEnd, e.ResponseEnd, e.FirstPaint, e.FirstContentfulPaint,
e.SpeedIndex, e.VisuallyComplete, e.TimeToInteractive, calcResponseTime(e), calcDomBuildingTime(e)); err != nil {
log.Printf("insert web page event in bulk err: %s", err)
@ -63,7 +63,7 @@ func (conn *Conn) InsertWebClickEvent(sessionID uint64, projectID uint32, e *Cli
WHERE session_id = $1 AND timestamp <= $3 ORDER BY timestamp DESC LIMIT 1
)
`
conn.batchQueue(sessionID, sqlRequest, sessionID, e.MessageID, e.Timestamp, e.Label, e.Selector)
conn.batchQueue(sessionID, sqlRequest, sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.Label, e.Selector)
// Accumulate session updates and exec inside batch with another sql commands
conn.updateSessionEvents(sessionID, 1, 0)
// Add new value set to autocomplete bulk
@ -79,7 +79,7 @@ func (conn *Conn) InsertWebInputEvent(sessionID uint64, projectID uint32, e *Inp
if e.ValueMasked {
value = nil
}
if err := conn.webInputEvents.Append(sessionID, e.MessageID, e.Timestamp, value, e.Label); err != nil {
if err := conn.webInputEvents.Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, value, e.Label); err != nil {
log.Printf("insert web input event err: %s", err)
}
conn.updateSessionEvents(sessionID, 1, 0)
@ -117,7 +117,7 @@ func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *typ
VALUES
($1, $2, $3, $4)
`,
sessionID, e.MessageID, e.Timestamp, errorID,
sessionID, truncSqIdx(e.MessageID), e.Timestamp, errorID,
); err != nil {
return err
}
@ -138,7 +138,7 @@ func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *typ
$1, $2, $3, $4, $5
) ON CONFLICT DO NOTHING`
for key, value := range e.Tags {
conn.batchQueue(sessionID, sqlRequest, sessionID, e.MessageID, errorID, key, value)
conn.batchQueue(sessionID, sqlRequest, sessionID, truncSqIdx(e.MessageID), errorID, key, value)
}
return
@ -169,7 +169,7 @@ func (conn *Conn) InsertWebFetchEvent(sessionID uint64, projectID uint32, savePa
$12, $13
) ON CONFLICT DO NOTHING`
conn.batchQueue(sessionID, sqlRequest,
sessionID, e.Timestamp, e.MessageID,
sessionID, e.Timestamp, truncSqIdx(e.MessageID),
e.URL, host, path, query,
request, response, e.Status, url.EnsureMethod(e.Method),
e.Duration, e.Status < 400,
@ -187,7 +187,7 @@ func (conn *Conn) InsertWebGraphQLEvent(sessionID uint64, projectID uint32, save
request = &e.Variables
response = &e.Response
}
if err := conn.webGraphQLEvents.Append(sessionID, e.Timestamp, e.MessageID, e.OperationName, request, response); err != nil {
if err := conn.webGraphQLEvents.Append(sessionID, e.Timestamp, truncSqIdx(e.MessageID), e.OperationName, request, response); err != nil {
log.Printf("insert web graphQL event err: %s", err)
}
conn.insertAutocompleteValue(sessionID, projectID, "GRAPHQL", e.OperationName)