jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[DB] removed pg transactions (#889)

Browse source 
* feat(backend): removed pg transactions for webErrors and issueEvents
This commit is contained in:
Alexander 2022-12-20 13:32:32 +01:00 • committed by GitHub
parent 943620e855
commit c54e23386a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 111 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
backend/pkg/db/postgres/connector.go
View file

@ -35,6 +35,12 @@ type Conn struct {
webPageEvents Bulk
webInputEvents Bulk
webGraphQL Bulk
webErrors Bulk
webErrorEvents Bulk
webErrorTags Bulk
webIssues Bulk
webIssueEvents Bulk
webCustomEvents Bulk
sessionUpdates map[uint64]*sessionUpdates
batchQueueLimit int
batchSizeLimit int
@ -152,6 +158,54 @@ func (conn *Conn) initBulks() {
if err != nil {
log.Fatalf("can't create webPageEvents bulk")
}
conn.webErrors, err = NewBulk(conn.c,
"errors",
"(error_id, project_id, source, name, message, payload)",
"($%d, $%d, $%d, $%d, $%d, $%d::jsonb)",
6, 100)
if err != nil {
log.Fatalf("can't create webErrors bulk")
}
conn.webErrorEvents, err = NewBulk(conn.c,
"events.errors",
"(session_id, message_id, timestamp, error_id)",
"($%d, $%d, $%d, $%d)",
4, 100)
if err != nil {
log.Fatalf("can't create webErrorEvents bulk")
}
conn.webErrorTags, err = NewBulk(conn.c,
"public.errors_tags",
"(session_id, message_id, error_id, key, value)",
"($%d, $%d, $%d, $%d, $%d)",
5, 100)
if err != nil {
log.Fatalf("can't create webErrorEvents bulk")
}
conn.webIssues, err = NewBulk(conn.c,
"issues",
"(project_id, issue_id, type, context_string)",
"($%d, $%d, $%d, $%d)",
4, 100)
if err != nil {
log.Fatalf("can't create webIssues bulk")
}
conn.webIssueEvents, err = NewBulk(conn.c,
"events_common.issues",
"(session_id, issue_id, timestamp, seq_index, payload)",
"($%d, $%d, $%d, $%d, CAST($%d AS jsonb))",
5, 100)
if err != nil {
log.Fatalf("can't create webIssueEvents bulk")
}
conn.webCustomEvents, err = NewBulk(conn.c,
"events_common.customs",
"(session_id, seq_index, timestamp, name, payload, level)",
"($%d, $%d, $%d, left($%d, 2700), $%d, $%d)",
6, 100)
if err != nil {
log.Fatalf("can't create webCustomEvents bulk")
}
}
func (conn *Conn) insertAutocompleteValue(sessionID uint64, projectID uint32, tp string, value string) {
@ -195,7 +249,14 @@ func (conn *Conn) updateSessionEvents(sessionID uint64, events, pages int) {
if _, ok := conn.sessionUpdates[sessionID]; !ok {
conn.sessionUpdates[sessionID] = NewSessionUpdates(sessionID)
}
conn.sessionUpdates[sessionID].add(pages, events)
conn.sessionUpdates[sessionID].addEvents(pages, events)
}
func (conn *Conn) updateSessionIssues(sessionID uint64, errors, issueScore int) {
if _, ok := conn.sessionUpdates[sessionID]; !ok {
conn.sessionUpdates[sessionID] = NewSessionUpdates(sessionID)
}
conn.sessionUpdates[sessionID].addIssues(errors, issueScore)
}
func (conn *Conn) sendBulks() {
@ -217,6 +278,24 @@ func (conn *Conn) sendBulks() {
if err := conn.webGraphQL.Send(); err != nil {
log.Printf("webGraphQL bulk send err: %s", err)
}
if err := conn.webErrors.Send(); err != nil {
log.Printf("webErrors bulk send err: %s", err)
}
if err := conn.webErrorEvents.Send(); err != nil {
log.Printf("webErrorEvents bulk send err: %s", err)
}
if err := conn.webErrorTags.Send(); err != nil {
log.Printf("webErrorTags bulk send err: %s", err)
}
if err := conn.webIssues.Send(); err != nil {
log.Printf("webIssues bulk send err: %s", err)
}
if err := conn.webIssueEvents.Send(); err != nil {
log.Printf("webIssueEvents bulk send err: %s", err)
}
if err := conn.webCustomEvents.Send(); err != nil {
log.Printf("webCustomEvents bulk send err: %s", err)
}
}
func (conn *Conn) CommitBatches() {

64
backend/pkg/db/postgres/messages-common.go
View file

@ -146,70 +146,24 @@ func (conn *Conn) InsertMetadata(sessionID uint64, keyNo uint, value string) err
return conn.c.Exec(fmt.Sprintf(sqlRequest, keyNo), value, sessionID)
}
func (conn *Conn) InsertIssueEvent(sessionID uint64, projectID uint32, e *messages.IssueEvent) (err error) {
tx, err := conn.c.Begin()
if err != nil {
return err
}
defer func() {
if err != nil {
if rollbackErr := tx.rollback(); rollbackErr != nil {
log.Printf("rollback err: %s", rollbackErr)
}
}
}()
func (conn *Conn) InsertIssueEvent(sessionID uint64, projectID uint32, e *messages.IssueEvent) error {
issueID := hashid.IssueID(projectID, e)
// TEMP. TODO: nullable & json message field type
payload := &e.Payload
if *payload == "" || *payload == "{}" {
payload = nil
}
if err = tx.exec(`
INSERT INTO issues (
project_id, issue_id, type, context_string
) (SELECT
project_id, $2, $3, $4
FROM sessions
WHERE session_id = $1
)ON CONFLICT DO NOTHING`,
sessionID, issueID, e.Type, e.ContextString,
); err != nil {
return err
if err := conn.webIssues.Append(projectID, issueID, e.Type, e.ContextString); err != nil {
log.Printf("insert web issue err: %s", err)
}
if err = tx.exec(`
INSERT INTO events_common.issues (
session_id, issue_id, timestamp, seq_index, payload
) VALUES (
$1, $2, $3, $4, CAST($5 AS jsonb)
)`,
sessionID, issueID, e.Timestamp,
truncSqIdx(e.MessageID),
payload,
); err != nil {
return err
if err := conn.webIssueEvents.Append(sessionID, issueID, e.Timestamp, truncSqIdx(e.MessageID), payload); err != nil {
log.Printf("insert web issue event err: %s", err)
}
if err = tx.exec(`
UPDATE sessions SET issue_score = issue_score + $2
WHERE session_id = $1`,
sessionID, getIssueScore(e),
); err != nil {
return err
}
// TODO: no redundancy. Deliver to UI in a different way
conn.updateSessionIssues(sessionID, 0, getIssueScore(e))
if e.Type == "custom" {
if err = tx.exec(`
INSERT INTO events_common.customs
(session_id, seq_index, timestamp, name, payload, level)
VALUES
($1, $2, $3, left($4, 2700), $5, 'error')
`,
sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.ContextString, e.Payload,
); err != nil {
return err
if err := conn.webCustomEvents.Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.ContextString, e.Payload, "error"); err != nil {
log.Printf("insert web custom event err: %s", err)
}
}
err = tx.commit()
return
return nil
}

60
backend/pkg/db/postgres/messages-web.go
View file

@ -91,61 +91,21 @@ func (conn *Conn) InsertWebInputEvent(sessionID uint64, projectID uint32, e *Inp
return nil
}
func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *types.ErrorEvent) (err error) {
tx, err := conn.c.Begin()
if err != nil {
return err
}
defer func() {
if err != nil {
if rollbackErr := tx.rollback(); rollbackErr != nil {
log.Printf("rollback err: %s", rollbackErr)
}
}
}()
func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *types.ErrorEvent) error {
errorID := e.ID(projectID)
if err = tx.exec(`
INSERT INTO errors
(error_id, project_id, source, name, message, payload)
VALUES
($1, $2, $3, $4, $5, $6::jsonb)
ON CONFLICT DO NOTHING`,
errorID, projectID, e.Source, e.Name, e.Message, e.Payload,
); err != nil {
return err
if err := conn.webErrors.Append(errorID, projectID, e.Source, e.Name, e.Message, e.Payload); err != nil {
log.Printf("insert web error err: %s", err)
}
if err = tx.exec(`
INSERT INTO events.errors
(session_id, message_id, timestamp, error_id)
VALUES
($1, $2, $3, $4)
`,
sessionID, truncSqIdx(e.MessageID), e.Timestamp, errorID,
); err != nil {
return err
if err := conn.webErrorEvents.Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, errorID); err != nil {
log.Printf("insert web error event err: %s", err)
}
if err = tx.exec(`
UPDATE sessions SET errors_count = errors_count + 1, issue_score = issue_score + 1000
WHERE session_id = $1`,
sessionID,
); err != nil {
return err
}
err = tx.commit()
// Insert tags
sqlRequest := `
INSERT INTO public.errors_tags (
session_id, message_id, error_id, key, value
) VALUES (
$1, $2, $3, $4, $5
) ON CONFLICT DO NOTHING`
conn.updateSessionIssues(sessionID, 1, 1000)
for key, value := range e.Tags {
conn.batchQueue(sessionID, sqlRequest, sessionID, truncSqIdx(e.MessageID), errorID, key, value)
if err := conn.webErrorTags.Append(sessionID, truncSqIdx(e.MessageID), errorID, key, value); err != nil {
log.Printf("insert web error token err: %s", err)
}
return
}
return nil
}
func (conn *Conn) InsertWebNetworkRequest(sessionID uint64, projectID uint32, savePayload bool, e *NetworkRequest) error {

15
backend/pkg/db/postgres/session-updates.go
View file

@ -1,12 +1,14 @@
package postgres
// Mechanism of combination several session updates into one
const sessionUpdateReq = `UPDATE sessions SET pages_count = pages_count + $1, events_count = events_count + $2 WHERE session_id = $3`
const sessionUpdateReq = `UPDATE sessions SET pages_count = pages_count + $1, events_count = events_count + $2, errors_count = errors_count + $3, issue_score = issue_score + $4 WHERE session_id = $5`
type sessionUpdates struct {
sessionID uint64
pages int
events int
errors int
issues int
}
func NewSessionUpdates(sessionID uint64) *sessionUpdates {
@ -14,17 +16,24 @@ func NewSessionUpdates(sessionID uint64) *sessionUpdates {
sessionID: sessionID,
pages: 0,
events: 0,
errors: 0,
issues: 0,
}
}
func (su *sessionUpdates) add(pages, events int) {
func (su *sessionUpdates) addEvents(pages, events int) {
su.pages += pages
su.events += events
}
func (su *sessionUpdates) addIssues(errors, issues int) {
su.errors += errors
su.issues += issues
}
func (su *sessionUpdates) request() (string, []interface{}) {
if su.pages == 0 && su.events == 0 {
return "", nil
}
return sessionUpdateReq, []interface{}{su.pages, su.events, su.sessionID}
return sessionUpdateReq, []interface{}{su.pages, su.events, su.errors, su.issues, su.sessionID}
}