[DB] removed pg transactions (#889)

* feat(backend): removed pg transactions for webErrors and issueEvents

backend/pkg/db/postgres/connector.go

@@ -35,6 +35,12 @@ type Conn struct {
 	webPageEvents     Bulk
 	webInputEvents    Bulk
 	webGraphQL        Bulk
+	webErrors         Bulk
+	webErrorEvents    Bulk
+	webErrorTags      Bulk
+	webIssues         Bulk
+	webIssueEvents    Bulk
+	webCustomEvents   Bulk
 	sessionUpdates    map[uint64]*sessionUpdates
 	batchQueueLimit   int
 	batchSizeLimit    int
@@ -152,6 +158,54 @@ func (conn *Conn) initBulks() {
 	if err != nil {
 		log.Fatalf("can't create webPageEvents bulk")
 	}
+	conn.webErrors, err = NewBulk(conn.c,
+		"errors",
+		"(error_id, project_id, source, name, message, payload)",
+		"($%d, $%d, $%d, $%d, $%d, $%d::jsonb)",
+		6, 100)
+	if err != nil {
+		log.Fatalf("can't create webErrors bulk")
+	}
+	conn.webErrorEvents, err = NewBulk(conn.c,
+		"events.errors",
+		"(session_id, message_id, timestamp, error_id)",
+		"($%d, $%d, $%d, $%d)",
+		4, 100)
+	if err != nil {
+		log.Fatalf("can't create webErrorEvents bulk")
+	}
+	conn.webErrorTags, err = NewBulk(conn.c,
+		"public.errors_tags",
+		"(session_id, message_id, error_id, key, value)",
+		"($%d, $%d, $%d, $%d, $%d)",
+		5, 100)
+	if err != nil {
+		log.Fatalf("can't create webErrorEvents bulk")
+	}
+	conn.webIssues, err = NewBulk(conn.c,
+		"issues",
+		"(project_id, issue_id, type, context_string)",
+		"($%d, $%d, $%d, $%d)",
+		4, 100)
+	if err != nil {
+		log.Fatalf("can't create webIssues bulk")
+	}
+	conn.webIssueEvents, err = NewBulk(conn.c,
+		"events_common.issues",
+		"(session_id, issue_id, timestamp, seq_index, payload)",
+		"($%d, $%d, $%d, $%d, CAST($%d AS jsonb))",
+		5, 100)
+	if err != nil {
+		log.Fatalf("can't create webIssueEvents bulk")
+	}
+	conn.webCustomEvents, err = NewBulk(conn.c,
+		"events_common.customs",
+		"(session_id, seq_index, timestamp, name, payload, level)",
+		"($%d, $%d, $%d, left($%d, 2700), $%d, $%d)",
+		6, 100)
+	if err != nil {
+		log.Fatalf("can't create webCustomEvents bulk")
+	}
 }
 
 func (conn *Conn) insertAutocompleteValue(sessionID uint64, projectID uint32, tp string, value string) {
@@ -195,7 +249,14 @@ func (conn *Conn) updateSessionEvents(sessionID uint64, events, pages int) {
 	if _, ok := conn.sessionUpdates[sessionID]; !ok {
 		conn.sessionUpdates[sessionID] = NewSessionUpdates(sessionID)
 	}
-	conn.sessionUpdates[sessionID].add(pages, events)
+	conn.sessionUpdates[sessionID].addEvents(pages, events)
+}
+
+func (conn *Conn) updateSessionIssues(sessionID uint64, errors, issueScore int) {
+	if _, ok := conn.sessionUpdates[sessionID]; !ok {
+		conn.sessionUpdates[sessionID] = NewSessionUpdates(sessionID)
+	}
+	conn.sessionUpdates[sessionID].addIssues(errors, issueScore)
 }
 
 func (conn *Conn) sendBulks() {
@@ -217,6 +278,24 @@ func (conn *Conn) sendBulks() {
 	if err := conn.webGraphQL.Send(); err != nil {
 		log.Printf("webGraphQL bulk send err: %s", err)
 	}
+	if err := conn.webErrors.Send(); err != nil {
+		log.Printf("webErrors bulk send err: %s", err)
+	}
+	if err := conn.webErrorEvents.Send(); err != nil {
+		log.Printf("webErrorEvents bulk send err: %s", err)
+	}
+	if err := conn.webErrorTags.Send(); err != nil {
+		log.Printf("webErrorTags bulk send err: %s", err)
+	}
+	if err := conn.webIssues.Send(); err != nil {
+		log.Printf("webIssues bulk send err: %s", err)
+	}
+	if err := conn.webIssueEvents.Send(); err != nil {
+		log.Printf("webIssueEvents bulk send err: %s", err)
+	}
+	if err := conn.webCustomEvents.Send(); err != nil {
+		log.Printf("webCustomEvents bulk send err: %s", err)
+	}
 }
 
 func (conn *Conn) CommitBatches() {

backend/pkg/db/postgres/messages-common.go

@@ -146,70 +146,24 @@ func (conn *Conn) InsertMetadata(sessionID uint64, keyNo uint, value string) err
 	return conn.c.Exec(fmt.Sprintf(sqlRequest, keyNo), value, sessionID)
 }
 
-func (conn *Conn) InsertIssueEvent(sessionID uint64, projectID uint32, e *messages.IssueEvent) (err error) {
+func (conn *Conn) InsertIssueEvent(sessionID uint64, projectID uint32, e *messages.IssueEvent) error {
-	tx, err := conn.c.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			if rollbackErr := tx.rollback(); rollbackErr != nil {
-				log.Printf("rollback err: %s", rollbackErr)
-			}
-		}
-	}()
 	issueID := hashid.IssueID(projectID, e)
-
-	// TEMP. TODO: nullable & json message field type
 	payload := &e.Payload
 	if *payload == "" || *payload == "{}" {
 		payload = nil
 	}
 
-	if err = tx.exec(`
+	if err := conn.webIssues.Append(projectID, issueID, e.Type, e.ContextString); err != nil {
-		INSERT INTO issues (
+		log.Printf("insert web issue err: %s", err)
-			project_id, issue_id, type, context_string
-		) (SELECT
-			project_id, $2, $3, $4
-			FROM sessions
-			WHERE session_id = $1
-		)ON CONFLICT DO NOTHING`,
-		sessionID, issueID, e.Type, e.ContextString,
-	); err != nil {
-		return err
 	}
-	if err = tx.exec(`
+	if err := conn.webIssueEvents.Append(sessionID, issueID, e.Timestamp, truncSqIdx(e.MessageID), payload); err != nil {
-		INSERT INTO events_common.issues (
+		log.Printf("insert web issue event err: %s", err)
-			session_id, issue_id, timestamp, seq_index, payload
-		) VALUES (
-			$1, $2, $3, $4, CAST($5 AS jsonb)
-		)`,
-		sessionID, issueID, e.Timestamp,
-		truncSqIdx(e.MessageID),
-		payload,
-	); err != nil {
-		return err
 	}
-	if err = tx.exec(`
+	conn.updateSessionIssues(sessionID, 0, getIssueScore(e))
-		UPDATE sessions SET issue_score = issue_score + $2
-		WHERE session_id = $1`,
-		sessionID, getIssueScore(e),
-	); err != nil {
-		return err
-	}
-	// TODO: no redundancy. Deliver to UI in a different way
 	if e.Type == "custom" {
-		if err = tx.exec(`
+		if err := conn.webCustomEvents.Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.ContextString, e.Payload, "error"); err != nil {
-			INSERT INTO events_common.customs
+			log.Printf("insert web custom event err: %s", err)
-				(session_id, seq_index, timestamp, name, payload, level)
-			VALUES
-				($1, $2, $3, left($4, 2700), $5, 'error')
-			`,
-			sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.ContextString, e.Payload,
-		); err != nil {
-			return err
 		}
 	}
-	err = tx.commit()
+	return nil
-	return
 }

backend/pkg/db/postgres/messages-web.go

@@ -91,61 +91,21 @@ func (conn *Conn) InsertWebInputEvent(sessionID uint64, projectID uint32, e *Inp
 	return nil
 }
 
-func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *types.ErrorEvent) (err error) {
+func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *types.ErrorEvent) error {
-	tx, err := conn.c.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			if rollbackErr := tx.rollback(); rollbackErr != nil {
-				log.Printf("rollback err: %s", rollbackErr)
-			}
-		}
-	}()
 	errorID := e.ID(projectID)
-
+	if err := conn.webErrors.Append(errorID, projectID, e.Source, e.Name, e.Message, e.Payload); err != nil {
-	if err = tx.exec(`
+		log.Printf("insert web error err: %s", err)
-		INSERT INTO errors
-			(error_id, project_id, source, name, message, payload)
-		VALUES
-			($1, $2, $3, $4, $5, $6::jsonb)
-		ON CONFLICT DO NOTHING`,
-		errorID, projectID, e.Source, e.Name, e.Message, e.Payload,
-	); err != nil {
-		return err
 	}
-	if err = tx.exec(`
+	if err := conn.webErrorEvents.Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, errorID); err != nil {
-		INSERT INTO events.errors
+		log.Printf("insert web error event err: %s", err)
-			(session_id, message_id, timestamp, error_id)
-		VALUES
-			($1, $2, $3, $4)
-		`,
-		sessionID, truncSqIdx(e.MessageID), e.Timestamp, errorID,
-	); err != nil {
-		return err
 	}
-	if err = tx.exec(`
+	conn.updateSessionIssues(sessionID, 1, 1000)
-		UPDATE sessions SET errors_count = errors_count + 1, issue_score = issue_score + 1000
-		WHERE session_id = $1`,
-		sessionID,
-	); err != nil {
-		return err
-	}
-	err = tx.commit()
-
-	// Insert tags
-	sqlRequest := `
-		INSERT INTO public.errors_tags (
-			session_id, message_id, error_id, key, value
-		) VALUES (
-			$1, $2, $3, $4, $5
-		) ON CONFLICT DO NOTHING`
 	for key, value := range e.Tags {
-		conn.batchQueue(sessionID, sqlRequest, sessionID, truncSqIdx(e.MessageID), errorID, key, value)
+		if err := conn.webErrorTags.Append(sessionID, truncSqIdx(e.MessageID), errorID, key, value); err != nil {
+			log.Printf("insert web error token err: %s", err)
+		}
 	}
-
+	return nil
-	return
 }
 
 func (conn *Conn) InsertWebNetworkRequest(sessionID uint64, projectID uint32, savePayload bool, e *NetworkRequest) error {

backend/pkg/db/postgres/session-updates.go

@@ -1,12 +1,14 @@
 package postgres
 
 // Mechanism of combination several session updates into one
-const sessionUpdateReq = `UPDATE sessions SET pages_count = pages_count + $1, events_count = events_count + $2 WHERE session_id = $3`
+const sessionUpdateReq = `UPDATE sessions SET pages_count = pages_count + $1, events_count = events_count + $2, errors_count = errors_count + $3, issue_score = issue_score + $4 WHERE session_id = $5`
 
 type sessionUpdates struct {
 	sessionID uint64
 	pages     int
 	events    int
+	errors    int
+	issues    int
 }
 
 func NewSessionUpdates(sessionID uint64) *sessionUpdates {
@@ -14,17 +16,24 @@ func NewSessionUpdates(sessionID uint64) *sessionUpdates {
 		sessionID: sessionID,
 		pages:     0,
 		events:    0,
+		errors:    0,
+		issues:    0,
 	}
 }
 
-func (su *sessionUpdates) add(pages, events int) {
+func (su *sessionUpdates) addEvents(pages, events int) {
 	su.pages += pages
 	su.events += events
 }
 
+func (su *sessionUpdates) addIssues(errors, issues int) {
+	su.errors += errors
+	su.issues += issues
+}
+
 func (su *sessionUpdates) request() (string, []interface{}) {
 	if su.pages == 0 && su.events == 0 {
 		return "", nil
 	}
-	return sessionUpdateReq, []interface{}{su.pages, su.events, su.sessionID}
+	return sessionUpdateReq, []interface{}{su.pages, su.events, su.errors, su.issues, su.sessionID}
 }