jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(backend): removed insert operations for session's resources

Browse source
This commit is contained in:
Alexander 2024-10-30 13:46:13 +01:00
parent 6834eb8d8d
commit a17796e811
7 changed files with 1 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/cmd/db/main.go
View file

@ -56,7 +56,7 @@ func main() {
// Web messages
messages.MsgMetadata, messages.MsgIssueEvent, messages.MsgSessionStart, messages.MsgSessionEnd,
messages.MsgUserID, messages.MsgUserAnonymousID, messages.MsgIntegrationEvent, messages.MsgPerformanceTrackAggr,
messages.MsgJSException, messages.MsgResourceTiming, messages.MsgCustomEvent, messages.MsgCustomIssue,
messages.MsgJSException, messages.MsgCustomEvent, messages.MsgCustomIssue,
messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL, messages.MsgStateAction, messages.MsgMouseClick,
messages.MsgMouseClickDeprecated, messages.MsgSetPageLocation, messages.MsgSetPageLocationDeprecated,
messages.MsgPageLoadTiming, messages.MsgPageRenderTiming,

2
backend/internal/db/datasaver/methods.go
View file

@ -12,8 +12,6 @@ func (s *saverImpl) handleExtraMessage(msg Message) error {
switch m := msg.(type) {
case *PerformanceTrackAggr:
return s.pg.InsertWebStatsPerformance(m)
case *ResourceTiming:
return s.pg.InsertWebStatsResourceEvent(m)
}
return nil
}

1
backend/pkg/db/clickhouse/connector.go
View file

@ -11,7 +11,6 @@ type Connector interface {
Commit() error
Stop() error
InsertWebSession(session *sessions.Session) error
InsertWebResourceEvent(session *sessions.Session, msg *messages.ResourceTiming) error
InsertWebPageEvent(session *sessions.Session, msg *messages.PageEvent) error
InsertWebClickEvent(session *sessions.Session, msg *messages.MouseClick) error
InsertWebInputEvent(session *sessions.Session, msg *messages.InputEvent) error

32
backend/pkg/db/postgres/events.go
View file

@ -280,35 +280,3 @@ func (conn *Conn) InsertWebStatsPerformance(p *messages.PerformanceTrackAggr) er
)
return nil
}
func (conn *Conn) InsertWebStatsResourceEvent(e *messages.ResourceTiming) error {
sessionID := e.SessionID()
host, _, _, err := url.GetURLParts(e.URL)
if err != nil {
return err
}
msgType := url.GetResourceType(e.Initiator, e.URL)
sqlRequest := `
INSERT INTO events.resources (
session_id, timestamp, message_id,
type,
url, url_host, url_hostpath,
success, status,
duration, ttfb, header_size, encoded_body_size, decoded_body_size
) VALUES (
$1, $2, $3,
$4,
LEFT($5, 8000), LEFT($6, 300), LEFT($7, 2000),
$8, $9,
NULLIF($10, 0), NULLIF($11, 0), NULLIF($12, 0), NULLIF($13, 0), NULLIF($14, 0)
)`
urlQuery := url.DiscardURLQuery(e.URL)
conn.BatchQueue(sessionID, sqlRequest,
sessionID, e.Timestamp, truncSqIdx(e.MsgID()),
msgType,
e.URL, host, urlQuery,
e.Duration != 0, 0,
e.Duration, e.TTFB, e.HeaderSize, e.EncodedBodySize, e.DecodedBodySize,
)
return nil
}

22
backend/pkg/url/url.go
View file

@ -44,25 +44,3 @@ func getURLExtension(URL string) string {
i := strings.LastIndex(u.Path, ".")
return u.Path[i+1:]
}
func GetResourceType(initiator string, URL string) string {
switch initiator {
case "xmlhttprequest", "fetch":
return "fetch"
case "img":
return "img"
default:
switch getURLExtension(URL) {
case "css":
return "stylesheet"
case "js":
return "script"
case "png", "gif", "jpg", "jpeg", "svg":
return "img"
case "mp4", "mkv", "ogg", "webm", "avi", "mp3":
return "media"
default:
return "other"
}
}
}

2
ee/backend/internal/db/datasaver/methods.go
View file

@ -53,8 +53,6 @@ func (s *saverImpl) handleExtraMessage(msg messages.Message) error {
// Unique for Web
case *messages.PageEvent:
return s.ch.InsertWebPageEvent(session, m)
case *messages.ResourceTiming:
return s.ch.InsertWebResourceEvent(session, m)
case *messages.JSException:
wrapper, _ := types.WrapJSException(m)
return s.ch.InsertWebErrorEvent(session, wrapper)

29
ee/backend/pkg/db/clickhouse/connector.go
View file

@ -24,7 +24,6 @@ type Connector interface {
Stop() error
// Web
InsertWebSession(session *sessions.Session) error
InsertWebResourceEvent(session *sessions.Session, msg *messages.ResourceTiming) error
InsertWebPageEvent(session *sessions.Session, msg *messages.PageEvent) error
InsertWebClickEvent(session *sessions.Session, msg *messages.MouseClick) error
InsertWebErrorEvent(session *sessions.Session, msg *types.ErrorEvent) error
@ -116,7 +115,6 @@ func (c *connectorImpl) newBatch(name, query string) error {
var batches = map[string]string{
// Web
"sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, timezone, utm_source, utm_medium, utm_campaign) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?, ?, ?)",
"resources": "INSERT INTO experimental.resources (session_id, project_id, message_id, datetime, url, type, duration, ttfb, header_size, encoded_body_size, decoded_body_size, success, url_path) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000))",
"autocompletes": "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, SUBSTR(?, 1, 8000))",
"pages": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint_time, speed_index, visually_complete, time_to_interactive, url_path, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?)",
"clicks": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, hesitation_time, event_type, selector, normalized_x, normalized_y) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
@ -338,33 +336,6 @@ func extractUrlPath(fullUrl string) string {
return strings.ToLower(pathQuery)
}
func (c *connectorImpl) InsertWebResourceEvent(session *sessions.Session, msg *messages.ResourceTiming) error {
msgType := url.GetResourceType(msg.Initiator, msg.URL)
resourceType := url.EnsureType(msgType)
if resourceType == "" {
return fmt.Errorf("can't parse resource type, sess: %d, type: %s", session.SessionID, msgType)
}
if err := c.batches["resources"].Append(
session.SessionID,
uint16(session.ProjectID),
msg.MsgID(),
datetime(msg.Timestamp),
url.DiscardURLQuery(msg.URL),
msgType,
nullableUint16(uint16(msg.Duration)),
nullableUint16(uint16(msg.TTFB)),
nullableUint16(uint16(msg.HeaderSize)),
nullableUint32(uint32(msg.EncodedBodySize)),
nullableUint32(uint32(msg.DecodedBodySize)),
msg.Duration != 0,
extractUrlPath(msg.URL),
); err != nil {
c.checkError("resources", err)
return fmt.Errorf("can't append to resources batch: %s", err)
}
return nil
}
func (c *connectorImpl) InsertWebPageEvent(session *sessions.Session, msg *messages.PageEvent) error {
if err := c.batches["pages"].Append(
session.SessionID,