From a17796e811068880196ef56c2b9a66e7c3d3c284 Mon Sep 17 00:00:00 2001 From: Alexander Date: Wed, 30 Oct 2024 13:46:13 +0100 Subject: [PATCH] feat(backend): removed insert operations for session's resources --- backend/cmd/db/main.go | 2 +- backend/internal/db/datasaver/methods.go | 2 -- backend/pkg/db/clickhouse/connector.go | 1 - backend/pkg/db/postgres/events.go | 32 --------------------- backend/pkg/url/url.go | 22 -------------- ee/backend/internal/db/datasaver/methods.go | 2 -- ee/backend/pkg/db/clickhouse/connector.go | 29 ------------------- 7 files changed, 1 insertion(+), 89 deletions(-) diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index 8585b8f66..5d75b02d7 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -56,7 +56,7 @@ func main() { // Web messages messages.MsgMetadata, messages.MsgIssueEvent, messages.MsgSessionStart, messages.MsgSessionEnd, messages.MsgUserID, messages.MsgUserAnonymousID, messages.MsgIntegrationEvent, messages.MsgPerformanceTrackAggr, - messages.MsgJSException, messages.MsgResourceTiming, messages.MsgCustomEvent, messages.MsgCustomIssue, + messages.MsgJSException, messages.MsgCustomEvent, messages.MsgCustomIssue, messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL, messages.MsgStateAction, messages.MsgMouseClick, messages.MsgMouseClickDeprecated, messages.MsgSetPageLocation, messages.MsgSetPageLocationDeprecated, messages.MsgPageLoadTiming, messages.MsgPageRenderTiming, diff --git a/backend/internal/db/datasaver/methods.go b/backend/internal/db/datasaver/methods.go index c4e83cf09..07a8b6ba2 100644 --- a/backend/internal/db/datasaver/methods.go +++ b/backend/internal/db/datasaver/methods.go @@ -12,8 +12,6 @@ func (s *saverImpl) handleExtraMessage(msg Message) error { switch m := msg.(type) { case *PerformanceTrackAggr: return s.pg.InsertWebStatsPerformance(m) - case *ResourceTiming: - return s.pg.InsertWebStatsResourceEvent(m) } return nil } diff --git a/backend/pkg/db/clickhouse/connector.go b/backend/pkg/db/clickhouse/connector.go index 2d7ee3ada..727ad7f7b 100644 --- a/backend/pkg/db/clickhouse/connector.go +++ b/backend/pkg/db/clickhouse/connector.go @@ -11,7 +11,6 @@ type Connector interface { Commit() error Stop() error InsertWebSession(session *sessions.Session) error - InsertWebResourceEvent(session *sessions.Session, msg *messages.ResourceTiming) error InsertWebPageEvent(session *sessions.Session, msg *messages.PageEvent) error InsertWebClickEvent(session *sessions.Session, msg *messages.MouseClick) error InsertWebInputEvent(session *sessions.Session, msg *messages.InputEvent) error diff --git a/backend/pkg/db/postgres/events.go b/backend/pkg/db/postgres/events.go index 47b9cb566..cea5f99d7 100644 --- a/backend/pkg/db/postgres/events.go +++ b/backend/pkg/db/postgres/events.go @@ -280,35 +280,3 @@ func (conn *Conn) InsertWebStatsPerformance(p *messages.PerformanceTrackAggr) er ) return nil } - -func (conn *Conn) InsertWebStatsResourceEvent(e *messages.ResourceTiming) error { - sessionID := e.SessionID() - host, _, _, err := url.GetURLParts(e.URL) - if err != nil { - return err - } - msgType := url.GetResourceType(e.Initiator, e.URL) - sqlRequest := ` - INSERT INTO events.resources ( - session_id, timestamp, message_id, - type, - url, url_host, url_hostpath, - success, status, - duration, ttfb, header_size, encoded_body_size, decoded_body_size - ) VALUES ( - $1, $2, $3, - $4, - LEFT($5, 8000), LEFT($6, 300), LEFT($7, 2000), - $8, $9, - NULLIF($10, 0), NULLIF($11, 0), NULLIF($12, 0), NULLIF($13, 0), NULLIF($14, 0) - )` - urlQuery := url.DiscardURLQuery(e.URL) - conn.BatchQueue(sessionID, sqlRequest, - sessionID, e.Timestamp, truncSqIdx(e.MsgID()), - msgType, - e.URL, host, urlQuery, - e.Duration != 0, 0, - e.Duration, e.TTFB, e.HeaderSize, e.EncodedBodySize, e.DecodedBodySize, - ) - return nil -} diff --git a/backend/pkg/url/url.go b/backend/pkg/url/url.go index 3f8620497..c32be9517 100644 --- a/backend/pkg/url/url.go +++ b/backend/pkg/url/url.go @@ -44,25 +44,3 @@ func getURLExtension(URL string) string { i := strings.LastIndex(u.Path, ".") return u.Path[i+1:] } - -func GetResourceType(initiator string, URL string) string { - switch initiator { - case "xmlhttprequest", "fetch": - return "fetch" - case "img": - return "img" - default: - switch getURLExtension(URL) { - case "css": - return "stylesheet" - case "js": - return "script" - case "png", "gif", "jpg", "jpeg", "svg": - return "img" - case "mp4", "mkv", "ogg", "webm", "avi", "mp3": - return "media" - default: - return "other" - } - } -} diff --git a/ee/backend/internal/db/datasaver/methods.go b/ee/backend/internal/db/datasaver/methods.go index 3583fc850..1644a1fc0 100644 --- a/ee/backend/internal/db/datasaver/methods.go +++ b/ee/backend/internal/db/datasaver/methods.go @@ -53,8 +53,6 @@ func (s *saverImpl) handleExtraMessage(msg messages.Message) error { // Unique for Web case *messages.PageEvent: return s.ch.InsertWebPageEvent(session, m) - case *messages.ResourceTiming: - return s.ch.InsertWebResourceEvent(session, m) case *messages.JSException: wrapper, _ := types.WrapJSException(m) return s.ch.InsertWebErrorEvent(session, wrapper) diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index 16b65fdc8..a76dfac5f 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -24,7 +24,6 @@ type Connector interface { Stop() error // Web InsertWebSession(session *sessions.Session) error - InsertWebResourceEvent(session *sessions.Session, msg *messages.ResourceTiming) error InsertWebPageEvent(session *sessions.Session, msg *messages.PageEvent) error InsertWebClickEvent(session *sessions.Session, msg *messages.MouseClick) error InsertWebErrorEvent(session *sessions.Session, msg *types.ErrorEvent) error @@ -116,7 +115,6 @@ func (c *connectorImpl) newBatch(name, query string) error { var batches = map[string]string{ // Web "sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, timezone, utm_source, utm_medium, utm_campaign) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?, ?, ?)", - "resources": "INSERT INTO experimental.resources (session_id, project_id, message_id, datetime, url, type, duration, ttfb, header_size, encoded_body_size, decoded_body_size, success, url_path) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000))", "autocompletes": "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, SUBSTR(?, 1, 8000))", "pages": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint_time, speed_index, visually_complete, time_to_interactive, url_path, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?)", "clicks": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, hesitation_time, event_type, selector, normalized_x, normalized_y) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", @@ -338,33 +336,6 @@ func extractUrlPath(fullUrl string) string { return strings.ToLower(pathQuery) } -func (c *connectorImpl) InsertWebResourceEvent(session *sessions.Session, msg *messages.ResourceTiming) error { - msgType := url.GetResourceType(msg.Initiator, msg.URL) - resourceType := url.EnsureType(msgType) - if resourceType == "" { - return fmt.Errorf("can't parse resource type, sess: %d, type: %s", session.SessionID, msgType) - } - if err := c.batches["resources"].Append( - session.SessionID, - uint16(session.ProjectID), - msg.MsgID(), - datetime(msg.Timestamp), - url.DiscardURLQuery(msg.URL), - msgType, - nullableUint16(uint16(msg.Duration)), - nullableUint16(uint16(msg.TTFB)), - nullableUint16(uint16(msg.HeaderSize)), - nullableUint32(uint32(msg.EncodedBodySize)), - nullableUint32(uint32(msg.DecodedBodySize)), - msg.Duration != 0, - extractUrlPath(msg.URL), - ); err != nil { - c.checkError("resources", err) - return fmt.Errorf("can't append to resources batch: %s", err) - } - return nil -} - func (c *connectorImpl) InsertWebPageEvent(session *sessions.Session, msg *messages.PageEvent) error { if err := c.batches["pages"].Append( session.SessionID,