From 7ed296736108f2c264933a0631fbb5f07e6f3fbd Mon Sep 17 00:00:00 2001 From: Alexander Date: Thu, 26 Oct 2023 17:11:19 +0200 Subject: [PATCH] feat(backend): added mobile event handlers for CH db --- backend/internal/db/datasaver/saver.go | 5 - ee/backend/internal/db/datasaver/methods.go | 16 ++ ee/backend/pkg/db/clickhouse/connector.go | 184 ++++++++++++++++++++ 3 files changed, 200 insertions(+), 5 deletions(-) diff --git a/backend/internal/db/datasaver/saver.go b/backend/internal/db/datasaver/saver.go index 386c2c1ee..72ffa380e 100644 --- a/backend/internal/db/datasaver/saver.go +++ b/backend/internal/db/datasaver/saver.go @@ -109,11 +109,6 @@ func (s *saverImpl) handleMobileMessage(msg Message) error { return err } return s.sessions.UpdateIssuesStats(session.SessionID, 1, 1000) - case *IOSIssueEvent: - if err = s.pg.InsertIOSIssueEvent(session, m); err != nil { - return err - } - return s.sessions.UpdateIssuesStats(session.SessionID, 0, postgres.GetIssueScore(m.Type)) } return nil } diff --git a/ee/backend/internal/db/datasaver/methods.go b/ee/backend/internal/db/datasaver/methods.go index 29f2e1c81..3f61b7a3e 100644 --- a/ee/backend/internal/db/datasaver/methods.go +++ b/ee/backend/internal/db/datasaver/methods.go @@ -73,6 +73,22 @@ func (s *saverImpl) handleExtraMessage(msg messages.Message) error { return s.ch.InsertWebInputDuration(session, m) case *messages.MouseThrashing: return s.ch.InsertMouseThrashing(session, m) + + // Mobile messages + case *messages.IOSSessionEnd: + return s.ch.InsertMobileSession(session) + case *messages.IOSCustomEvent: + return s.ch.InsertMobileCustom(session, m) + case *messages.IOSClickEvent: + return s.ch.InsertMobileClick(session, m) + case *messages.IOSSwipeEvent: + return s.ch.InsertMobileSwipe(session, m) + case *messages.IOSInputEvent: + return s.ch.InsertMobileInput(session, m) + case *messages.IOSNetworkCall: + return s.ch.InsertMobileRequest(session, m, session.SaveRequestPayload) + case *messages.IOSCrash: + return s.ch.InsertMobileCrash(session, m) } return nil } diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index c4bc8449d..538ea805a 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -22,6 +22,7 @@ type Connector interface { Prepare() error Commit() error Stop() error + // Web InsertWebSession(session *sessions.Session) error InsertWebResourceEvent(session *sessions.Session, msg *messages.ResourceTiming) error InsertWebPageEvent(session *sessions.Session, msg *messages.PageEvent) error @@ -35,6 +36,14 @@ type Connector interface { InsertIssue(session *sessions.Session, msg *messages.IssueEvent) error InsertWebInputDuration(session *sessions.Session, msg *messages.InputChange) error InsertMouseThrashing(session *sessions.Session, msg *messages.MouseThrashing) error + // Mobile + InsertMobileSession(session *sessions.Session) error + InsertMobileCustom(session *sessions.Session, msg *messages.IOSCustomEvent) error + InsertMobileClick(session *sessions.Session, msg *messages.IOSClickEvent) error + InsertMobileSwipe(session *sessions.Session, msg *messages.IOSSwipeEvent) error + InsertMobileInput(session *sessions.Session, msg *messages.IOSInputEvent) error + InsertMobileRequest(session *sessions.Session, msg *messages.IOSNetworkCall, savePayload bool) error + InsertMobileCrash(session *sessions.Session, msg *messages.IOSCrash) error } type task struct { @@ -105,6 +114,7 @@ func (c *connectorImpl) newBatch(name, query string) error { } var batches = map[string]string{ + // Web "sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, timezone) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?)", "resources": "INSERT INTO experimental.resources (session_id, project_id, message_id, datetime, url, type, duration, ttfb, header_size, encoded_body_size, decoded_body_size, success) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?)", "autocompletes": "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, ?)", @@ -118,6 +128,14 @@ var batches = map[string]string{ "graphql": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, name, request_body, response_body, event_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "issuesEvents": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, issue_id, issue_type, event_type, url) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "issues": "INSERT INTO experimental.issues (project_id, issue_id, type, context_string) VALUES (?, ?, ?, ?)", + //Mobile + "ios_sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, platform, timezone) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?)", + "ios_custom": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, name, payload, event_type) VALUES (?, ?, ?, ?, ?, ?, ?)", + "ios_clicks": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, label, event_type) VALUES (?, ?, ?, ?, ?, ?)", + "ios_swipes": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, label, direction, event_type) VALUES (?, ?, ?, ?, ?, ?, ?)", + "ios_inputs": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, label, event_type) VALUES (?, ?, ?, ?, ?, ?)", + "ios_requests": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, url, request_body, response_body, status, method, duration, success, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?)", + "ios_crashes": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, name, reason, stacktrace, event_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", } func (c *connectorImpl) Prepare() error { @@ -514,3 +532,169 @@ func (c *connectorImpl) InsertGraphQL(session *sessions.Session, msg *messages.G } return nil } + +// Mobile events + +func (c *connectorImpl) InsertMobileSession(session *sessions.Session) error { + if session.Duration == nil { + return errors.New("trying to insert mobile session with nil duration") + } + if err := c.batches["ios_sessions"].Append( + session.SessionID, + uint16(session.ProjectID), + session.UserID, + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + session.UserState, + session.UserCity, + datetime(session.Timestamp), + uint32(*session.Duration), + uint16(session.PagesCount), + uint16(session.EventsCount), + uint16(session.ErrorsCount), + uint32(session.IssueScore), + session.Referrer, + session.IssueTypes, + session.TrackerVersion, + session.UserBrowser, + nullableString(session.UserBrowserVersion), + session.Metadata1, + session.Metadata2, + session.Metadata3, + session.Metadata4, + session.Metadata5, + session.Metadata6, + session.Metadata7, + session.Metadata8, + session.Metadata9, + session.Metadata10, + "ios", + session.Timezone, + ); err != nil { + c.checkError("ios_sessions", err) + return fmt.Errorf("can't append to sessions batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertMobileCustom(session *sessions.Session, msg *messages.IOSCustomEvent) error { + if err := c.batches["ios_custom"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.Meta().Index, + datetime(uint64(msg.Meta().Timestamp)), + msg.Name, + msg.Payload, + "CUSTOM", + ); err != nil { + c.checkError("ios_custom", err) + return fmt.Errorf("can't append to mobile custom batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertMobileClick(session *sessions.Session, msg *messages.IOSClickEvent) error { + if msg.Label == "" { + return nil + } + if err := c.batches["ios_clicks"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.MsgID(), + datetime(msg.Timestamp), + msg.Label, + "TAP", + ); err != nil { + c.checkError("ios_clicks", err) + return fmt.Errorf("can't append to mobile clicks batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertMobileSwipe(session *sessions.Session, msg *messages.IOSSwipeEvent) error { + if msg.Label == "" { + return nil + } + if err := c.batches["ios_swipes"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.MsgID(), + datetime(msg.Timestamp), + msg.Label, + nullableString(msg.Direction), + "SWIPE", + ); err != nil { + c.checkError("ios_clicks", err) + return fmt.Errorf("can't append to mobile clicks batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertMobileInput(session *sessions.Session, msg *messages.IOSInputEvent) error { + if msg.Label == "" { + return nil + } + if err := c.batches["ios_inputs"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.MsgID(), + datetime(msg.Timestamp), + msg.Label, + "INPUT", + ); err != nil { + c.checkError("ios_inputs", err) + return fmt.Errorf("can't append to mobile inputs batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertMobileRequest(session *sessions.Session, msg *messages.IOSNetworkCall, savePayload bool) error { + urlMethod := url.EnsureMethod(msg.Method) + if urlMethod == "" { + return fmt.Errorf("can't parse http method. sess: %d, method: %s", session.SessionID, msg.Method) + } + var request, response *string + if savePayload { + request = &msg.Request + response = &msg.Response + } + if err := c.batches["ios_requests"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.Meta().Index, + datetime(uint64(msg.Meta().Timestamp)), + msg.URL, + request, + response, + uint16(msg.Status), + url.EnsureMethod(msg.Method), + uint16(msg.Duration), + msg.Status < 400, + "REQUEST", + ); err != nil { + c.checkError("ios_requests", err) + return fmt.Errorf("can't append to mobile requests batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertMobileCrash(session *sessions.Session, msg *messages.IOSCrash) error { + if err := c.batches["ios_crashes"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.MsgID(), + datetime(msg.Timestamp), + msg.Name, + msg.Reason, + msg.Stacktrace, + "CRASH", + ); err != nil { + c.checkError("ios_crashes", err) + return fmt.Errorf("can't append to mobile crashges batch: %s", err) + } + return nil +}