From 993b6b58ab137689390ae0d4d88348e6867944ea Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 24 May 2024 08:33:28 +0200 Subject: [PATCH] feat(backend): added utm data to PG (#2152) * feat(backend): added utm data to PG * feat(backend): added utm data to CH * feat(db): removed nullableString() calls --- backend/internal/db/datasaver/saver.go | 1 + backend/pkg/sessions/model.go | 3 ++ backend/pkg/sessions/sessions.go | 16 ++++++++++ backend/pkg/sessions/storage.go | 6 ++-- backend/pkg/sessions/updates.go | 38 +++++++++++++++++++++++ backend/pkg/url/url.go | 13 ++++++++ ee/backend/pkg/db/clickhouse/connector.go | 5 ++- 7 files changed, 79 insertions(+), 3 deletions(-) diff --git a/backend/internal/db/datasaver/saver.go b/backend/internal/db/datasaver/saver.go index ab5e703c9..fe153b558 100644 --- a/backend/internal/db/datasaver/saver.go +++ b/backend/internal/db/datasaver/saver.go @@ -180,6 +180,7 @@ func (s *saverImpl) handleMessage(msg Message) error { return err } s.sessions.UpdateReferrer(session.SessionID, m.Referrer) + s.sessions.UpdateUTM(session.SessionID, m.URL) return s.sessions.UpdateEventsStats(session.SessionID, 1, 1) case *NetworkRequest: return s.pg.InsertWebNetworkRequest(session, m) diff --git a/backend/pkg/sessions/model.go b/backend/pkg/sessions/model.go index 02e511835..75c2d9ca6 100644 --- a/backend/pkg/sessions/model.go +++ b/backend/pkg/sessions/model.go @@ -45,6 +45,9 @@ type Session struct { EncryptionKey string ScreenWidth int ScreenHeight int + UtmSource *string + UtmMedium *string + UtmCampaign *string } func (s *Session) SetMetadata(keyNo uint, value string) { diff --git a/backend/pkg/sessions/sessions.go b/backend/pkg/sessions/sessions.go index 546e0e34d..446fd1b1f 100644 --- a/backend/pkg/sessions/sessions.go +++ b/backend/pkg/sessions/sessions.go @@ -24,6 +24,7 @@ type Sessions interface { UpdateUserID(sessionID uint64, userID string) error UpdateAnonymousID(sessionID uint64, userAnonymousID string) error UpdateReferrer(sessionID uint64, referrer string) error + UpdateUTM(sessionID uint64, url string) error UpdateMetadata(sessionID uint64, key, value string) error UpdateEventsStats(sessionID uint64, events, pages int) error UpdateIssuesStats(sessionID uint64, errors, issueScore int) error @@ -211,6 +212,21 @@ func (s *sessionsImpl) UpdateReferrer(sessionID uint64, referrer string) error { return nil } +func (s *sessionsImpl) UpdateUTM(sessionID uint64, pageUrl string) error { + params, err := url.GetURLQueryParams(pageUrl) + if err != nil { + return err + } + utmSource := params["utm_source"] + utmMedium := params["utm_medium"] + utmCampaign := params["utm_campaign"] + if utmSource == "" && utmMedium == "" && utmCampaign == "" { + return nil + } + s.updates.SetUTM(sessionID, utmSource, utmMedium, utmCampaign) + return nil +} + // UpdateMetadata usage: in db handler on each metadata event func (s *sessionsImpl) UpdateMetadata(sessionID uint64, key, value string) error { session, err := s.Get(sessionID) diff --git a/backend/pkg/sessions/storage.go b/backend/pkg/sessions/storage.go index e3028b9bc..7668ba075 100644 --- a/backend/pkg/sessions/storage.go +++ b/backend/pkg/sessions/storage.go @@ -106,7 +106,8 @@ func (s *storageImpl) Get(sessionID uint64) (*Session, error) { pages_count, events_count, errors_count, issue_types, user_browser, user_browser_version, issue_score, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, - metadata_6, metadata_7, metadata_8, metadata_9, metadata_10 + metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, + utm_source, utm_medium, utm_campaign FROM sessions WHERE session_id=$1 `, @@ -120,7 +121,8 @@ func (s *storageImpl) Get(sessionID uint64) (*Session, error) { &sess.PagesCount, &sess.EventsCount, &sess.ErrorsCount, &issueTypes, &userBrowser, &userBrowserVersion, &sess.IssueScore, &sess.Metadata1, &sess.Metadata2, &sess.Metadata3, &sess.Metadata4, &sess.Metadata5, - &sess.Metadata6, &sess.Metadata7, &sess.Metadata8, &sess.Metadata9, &sess.Metadata10); err != nil { + &sess.Metadata6, &sess.Metadata7, &sess.Metadata8, &sess.Metadata9, &sess.Metadata10, + &sess.UtmSource, &sess.UtmMedium, &sess.UtmCampaign); err != nil { return nil, err } if userOSVersion != nil { diff --git a/backend/pkg/sessions/updates.go b/backend/pkg/sessions/updates.go index bf217d05f..848d9218c 100644 --- a/backend/pkg/sessions/updates.go +++ b/backend/pkg/sessions/updates.go @@ -17,6 +17,7 @@ type Updates interface { AddAnonID(sessionID uint64, userID string) SetReferrer(sessionID uint64, referrer, baseReferrer string) SetMetadata(sessionID uint64, keyNo uint, value string) + SetUTM(sessionID uint64, utmSource, utmMedium, utmCampaign string) AddEvents(sessionID uint64, events, pages int) AddIssues(sessionID uint64, errors, issues int) Commit() @@ -64,6 +65,13 @@ func (u *updatesImpl) SetMetadata(sessionID uint64, keyNo uint, value string) { u.updates[sessionID].setMetadata(keyNo, value) } +func (u *updatesImpl) SetUTM(sessionID uint64, utmSource, utmMedium, utmCampaign string) { + if u.updates[sessionID] == nil { + u.updates[sessionID] = NewSessionUpdate(sessionID) + } + u.updates[sessionID].setUTM(utmSource, utmMedium, utmCampaign) +} + func (u *updatesImpl) AddEvents(sessionID uint64, events, pages int) { if u.updates[sessionID] == nil { u.updates[sessionID] = NewSessionUpdate(sessionID) @@ -128,6 +136,9 @@ type sessionUpdate struct { events int errors int issues int + utmSource *string + utmMedium *string + utmCampaign *string } func NewSessionUpdate(sessionID uint64) *sessionUpdate { @@ -158,6 +169,18 @@ func (su *sessionUpdate) setMetadata(keyNo uint, value string) { su.metadata[keyNo] = value } +func (su *sessionUpdate) setUTM(utmSource, utmMedium, utmCampaign string) { + if utmSource != "" { + su.utmSource = &utmSource + } + if utmMedium != "" { + su.utmMedium = &utmMedium + } + if utmCampaign != "" { + su.utmCampaign = &utmCampaign + } +} + func (su *sessionUpdate) addEvents(events, pages int) { su.events += events su.pages += pages @@ -213,6 +236,21 @@ func (su *sessionUpdate) request() (string, []interface{}) { sqlReq += fmt.Sprintf(" issue_score = issue_score + $%d,", varsCounter) sqlArgs = append(sqlArgs, su.issues) } + if su.utmSource != nil { + varsCounter++ + sqlReq += fmt.Sprintf(" utm_source = LEFT($%d, 8000),", varsCounter) + sqlArgs = append(sqlArgs, *su.utmSource) + } + if su.utmMedium != nil { + varsCounter++ + sqlReq += fmt.Sprintf(" utm_medium = LEFT($%d, 8000),", varsCounter) + sqlArgs = append(sqlArgs, *su.utmMedium) + } + if su.utmCampaign != nil { + varsCounter++ + sqlReq += fmt.Sprintf(" utm_campaign = LEFT($%d, 8000),", varsCounter) + sqlArgs = append(sqlArgs, *su.utmCampaign) + } if varsCounter == 0 { return "", nil diff --git a/backend/pkg/url/url.go b/backend/pkg/url/url.go index 30b3f2a7e..3f8620497 100644 --- a/backend/pkg/url/url.go +++ b/backend/pkg/url/url.go @@ -23,6 +23,19 @@ func GetURLParts(rawURL string) (string, string, string, error) { return u.Host, path, u.RawQuery, nil } +func GetURLQueryParams(rawURL string) (map[string]string, error) { + rawURL = strings.Replace(rawURL, "\t", "", -1) + u, err := url.Parse(rawURL) + if err != nil { + return nil, err + } + params := make(map[string]string) + for key, values := range u.Query() { + params[key] = values[0] + } + return params, nil +} + func getURLExtension(URL string) string { u, err := url.Parse(URL) if err != nil { diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index 344d1b9c0..886802765 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -115,7 +115,7 @@ func (c *connectorImpl) newBatch(name, query string) error { var batches = map[string]string{ // Web - "sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, timezone) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?)", + "sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, timezone, utm_source, utm_medium, utm_campaign) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?, ?, ?)", "resources": "INSERT INTO experimental.resources (session_id, project_id, message_id, datetime, url, type, duration, ttfb, header_size, encoded_body_size, decoded_body_size, success) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?)", "autocompletes": "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, ?)", "pages": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint_time, speed_index, visually_complete, time_to_interactive, event_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", @@ -315,6 +315,9 @@ func (c *connectorImpl) InsertWebSession(session *sessions.Session) error { session.Metadata9, session.Metadata10, session.Timezone, + session.UtmSource, + session.UtmMedium, + session.UtmCampaign, ); err != nil { c.checkError("sessions", err) return fmt.Errorf("can't append to sessions batch: %s", err)