jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(backend): added utm data to PG (#2152)

Browse source 
* feat(backend): added utm data to PG

* feat(backend): added utm data to CH

* feat(db): removed nullableString() calls
This commit is contained in:
Alexander 2024-05-24 08:33:28 +02:00 • committed by GitHub
parent 60561da3c9
commit 993b6b58ab
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 79 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
backend/internal/db/datasaver/saver.go
View file

@ -180,6 +180,7 @@ func (s *saverImpl) handleMessage(msg Message) error {
return err
}
s.sessions.UpdateReferrer(session.SessionID, m.Referrer)
s.sessions.UpdateUTM(session.SessionID, m.URL)
return s.sessions.UpdateEventsStats(session.SessionID, 1, 1)
case *NetworkRequest:
return s.pg.InsertWebNetworkRequest(session, m)

3
backend/pkg/sessions/model.go
View file

@ -45,6 +45,9 @@ type Session struct {
EncryptionKey string
ScreenWidth int
ScreenHeight int
UtmSource *string
UtmMedium *string
UtmCampaign *string
}
func (s *Session) SetMetadata(keyNo uint, value string) {

16
backend/pkg/sessions/sessions.go
View file

@ -24,6 +24,7 @@ type Sessions interface {
UpdateUserID(sessionID uint64, userID string) error
UpdateAnonymousID(sessionID uint64, userAnonymousID string) error
UpdateReferrer(sessionID uint64, referrer string) error
UpdateUTM(sessionID uint64, url string) error
UpdateMetadata(sessionID uint64, key, value string) error
UpdateEventsStats(sessionID uint64, events, pages int) error
UpdateIssuesStats(sessionID uint64, errors, issueScore int) error
@ -211,6 +212,21 @@ func (s *sessionsImpl) UpdateReferrer(sessionID uint64, referrer string) error {
return nil
}
func (s *sessionsImpl) UpdateUTM(sessionID uint64, pageUrl string) error {
params, err := url.GetURLQueryParams(pageUrl)
if err != nil {
return err
}
utmSource := params["utm_source"]
utmMedium := params["utm_medium"]
utmCampaign := params["utm_campaign"]
if utmSource == "" && utmMedium == "" && utmCampaign == "" {
return nil
}
s.updates.SetUTM(sessionID, utmSource, utmMedium, utmCampaign)
return nil
}
// UpdateMetadata usage: in db handler on each metadata event
func (s *sessionsImpl) UpdateMetadata(sessionID uint64, key, value string) error {
session, err := s.Get(sessionID)

6
backend/pkg/sessions/storage.go
View file

@ -106,7 +106,8 @@ func (s *storageImpl) Get(sessionID uint64) (*Session, error) {
pages_count, events_count, errors_count, issue_types,
user_browser, user_browser_version, issue_score,
metadata_1, metadata_2, metadata_3, metadata_4, metadata_5,
metadata_6, metadata_7, metadata_8, metadata_9, metadata_10
metadata_6, metadata_7, metadata_8, metadata_9, metadata_10,
utm_source, utm_medium, utm_campaign
FROM sessions
WHERE session_id=$1
`,
@ -120,7 +121,8 @@ func (s *storageImpl) Get(sessionID uint64) (*Session, error) {
&sess.PagesCount, &sess.EventsCount, &sess.ErrorsCount, &issueTypes,
&userBrowser, &userBrowserVersion, &sess.IssueScore,
&sess.Metadata1, &sess.Metadata2, &sess.Metadata3, &sess.Metadata4, &sess.Metadata5,
&sess.Metadata6, &sess.Metadata7, &sess.Metadata8, &sess.Metadata9, &sess.Metadata10); err != nil {
&sess.Metadata6, &sess.Metadata7, &sess.Metadata8, &sess.Metadata9, &sess.Metadata10,
&sess.UtmSource, &sess.UtmMedium, &sess.UtmCampaign); err != nil {
return nil, err
}
if userOSVersion != nil {

38
backend/pkg/sessions/updates.go
View file

@ -17,6 +17,7 @@ type Updates interface {
AddAnonID(sessionID uint64, userID string)
SetReferrer(sessionID uint64, referrer, baseReferrer string)
SetMetadata(sessionID uint64, keyNo uint, value string)
SetUTM(sessionID uint64, utmSource, utmMedium, utmCampaign string)
AddEvents(sessionID uint64, events, pages int)
AddIssues(sessionID uint64, errors, issues int)
Commit()
@ -64,6 +65,13 @@ func (u *updatesImpl) SetMetadata(sessionID uint64, keyNo uint, value string) {
u.updates[sessionID].setMetadata(keyNo, value)
}
func (u *updatesImpl) SetUTM(sessionID uint64, utmSource, utmMedium, utmCampaign string) {
if u.updates[sessionID] == nil {
u.updates[sessionID] = NewSessionUpdate(sessionID)
}
u.updates[sessionID].setUTM(utmSource, utmMedium, utmCampaign)
}
func (u *updatesImpl) AddEvents(sessionID uint64, events, pages int) {
if u.updates[sessionID] == nil {
u.updates[sessionID] = NewSessionUpdate(sessionID)
@ -128,6 +136,9 @@ type sessionUpdate struct {
events int
errors int
issues int
utmSource *string
utmMedium *string
utmCampaign *string
}
func NewSessionUpdate(sessionID uint64) *sessionUpdate {
@ -158,6 +169,18 @@ func (su *sessionUpdate) setMetadata(keyNo uint, value string) {
su.metadata[keyNo] = value
}
func (su *sessionUpdate) setUTM(utmSource, utmMedium, utmCampaign string) {
if utmSource != "" {
su.utmSource = &utmSource
}
if utmMedium != "" {
su.utmMedium = &utmMedium
}
if utmCampaign != "" {
su.utmCampaign = &utmCampaign
}
}
func (su *sessionUpdate) addEvents(events, pages int) {
su.events += events
su.pages += pages
@ -213,6 +236,21 @@ func (su *sessionUpdate) request() (string, []interface{}) {
sqlReq += fmt.Sprintf(" issue_score = issue_score + $%d,", varsCounter)
sqlArgs = append(sqlArgs, su.issues)
}
if su.utmSource != nil {
varsCounter++
sqlReq += fmt.Sprintf(" utm_source = LEFT($%d, 8000),", varsCounter)
sqlArgs = append(sqlArgs, *su.utmSource)
}
if su.utmMedium != nil {
varsCounter++
sqlReq += fmt.Sprintf(" utm_medium = LEFT($%d, 8000),", varsCounter)
sqlArgs = append(sqlArgs, *su.utmMedium)
}
if su.utmCampaign != nil {
varsCounter++
sqlReq += fmt.Sprintf(" utm_campaign = LEFT($%d, 8000),", varsCounter)
sqlArgs = append(sqlArgs, *su.utmCampaign)
}
if varsCounter == 0 {
return "", nil

13
backend/pkg/url/url.go
View file

@ -23,6 +23,19 @@ func GetURLParts(rawURL string) (string, string, string, error) {
return u.Host, path, u.RawQuery, nil
}
func GetURLQueryParams(rawURL string) (map[string]string, error) {
rawURL = strings.Replace(rawURL, "\t", "", -1)
u, err := url.Parse(rawURL)
if err != nil {
return nil, err
}
params := make(map[string]string)
for key, values := range u.Query() {
params[key] = values[0]
}
return params, nil
}
func getURLExtension(URL string) string {
u, err := url.Parse(URL)
if err != nil {

5
ee/backend/pkg/db/clickhouse/connector.go
View file

@ -115,7 +115,7 @@ func (c *connectorImpl) newBatch(name, query string) error {
var batches = map[string]string{
// Web
"sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, timezone) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?)",
"sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, timezone, utm_source, utm_medium, utm_campaign) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?, ?, ?)",
"resources": "INSERT INTO experimental.resources (session_id, project_id, message_id, datetime, url, type, duration, ttfb, header_size, encoded_body_size, decoded_body_size, success) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?)",
"autocompletes": "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, ?)",
"pages": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint_time, speed_index, visually_complete, time_to_interactive, event_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
@ -315,6 +315,9 @@ func (c *connectorImpl) InsertWebSession(session *sessions.Session) error {
session.Metadata9,
session.Metadata10,
session.Timezone,
session.UtmSource,
session.UtmMedium,
session.UtmCampaign,
); err != nil {
c.checkError("sessions", err)
return fmt.Errorf("can't append to sessions batch: %s", err)