jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(backend): insert url parts for pages and requests

Browse source
This commit is contained in:
ShiKhu 2022-04-13 14:26:39 +02:00
parent 9a33a7c923
commit 3bc9bd8df4
3 changed files with 37 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/pkg/db/postgres/messages-web-stats.go
View file

@ -35,7 +35,7 @@ func (conn *Conn) InsertWebStatsPerformance(sessionID uint64, p *PerformanceTrac
} }
func (conn *Conn) InsertWebStatsResourceEvent(sessionID uint64, e *ResourceEvent) error { func (conn *Conn) InsertWebStatsResourceEvent(sessionID uint64, e *ResourceEvent) error {
host, _, err := url.GetURLParts(e.URL) host, _, _, err := url.GetURLParts(e.URL)
if err != nil { if err != nil {
return err return err
} }

44
backend/pkg/db/postgres/messages-web.go
View file

@ -55,7 +55,7 @@ func (conn *Conn) InsertWebUserAnonymousID(sessionID uint64, userAnonymousID *Us
// TODO: fix column "dom_content_loaded_event_end" of relation "pages" // TODO: fix column "dom_content_loaded_event_end" of relation "pages"
func (conn *Conn) InsertWebPageEvent(sessionID uint64, e *PageEvent) error { func (conn *Conn) InsertWebPageEvent(sessionID uint64, e *PageEvent) error {
host, path, err := url.GetURLParts(e.URL) host, path, query, err := url.GetURLParts(e.URL)
if err != nil { if err != nil {
return err return err
} }
@ -64,20 +64,27 @@ func (conn *Conn) InsertWebPageEvent(sessionID uint64, e *PageEvent) error {
return err return err
} }
defer tx.rollback() defer tx.rollback()
// base_path is depricated
if err := tx.exec(` if err := tx.exec(`
INSERT INTO events.pages ( INSERT INTO events.pages (
session_id, message_id, timestamp, referrer, base_referrer, host, path, base_path, session_id, message_id, timestamp, referrer, base_referrer, host, path, query,
dom_content_loaded_time, load_time, response_end, first_paint_time, first_contentful_paint_time, dom_content_loaded_time, load_time, response_end, first_paint_time, first_contentful_paint_time,
speed_index, visually_complete, time_to_interactive, speed_index, visually_complete, time_to_interactive,
response_time, dom_building_time response_time, dom_building_time,
base_path
) VALUES ( ) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $1, $2, $3,
$4, $5,
$6, $7, $8,
NULLIF($9, 0), NULLIF($10, 0), NULLIF($11, 0), NULLIF($12, 0), NULLIF($13, 0), NULLIF($9, 0), NULLIF($10, 0), NULLIF($11, 0), NULLIF($12, 0), NULLIF($13, 0),
NULLIF($14, 0), NULLIF($15, 0), NULLIF($16, 0), NULLIF($14, 0), NULLIF($15, 0), NULLIF($16, 0),
NULLIF($17, 0), NULLIF($18, 0) NULLIF($17, 0), NULLIF($18, 0),
'',
) )
`, `,
sessionID, e.MessageID, e.Timestamp, e.Referrer, url.DiscardURLQuery(e.Referrer), host, path, url.DiscardURLQuery(path), sessionID, e.MessageID, e.Timestamp,
e.Referrer, url.DiscardURLQuery(e.Referrer),
host, path, query,
e.DomContentLoadedEventEnd, e.LoadEventEnd, e.ResponseEnd, e.FirstPaint, e.FirstContentfulPaint, e.DomContentLoadedEventEnd, e.LoadEventEnd, e.ResponseEnd, e.FirstPaint, e.FirstContentfulPaint,
e.SpeedIndex, e.VisuallyComplete, e.TimeToInteractive, e.SpeedIndex, e.VisuallyComplete, e.TimeToInteractive,
calcResponseTime(e), calcDomBuildingTime(e), calcResponseTime(e), calcDomBuildingTime(e),
@ -109,7 +116,7 @@ func (conn *Conn) InsertWebClickEvent(sessionID uint64, e *ClickEvent) error {
INSERT INTO events.clicks INSERT INTO events.clicks
(session_id, message_id, timestamp, label, selector, url) (session_id, message_id, timestamp, label, selector, url)
(SELECT (SELECT
$1, $2, $3, NULLIF($4, ''), $5, host || base_path $1, $2, $3, NULLIF($4, ''), $5, host || path
FROM events.pages FROM events.pages
WHERE session_id = $1 AND timestamp <= $3 ORDER BY timestamp DESC LIMIT 1 WHERE session_id = $1 AND timestamp <= $3 ORDER BY timestamp DESC LIMIT 1
) )
@ -211,19 +218,26 @@ func (conn *Conn) InsertWebFetchEvent(sessionID uint64, savePayload bool, e *Fet
response = &e.Response response = &e.Response
} }
conn.insertAutocompleteValue(sessionID, "REQUEST", url.DiscardURLQuery(e.URL)) conn.insertAutocompleteValue(sessionID, "REQUEST", url.DiscardURLQuery(e.URL))
host, path, query, err := url.GetURLParts(e.URL)
if err != nil {
return err
}
return conn.batchQueue(sessionID, ` return conn.batchQueue(sessionID, `
INSERT INTO events_common.requests ( INSERT INTO events_common.requests (
session_id, timestamp, session_id, timestamp, seq_index,
seq_index, url, duration, success, url, host, path, query,
request_body, response_body, status_code, method request_body, response_body, status_code, method,
duration, success,
) VALUES ( ) VALUES (
$1, $2, $1, $2, $3,
$3, $4, $5, $6, $4, $5, $6, $7
$7, $8, $9::smallint, NULLIF($10, '')::http_method $8, $9, $10::smallint, NULLIF($11, '')::http_method,
$12, $13
) ON CONFLICT DO NOTHING`, ) ON CONFLICT DO NOTHING`,
sessionID, e.Timestamp, sessionID, e.Timestamp, getSqIdx(e.MessageID),
getSqIdx(e.MessageID), e.URL, e.Duration, e.Status < 400, e.URL, host, path, query,
request, response, e.Status, url.EnsureMethod(e.Method), request, response, e.Status, url.EnsureMethod(e.Method),
e.Duration, e.Status < 400,
) )
} }

13
backend/pkg/url/url.go
View file

@ -1,18 +1,19 @@
package url package url
import ( import (
"strings"
_url "net/url" _url "net/url"
"strings"
) )
func DiscardURLQuery(url string) string { func DiscardURLQuery(url string) string {
return strings.Split(url, "?")[0] return strings.Split(url, "?")[0]
} }
func GetURLParts(rawURL string) (string, string, error) { func GetURLParts(rawURL string) (string, string, string, error) {
u, err := _url.Parse(rawURL) u, err := _url.Parse(rawURL)
if err != nil { if err != nil {
return "", "", err return "", "", "", err
} }
return u.Host, u.RequestURI(), nil // u.Scheme ?
} return u.Host, u.RawPath, u.RawQuery, nil
}