diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index d22b7b363..51fa99122 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -62,7 +62,7 @@ func main() { messages.MsgUserID, messages.MsgUserAnonymousID, messages.MsgClickEvent, messages.MsgIntegrationEvent, messages.MsgPerformanceTrackAggr, messages.MsgJSException, messages.MsgResourceTiming, - messages.MsgCustomEvent, messages.MsgCustomIssue, messages.MsgFetch, messages.MsgGraphQL, + messages.MsgCustomEvent, messages.MsgCustomIssue, messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL, messages.MsgStateAction, messages.MsgSetInputTarget, messages.MsgSetInputValue, messages.MsgCreateDocument, messages.MsgMouseClick, messages.MsgSetPageLocation, messages.MsgPageLoadTiming, messages.MsgPageRenderTiming} diff --git a/backend/internal/db/datasaver/fts.go b/backend/internal/db/datasaver/fts.go index afdb3ca39..6b6bdbbae 100644 --- a/backend/internal/db/datasaver/fts.go +++ b/backend/internal/db/datasaver/fts.go @@ -6,7 +6,7 @@ import ( "openreplay/backend/pkg/messages" ) -type FetchFTS struct { +type NetworkRequestFTS struct { Method string `json:"method"` URL string `json:"url"` Request string `json:"request"` @@ -56,8 +56,8 @@ func (s *Saver) sendToFTS(msg messages.Message, sessionID uint64) { switch m := msg.(type) { // Common - case *messages.Fetch: - event, err = json.Marshal(FetchFTS{ + case *messages.NetworkRequest: + event, err = json.Marshal(NetworkRequestFTS{ Method: m.Method, URL: m.URL, Request: m.Request, diff --git a/backend/internal/db/datasaver/messages.go b/backend/internal/db/datasaver/messages.go index b07eb7c0d..834db33ed 100644 --- a/backend/internal/db/datasaver/messages.go +++ b/backend/internal/db/datasaver/messages.go @@ -38,9 +38,9 @@ func (mi *Saver) InsertMessage(msg Message) error { case *PageEvent: mi.sendToFTS(msg, sessionID) return mi.pg.InsertWebPageEvent(sessionID, m) - case *Fetch: + case *NetworkRequest: mi.sendToFTS(msg, sessionID) - return mi.pg.InsertWebFetch(sessionID, m) + return mi.pg.InsertWebNetworkRequest(sessionID, m) case *GraphQL: mi.sendToFTS(msg, sessionID) return mi.pg.InsertWebGraphQL(sessionID, m) diff --git a/backend/pkg/db/cache/messages-web.go b/backend/pkg/db/cache/messages-web.go index 55588c245..1df3d1520 100644 --- a/backend/pkg/db/cache/messages-web.go +++ b/backend/pkg/db/cache/messages-web.go @@ -99,7 +99,7 @@ func (c *PGCache) InsertSessionReferrer(sessionID uint64, referrer string) error return c.Conn.InsertSessionReferrer(sessionID, referrer) } -func (c *PGCache) InsertWebFetch(sessionID uint64, e *Fetch) error { +func (c *PGCache) InsertWebNetworkRequest(sessionID uint64, e *NetworkRequest) error { session, err := c.Cache.GetSession(sessionID) if err != nil { return err @@ -108,7 +108,7 @@ func (c *PGCache) InsertWebFetch(sessionID uint64, e *Fetch) error { if err != nil { return err } - return c.Conn.InsertWebFetch(sessionID, session.ProjectID, project.SaveRequestPayloads, e) + return c.Conn.InsertWebNetworkRequest(sessionID, session.ProjectID, project.SaveRequestPayloads, e) } func (c *PGCache) InsertWebGraphQL(sessionID uint64, e *GraphQL) error { diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index 10c953fcb..e0dfa29a8 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -148,7 +148,7 @@ func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *typ return } -func (conn *Conn) InsertWebFetch(sessionID uint64, projectID uint32, savePayload bool, e *Fetch) error { +func (conn *Conn) InsertWebNetworkRequest(sessionID uint64, projectID uint32, savePayload bool, e *NetworkRequest) error { var request, response *string if savePayload { request = &e.Request diff --git a/backend/pkg/handlers/web/networkIssue.go b/backend/pkg/handlers/web/networkIssue.go index 67522c850..5e7119c20 100644 --- a/backend/pkg/handlers/web/networkIssue.go +++ b/backend/pkg/handlers/web/networkIssue.go @@ -7,7 +7,7 @@ import ( /* Handler name: NetworkIssue Input events: ResourceTiming, - Fetch + NetworkRequest Output event: IssueEvent */ @@ -33,7 +33,7 @@ func (f *NetworkIssueDetector) Handle(message Message, messageID uint64, timesta // ContextString: msg.URL, // } // } - case *Fetch: + case *NetworkRequest: if msg.Status >= 400 { return &IssueEvent{ Type: "bad_request", diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index ee5a08d8e..6c96383c9 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -2,7 +2,7 @@ package messages func IsReplayerType(id int) bool { - return 80 != id && 81 != id && 82 != id && 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 53 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id + return 80 != id && 81 != id && 82 != id && 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id } func IsIOSType(id int) bool { diff --git a/backend/pkg/messages/legacy-message-transform.go b/backend/pkg/messages/legacy-message-transform.go index 5e2bd3ed7..72757baf4 100644 --- a/backend/pkg/messages/legacy-message-transform.go +++ b/backend/pkg/messages/legacy-message-transform.go @@ -14,6 +14,17 @@ func transformDeprecated(msg Message) Message { Timestamp: m.Timestamp, EncryptionKey: "", } + case *Fetch: + return &NetworkRequest{ + Type: "fetch", + Method: m.Method, + URL: m.URL, + Request: m.Request, + Response: m.Response, + Status: m.Status, + Timestamp: m.Timestamp, + Duration: m.Duration, + } } return msg } diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index c8a0c620e..418d34867 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -25,6 +25,7 @@ const ( MsgSetInputValue = 18 MsgSetInputChecked = 19 MsgMouseMove = 20 + MsgNetworkRequest = 21 MsgConsoleLog = 22 MsgPageLoadTiming = 23 MsgPageRenderTiming = 24 @@ -673,6 +674,41 @@ func (msg *MouseMove) TypeID() int { return 20 } +type NetworkRequest struct { + message + Type string + Method string + URL string + Request string + Response string + Status uint64 + Timestamp uint64 + Duration uint64 +} + +func (msg *NetworkRequest) Encode() []byte { + buf := make([]byte, 81+len(msg.Type)+len(msg.Method)+len(msg.URL)+len(msg.Request)+len(msg.Response)) + buf[0] = 21 + p := 1 + p = WriteString(msg.Type, buf, p) + p = WriteString(msg.Method, buf, p) + p = WriteString(msg.URL, buf, p) + p = WriteString(msg.Request, buf, p) + p = WriteString(msg.Response, buf, p) + p = WriteUint(msg.Status, buf, p) + p = WriteUint(msg.Timestamp, buf, p) + p = WriteUint(msg.Duration, buf, p) + return buf[:p] +} + +func (msg *NetworkRequest) Decode() Message { + return msg +} + +func (msg *NetworkRequest) TypeID() int { + return 21 +} + type ConsoleLog struct { message Level string diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 6e84d7cfc..29dddb02f 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -348,6 +348,36 @@ func DecodeMouseMove(reader BytesReader) (Message, error) { return msg, err } +func DecodeNetworkRequest(reader BytesReader) (Message, error) { + var err error = nil + msg := &NetworkRequest{} + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Method, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.URL, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Request, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Response, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Status, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Duration, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err +} + func DecodeConsoleLog(reader BytesReader) (Message, error) { var err error = nil msg := &ConsoleLog{} @@ -1710,6 +1740,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeSetInputChecked(reader) case 20: return DecodeMouseMove(reader) + case 21: + return DecodeNetworkRequest(reader) case 22: return DecodeConsoleLog(reader) case 23: diff --git a/ee/backend/internal/db/datasaver/messages.go b/ee/backend/internal/db/datasaver/messages.go index 30aa2a368..7f2863d0b 100644 --- a/ee/backend/internal/db/datasaver/messages.go +++ b/ee/backend/internal/db/datasaver/messages.go @@ -58,7 +58,7 @@ func (mi *Saver) InsertMessage(msg Message) error { return mi.pg.InsertWebJSException(m) case *IntegrationEvent: return mi.pg.InsertWebIntegrationEvent(m) - case *Fetch: + case *NetworkRequest: session, err := mi.pg.GetSession(sessionID) if err != nil { log.Printf("can't get session info for CH: %s", err) @@ -72,7 +72,7 @@ func (mi *Saver) InsertMessage(msg Message) error { } } } - return mi.pg.InsertWebFetch(sessionID, m) + return mi.pg.InsertWebNetworkRequest(sessionID, m) case *GraphQL: session, err := mi.pg.GetSession(sessionID) if err != nil { diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index 7fa78897a..de94fedc2 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -27,7 +27,7 @@ type Connector interface { InsertWebErrorEvent(session *types.Session, msg *types.ErrorEvent) error InsertWebPerformanceTrackAggr(session *types.Session, msg *messages.PerformanceTrackAggr) error InsertAutocomplete(session *types.Session, msgType, msgValue string) error - InsertRequest(session *types.Session, msg *messages.Fetch, savePayload bool) error + InsertRequest(session *types.Session, msg *messages.NetworkRequest, savePayload bool) error InsertCustom(session *types.Session, msg *messages.CustomEvent) error InsertGraphQL(session *types.Session, msg *messages.GraphQL) error InsertIssue(session *types.Session, msg *messages.IssueEvent) error @@ -352,7 +352,7 @@ func (c *connectorImpl) InsertAutocomplete(session *types.Session, msgType, msgV return nil } -func (c *connectorImpl) InsertRequest(session *types.Session, msg *messages.Fetch, savePayload bool) error { +func (c *connectorImpl) InsertRequest(session *types.Session, msg *messages.NetworkRequest, savePayload bool) error { urlMethod := url.EnsureMethod(msg.Method) if urlMethod == "" { return fmt.Errorf("can't parse http method. sess: %d, method: %s", session.SessionID, msg.Method) diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index cf426cb71..313fdf12c 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -213,6 +213,20 @@ class MouseMove(Message): self.y = y +class NetworkRequest(Message): + __id__ = 21 + + def __init__(self, type, method, url, request, response, status, timestamp, duration): + self.type = type + self.method = method + self.url = url + self.request = request + self.response = response + self.status = status + self.timestamp = timestamp + self.duration = duration + + class ConsoleLog(Message): __id__ = 22 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 249e5cb93..dc63ffa79 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -237,6 +237,18 @@ class MessageCodec(Codec): y=self.read_uint(reader) ) + if message_id == 21: + return NetworkRequest( + type=self.read_string(reader), + method=self.read_string(reader), + url=self.read_string(reader), + request=self.read_string(reader), + response=self.read_string(reader), + status=self.read_uint(reader), + timestamp=self.read_uint(reader), + duration=self.read_uint(reader) + ) + if message_id == 22: return ConsoleLog( level=self.read_string(reader), diff --git a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx index 51b5408e7..f34b3e04e 100644 --- a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx +++ b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx @@ -29,6 +29,7 @@ const OTHER = 'other'; const TYPE_TO_TAB = { [TYPES.XHR]: XHR, + [TYPES.FETCH]: XHR, [TYPES.JS]: JS, [TYPES.CSS]: CSS, [TYPES.IMG]: IMG, @@ -153,9 +154,11 @@ function NetworkPanel({ startedAt }: { startedAt: number }) { const activeIndex = devTools[INDEX_KEY].index; const list = useMemo(() => + // TODO: better merge (with body size info) resourceList.filter(res => !fetchList.some(ft => { - if (res.url !== ft.url) { return false } - if (Math.abs(res.time - ft.time) > 200) { return false } // TODO: find good epsilons + // res.url !== ft.url doesn't work on relative URLs appearing within fetchList (to-fix in player) + if (res.name !== ft.name) { return false } + if (Math.abs(res.time - ft.time) > 150) { return false } // TODO: find good epsilons if (Math.abs(res.duration - ft.duration) > 100) { return false } return true })) diff --git a/frontend/app/components/shared/FetchDetailsModal/components/FetchTabs/FetchTabs.tsx b/frontend/app/components/shared/FetchDetailsModal/components/FetchTabs/FetchTabs.tsx index 837a61ec6..f9ed3a745 100644 --- a/frontend/app/components/shared/FetchDetailsModal/components/FetchTabs/FetchTabs.tsx +++ b/frontend/app/components/shared/FetchDetailsModal/components/FetchTabs/FetchTabs.tsx @@ -1,4 +1,5 @@ import React, { useEffect, useState } from 'react'; +import logger from 'App/logger' import Headers from '../Headers'; import { JSONTree, Tabs, NoContent } from 'UI'; import AnimatedSVG, { ICONS } from 'Shared/AnimatedSVG/AnimatedSVG'; @@ -8,44 +9,67 @@ const REQUEST = 'REQUEST'; const RESPONSE = 'RESPONSE'; const TABS = [HEADERS, REQUEST, RESPONSE].map((tab) => ({ text: tab, key: tab })); -interface Props { - resource: any; +function parseRequestResponse( + r: string, + setHeaders: (hs: Record) => void, + setJSONBody: (body: Object) => void, + setStringBody: (body: string) => void, +) { + try { + let json = JSON.parse(r) + const hs = json.headers + const bd = json.body as string + if (typeof hs === "object") { + setHeaders(hs); + } + if (typeof bd !== 'string') { + throw new Error(`body is not a string`) + } + try { + let jBody = JSON.parse(bd) + if (typeof jBody === "object" && jBody != null) { + setJSONBody(jBody) + } else { + setStringBody(bd) + } + } catch { + setStringBody(bd) + } + } catch(e) { logger.error("Error decoding payload json:", e, r)} } -function FetchTabs(props: Props) { - const { resource } = props; + + +interface Props { + resource: { request: string, response: string }; +} +function FetchTabs({ resource }: Props) { const [activeTab, setActiveTab] = useState(HEADERS); const onTabClick = (tab: string) => setActiveTab(tab); - const [jsonPayload, setJsonPayload] = useState(null); - const [jsonResponse, setJsonResponse] = useState(null); - const [requestHeaders, setRequestHeaders] = useState(null); - const [responseHeaders, setResponseHeaders] = useState(null); + const [jsonRequest, setJsonRequest] = useState(null); + const [jsonResponse, setJsonResponse] = useState(null); + const [stringRequest, setStringRequest] = useState(''); + const [stringResponse, setStringResponse ] = useState(''); + const [requestHeaders, setRequestHeaders] = useState | null>(null); + const [responseHeaders, setResponseHeaders] = useState | null>(null); useEffect(() => { - const { payload, response } = resource; - - try { - let jsonPayload = typeof payload === 'string' ? JSON.parse(payload) : payload; - let requestHeaders = jsonPayload.headers; - jsonPayload.body = - typeof jsonPayload.body === 'string' ? JSON.parse(jsonPayload.body) : jsonPayload.body; - delete jsonPayload.headers; - setJsonPayload(jsonPayload); - setRequestHeaders(requestHeaders); - } catch (e) {} - - try { - let jsonResponse = typeof response === 'string' ? JSON.parse(response) : response; - let responseHeaders = jsonResponse.headers; - jsonResponse.body = - typeof jsonResponse.body === 'string' ? JSON.parse(jsonResponse.body) : jsonResponse.body; - delete jsonResponse.headers; - setJsonResponse(jsonResponse); - setResponseHeaders(responseHeaders); - } catch (e) {} - }, [resource, activeTab]); + const { request, response } = resource; + parseRequestResponse( + request, + setRequestHeaders, + setJsonRequest, + setStringRequest, + ) + parseRequestResponse( + response, + setResponseHeaders, + setJsonResponse, + setStringResponse, + ) + }, [resource]); const renderActiveTab = () => { - const { payload, response } = resource; + const { request, response } = resource; switch (activeTab) { case REQUEST: return ( @@ -57,16 +81,15 @@ function FetchTabs(props: Props) { } size="small" - show={!payload} + show={!jsonRequest && !stringRequest} // animatedIcon="no-results" >
- {jsonPayload === undefined ? ( -
{payload}
- ) : ( - - )} + { jsonRequest + ? + :
{stringRequest}
+ }
@@ -82,16 +105,15 @@ function FetchTabs(props: Props) {
} size="small" - show={!response} + show={!jsonResponse && !stringResponse} // animatedIcon="no-results" >
- {jsonResponse === undefined ? ( -
{response}
- ) : ( - - )} + { jsonResponse + ? + :
{stringResponse}
+ }
diff --git a/frontend/app/components/shared/FetchDetailsModal/components/Headers/Headers.tsx b/frontend/app/components/shared/FetchDetailsModal/components/Headers/Headers.tsx index 78c4a5e64..49bf12676 100644 --- a/frontend/app/components/shared/FetchDetailsModal/components/Headers/Headers.tsx +++ b/frontend/app/components/shared/FetchDetailsModal/components/Headers/Headers.tsx @@ -4,8 +4,8 @@ import stl from './headers.module.css'; import AnimatedSVG, { ICONS } from 'Shared/AnimatedSVG/AnimatedSVG'; interface Props { - requestHeaders: any; - responseHeaders: any; + requestHeaders: Record + responseHeaders: Record } function Headers(props: Props) { return ( @@ -21,7 +21,7 @@ function Headers(props: Props) { show={!props.requestHeaders && !props.responseHeaders} // animatedIcon="no-results" > - {props.requestHeaders && ( + {props.requestHeaders && Object.values(props.requestHeaders).length > 0 && ( <>
Request Headers
@@ -36,7 +36,7 @@ function Headers(props: Props) { )} - {props.responseHeaders && ( + {props.responseHeaders && Object.values(props.responseHeaders).length > 0 && (
Response Headers
{Object.keys(props.responseHeaders).map((h) => ( diff --git a/frontend/app/player/common/ListWalker.ts b/frontend/app/player/common/ListWalker.ts index 7d92fa285..43cf88ea9 100644 --- a/frontend/app/player/common/ListWalker.ts +++ b/frontend/app/player/common/ListWalker.ts @@ -12,6 +12,15 @@ export default class ListWalker { this.list.push(m); } + insert(m: T): void { + let index = this.list.findIndex(om => om.time > m.time) + if (index === -1) { + index = this.length + } + const oldList = this.list + this._list = [...oldList.slice(0, index), m, ...oldList.slice(index)] + } + reset(): void { this.p = 0 } diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index 8e4543489..e548f406e 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -2,7 +2,7 @@ import { Decoder } from "syncod"; import logger from 'App/logger'; -import Resource, { TYPES } from 'Types/session/resource'; +import Resource, { TYPES as RES_TYPES } from 'Types/session/resource'; import { TYPES as EVENT_TYPES } from 'Types/session/event'; import { Log } from './types'; @@ -187,13 +187,12 @@ export default class MessageManager { const stateToUpdate : Partial= { performanceChartData: this.performanceTrackManager.chartData, performanceAvaliability: this.performanceTrackManager.avaliability, - ...this.lists.getFullListsState() + ...this.lists.getFullListsState(), } if (this.activityManager) { this.activityManager.end() stateToUpdate.skipIntervals = this.activityManager.list } - this.state.update(stateToUpdate) } private onFileReadFailed = (e: any) => { @@ -227,26 +226,21 @@ export default class MessageManager { this.setMessagesLoading(true) this.waitingForFiles = true - try { - if (this.session.domURL && this.session.domURL.length > 0) { - await loadFiles(this.session.domURL, createNewParser()) - this.onFileReadSuccess() - } else { - const file = await requestEFSDom(this.session.sessionId) - const parser = createNewParser(false) - await parser(file) - } - } catch (e) { - console.error('Cant get session replay file:', e) - try { - // back compat with old mobsUrl - await loadFiles(this.session.mobsUrl, createNewParser(false)) - } catch (e) { - this.onFileReadFailed(e) - } - } finally { - this.onFileReadFinally() - } + let fileReadPromise = this.session.domURL && this.session.domURL.length > 0 + ? loadFiles(this.session.domURL, createNewParser()) + : Promise.reject() + fileReadPromise + // EFS fallback + .catch(() => requestEFSDom(this.session.sessionId).then(createNewParser(false))) + // old url fallback + .catch(e => { + logger.error('Can not get normal session replay file:', e) + // back compat fallback to an old mobsUrl + return loadFiles(this.session.mobsUrl, createNewParser(false)) + }) + .then(this.onFileReadSuccess) + .catch(this.onFileReadFailed) + .finally(this.onFileReadFinally) // load devtools if (this.session.devtoolsURL.length) { @@ -256,7 +250,10 @@ export default class MessageManager { requestEFSDevtools(this.session.sessionId) .then(createNewParser(false)) ) - //.catch() // not able to download the devtools file + .then(() => { + this.state.update(this.lists.getFullListsState()) + }) + .catch(e => logger.error("Can not download the devtools file", e)) .finally(() => this.state.update({ devtoolsLoading: false })) } } @@ -439,15 +436,16 @@ export default class MessageManager { ) break; case MType.Fetch: + case MType.NetworkRequest: // @ts-ignore burn immutable - this.lists.lists.fetch.append(Resource({ + this.lists.lists.fetch.insert(Resource({ method: msg.method, url: msg.url, - payload: msg.request, + request: msg.request, response: msg.response, status: msg.status, duration: msg.duration, - type: TYPES.XHR, + type: msg.type === "xhr" ? RES_TYPES.XHR : RES_TYPES.FETCH, time: Math.max(msg.timestamp - this.sessionStart, 0), // !!! doesn't look good. TODO: find solution to show negative timings index, }) as Timed) diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 0bb2813ba..5201913b9 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -201,6 +201,28 @@ export default class RawMessageReader extends PrimitiveReader { }; } + case 21: { + const type = this.readString(); if (type === null) { return resetPointer() } + const method = this.readString(); if (method === null) { return resetPointer() } + const url = this.readString(); if (url === null) { return resetPointer() } + const request = this.readString(); if (request === null) { return resetPointer() } + const response = this.readString(); if (response === null) { return resetPointer() } + const status = this.readUint(); if (status === null) { return resetPointer() } + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const duration = this.readUint(); if (duration === null) { return resetPointer() } + return { + tp: MType.NetworkRequest, + type, + method, + url, + request, + response, + status, + timestamp, + duration, + }; + } + case 22: { const level = this.readString(); if (level === null) { return resetPointer() } const value = this.readString(); if (value === null) { return resetPointer() } @@ -349,6 +371,28 @@ export default class RawMessageReader extends PrimitiveReader { }; } + case 53: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const duration = this.readUint(); if (duration === null) { return resetPointer() } + const ttfb = this.readUint(); if (ttfb === null) { return resetPointer() } + const headerSize = this.readUint(); if (headerSize === null) { return resetPointer() } + const encodedBodySize = this.readUint(); if (encodedBodySize === null) { return resetPointer() } + const decodedBodySize = this.readUint(); if (decodedBodySize === null) { return resetPointer() } + const url = this.readString(); if (url === null) { return resetPointer() } + const initiator = this.readString(); if (initiator === null) { return resetPointer() } + return { + tp: MType.ResourceTiming, + timestamp, + duration, + ttfb, + headerSize, + encodedBodySize, + decodedBodySize, + url, + initiator, + }; + } + case 54: { const downlink = this.readUint(); if (downlink === null) { return resetPointer() } const type = this.readString(); if (type === null) { return resetPointer() } diff --git a/frontend/app/player/web/messages/message.gen.ts b/frontend/app/player/web/messages/message.gen.ts index bfe658690..0900cc471 100644 --- a/frontend/app/player/web/messages/message.gen.ts +++ b/frontend/app/player/web/messages/message.gen.ts @@ -21,6 +21,7 @@ import type { RawSetInputValue, RawSetInputChecked, RawMouseMove, + RawNetworkRequest, RawConsoleLog, RawCssInsertRule, RawCssDeleteRule, @@ -33,6 +34,7 @@ import type { RawNgRx, RawGraphQl, RawPerformanceTrack, + RawResourceTiming, RawConnectionInformation, RawSetPageVisibility, RawLoadFontFace, @@ -97,6 +99,8 @@ export type SetInputChecked = RawSetInputChecked & Timed export type MouseMove = RawMouseMove & Timed +export type NetworkRequest = RawNetworkRequest & Timed + export type ConsoleLog = RawConsoleLog & Timed export type CssInsertRule = RawCssInsertRule & Timed @@ -121,6 +125,8 @@ export type GraphQl = RawGraphQl & Timed export type PerformanceTrack = RawPerformanceTrack & Timed +export type ResourceTiming = RawResourceTiming & Timed + export type ConnectionInformation = RawConnectionInformation & Timed export type SetPageVisibility = RawSetPageVisibility & Timed diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index 960c17617..2de1ae946 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -19,6 +19,7 @@ export const enum MType { SetInputValue = 18, SetInputChecked = 19, MouseMove = 20, + NetworkRequest = 21, ConsoleLog = 22, CssInsertRule = 37, CssDeleteRule = 38, @@ -31,6 +32,7 @@ export const enum MType { NgRx = 47, GraphQl = 48, PerformanceTrack = 49, + ResourceTiming = 53, ConnectionInformation = 54, SetPageVisibility = 55, LoadFontFace = 57, @@ -167,6 +169,18 @@ export interface RawMouseMove { y: number, } +export interface RawNetworkRequest { + tp: MType.NetworkRequest, + type: string, + method: string, + url: string, + request: string, + response: string, + status: number, + timestamp: number, + duration: number, +} + export interface RawConsoleLog { tp: MType.ConsoleLog, level: string, @@ -253,6 +267,18 @@ export interface RawPerformanceTrack { usedJSHeapSize: number, } +export interface RawResourceTiming { + tp: MType.ResourceTiming, + timestamp: number, + duration: number, + ttfb: number, + headerSize: number, + encodedBodySize: number, + decodedBodySize: number, + url: string, + initiator: string, +} + export interface RawConnectionInformation { tp: MType.ConnectionInformation, downlink: number, @@ -448,4 +474,4 @@ export interface RawIosNetworkCall { } -export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; +export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawResourceTiming | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; diff --git a/frontend/app/player/web/messages/tracker-legacy.gen.ts b/frontend/app/player/web/messages/tracker-legacy.gen.ts index 5e6067d3c..3d5f81b0e 100644 --- a/frontend/app/player/web/messages/tracker-legacy.gen.ts +++ b/frontend/app/player/web/messages/tracker-legacy.gen.ts @@ -20,6 +20,7 @@ export const TP_MAP = { 18: MType.SetInputValue, 19: MType.SetInputChecked, 20: MType.MouseMove, + 21: MType.NetworkRequest, 22: MType.ConsoleLog, 37: MType.CssInsertRule, 38: MType.CssDeleteRule, @@ -32,6 +33,7 @@ export const TP_MAP = { 47: MType.NgRx, 48: MType.GraphQl, 49: MType.PerformanceTrack, + 53: MType.ResourceTiming, 54: MType.ConnectionInformation, 55: MType.SetPageVisibility, 57: MType.LoadFontFace, diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index 28fd88eff..0788b85d1 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -128,6 +128,18 @@ type TrMouseMove = [ y: number, ] +type TrNetworkRequest = [ + type: 21, + type: string, + method: string, + url: string, + request: string, + response: string, + status: number, + timestamp: number, + duration: number, +] + type TrConsoleLog = [ type: 22, level: string, @@ -412,7 +424,7 @@ type TrJSException = [ ] -export type TrackerMessage = TrBatchMetadata | TrPartitionedMessage | TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrJSExceptionDeprecated | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrZustand | TrJSException +export type TrackerMessage = TrBatchMetadata | TrPartitionedMessage | TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrJSExceptionDeprecated | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrZustand | TrJSException export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { @@ -551,6 +563,20 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { } } + case 21: { + return { + tp: MType.NetworkRequest, + type: tMsg[1], + method: tMsg[2], + url: tMsg[3], + request: tMsg[4], + response: tMsg[5], + status: tMsg[6], + timestamp: tMsg[7], + duration: tMsg[8], + } + } + case 22: { return { tp: MType.ConsoleLog, @@ -661,6 +687,20 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { } } + case 53: { + return { + tp: MType.ResourceTiming, + timestamp: tMsg[1], + duration: tMsg[2], + ttfb: tMsg[3], + headerSize: tMsg[4], + encodedBodySize: tMsg[5], + decodedBodySize: tMsg[6], + url: tMsg[7], + initiator: tMsg[8], + } + } + case 54: { return { tp: MType.ConnectionInformation, diff --git a/frontend/app/types/session/resource.js b/frontend/app/types/session/resource.js index 74cbceaf6..3b0523f78 100644 --- a/frontend/app/types/session/resource.js +++ b/frontend/app/types/session/resource.js @@ -3,6 +3,7 @@ import Record from 'Types/Record'; import { getResourceName } from 'App/utils'; const XHR = 'xhr'; +const FETCH = 'fetch'; const JS = 'script'; const CSS = 'css'; const IMG = 'img'; @@ -32,7 +33,6 @@ const OTHER = 'other'; const TYPES_MAP = { "stylesheet": CSS, - "fetch": XHR, } function getResourceStatus(status, success) { @@ -53,6 +53,7 @@ function getResourceSuccess(success, status) { export const TYPES = { XHR, + FETCH, JS, CSS, IMG, @@ -85,7 +86,7 @@ export default Record({ // initiator: "other", // pagePath: "", method: '', - payload:'', + request:'', response: '', headerSize: 0, encodedBodySize: 0, @@ -93,14 +94,13 @@ export default Record({ responseBodySize: 0, timings: List(), }, { - fromJS: ({ responseBody, response, type, initiator, status, success, time, datetime, timestamp, timings, ...resource }) => ({ + fromJS: ({ type, initiator, status, success, time, datetime, timestamp, timings, ...resource }) => ({ ...resource, type: TYPES_MAP[type] || type, name: getResourceName(resource.url), status: getResourceStatus(status, success), success: getResourceSuccess(success, status), time: typeof time === 'number' ? time : datetime || timestamp, - response: responseBody || response, ttfb: timings && timings.ttfb, timewidth: timings && timings.timewidth, timings, diff --git a/frontend/app/types/session/session.ts b/frontend/app/types/session/session.ts index 5a472b043..49d5f6962 100644 --- a/frontend/app/types/session/session.ts +++ b/frontend/app/types/session/session.ts @@ -115,12 +115,8 @@ export default Record( .filter(({ type, time }) => type !== TYPES.CONSOLE && time <= durationSeconds); let resources = List(session.resources).map(Resource); - // this code shoud die. - const firstResourceTime = resources - .map((r) => r.time) - .reduce((a, b) => Math.min(a, b), Number.MAX_SAFE_INTEGER); resources = resources - .map((r) => r.set('time', r.time - firstResourceTime)) + .map((r) => r.set('time', Math.max(0, r.time - startedAt))) .sort((r1, r2) => r1.time - r2.time); const missedResources = resources.filter(({ success }) => !success); @@ -171,7 +167,6 @@ export default Record( ), userDisplayName: session.userId || session.userAnonymousId || session.userID || 'Anonymous User', - firstResourceTime, issues: issuesList, sessionId: sessionId || sessionID, userId: session.userId || session.userID, diff --git a/frontend/app/utils/index.ts b/frontend/app/utils/index.ts index 455e04e3d..741e91e8a 100644 --- a/frontend/app/utils/index.ts +++ b/frontend/app/utils/index.ts @@ -17,7 +17,7 @@ export function debounce(callback, wait, context = this) { }; } -export function getResourceName(url = '') { +export function getResourceName(url: string) { return url .split('/') .filter((s) => s !== '') diff --git a/mobs/messages.rb b/mobs/messages.rb index d63644a35..7090aaeb1 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -125,7 +125,16 @@ message 20, 'MouseMove' do uint 'X' uint 'Y' end -# 21 +message 21, 'NetworkRequest', :replayer => :devtools do + string 'Type' # fetch/xhr/anythingElse(axios,gql,fonts,image?) + string 'Method' + string 'URL' + string 'Request' + string 'Response' + uint 'Status' + uint 'Timestamp' + uint 'Duration' +end message 22, 'ConsoleLog', :replayer => :devtools do string 'Level' string 'Value' @@ -236,6 +245,7 @@ end # string 'Name' # string 'Payload' # end + # deprecated since 4.0.2 in favor of AdoptedSSInsertRule + AdoptedSSAddOwner message 37, 'CSSInsertRule' do uint 'ID' @@ -248,6 +258,7 @@ message 38, 'CSSDeleteRule' do uint 'Index' end +# deprecated since 4.1.10 in favor of NetworkRequest message 39, 'Fetch', :replayer => :devtools do string 'Method' string 'URL' @@ -329,7 +340,7 @@ end message 52, 'DOMDrop', :tracker => false, :replayer => false do uint 'Timestamp' end -message 53, 'ResourceTiming', :replayer => false do +message 53, 'ResourceTiming', :replayer => :devtools do uint 'Timestamp' uint 'Duration' uint 'TTFB' diff --git a/mobs/templates/tracker~tracker~src~main~app~messages.gen.ts.erb b/mobs/templates/tracker~tracker~src~main~app~messages.gen.ts.erb index d4c132f8a..40aec9ce0 100644 --- a/mobs/templates/tracker~tracker~src~main~app~messages.gen.ts.erb +++ b/mobs/templates/tracker~tracker~src~main~app~messages.gen.ts.erb @@ -2,7 +2,7 @@ /* eslint-disable */ import * as Messages from '../../common/messages.gen.js' -export { default } from '../../common/messages.gen.js' +export { default, Type } from '../../common/messages.gen.js' <% $messages.select { |msg| msg.tracker }.each do |msg| %> export function <%= msg.name %>( diff --git a/tracker/tracker/.eslintrc.cjs b/tracker/tracker/.eslintrc.cjs index de1fed019..72c31e9a4 100644 --- a/tracker/tracker/.eslintrc.cjs +++ b/tracker/tracker/.eslintrc.cjs @@ -45,5 +45,6 @@ module.exports = { 'no-unused-expressions': 'off', '@typescript-eslint/no-unused-expressions': 'warn', '@typescript-eslint/no-useless-constructor': 'warn', + 'prefer-rest-params': 'off', }, }; diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index a02a0199f..5e22d5e2c 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -2,6 +2,8 @@ - Added "tel" to supported input types - Added `{ withCurrentTime: true }` to `tracker.getSessionURL` method which will return sessionURL with current session's timestamp +- Added Network module that captures fetch/xhr by default (with no plugin required) +- Use `timeOrigin()` instead of `performance.timing.navigationStart` in ResourceTiming messages ## 4.1.8 diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 9dbdb755a..2d6624b5e 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -21,6 +21,7 @@ export declare const enum Type { SetInputValue = 18, SetInputChecked = 19, MouseMove = 20, + NetworkRequest = 21, ConsoleLog = 22, PageLoadTiming = 23, PageRenderTiming = 24, @@ -187,6 +188,18 @@ export type MouseMove = [ /*y:*/ number, ] +export type NetworkRequest = [ + /*type:*/ Type.NetworkRequest, + /*type:*/ string, + /*method:*/ string, + /*url:*/ string, + /*request:*/ string, + /*response:*/ string, + /*status:*/ number, + /*timestamp:*/ number, + /*duration:*/ number, +] + export type ConsoleLog = [ /*type:*/ Type.ConsoleLog, /*level:*/ string, @@ -471,5 +484,5 @@ export type JSException = [ ] -type Message = BatchMetadata | PartitionedMessage | Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | ConsoleLog | PageLoadTiming | PageRenderTiming | JSExceptionDeprecated | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | Zustand | JSException +type Message = BatchMetadata | PartitionedMessage | Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | JSExceptionDeprecated | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | Zustand | JSException export default Message diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 90e766f56..5e33b84bc 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -1,5 +1,5 @@ import type Message from './messages.gen.js' -import { Timestamp, Metadata, UserID } from './messages.gen.js' +import { Timestamp, Metadata, UserID, Type as MType } from './messages.gen.js' import { now, adjustTimeOrigin, deprecationWarn } from '../utils.js' import Nodes from './nodes.js' import Observer from './observer/top_observer.js' @@ -199,10 +199,22 @@ export default class App { this.debug.error('OpenReplay error: ', context, e) } + private _usingOldFetchPlugin = false send(message: Message, urgent = false): void { if (this.activityState === ActivityState.NotActive) { return } + // === Back compatibility with Fetch/Axios plugins === + if (message[0] === MType.Fetch) { + this._usingOldFetchPlugin = true + deprecationWarn('Fetch plugin', "'network' init option") + deprecationWarn('Axios plugin', "'network' init option") + } + if (this._usingOldFetchPlugin && message[0] === MType.NetworkRequest) { + return + } + // ==================================================== + this.messages.push(message) // TODO: commit on start if there were `urgent` sends; // Clarify where urgent can be used for; diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index c9f42e610..82f106e7d 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -2,7 +2,7 @@ /* eslint-disable */ import * as Messages from '../../common/messages.gen.js' -export { default } from '../../common/messages.gen.js' +export { default, Type } from '../../common/messages.gen.js' export function BatchMetadata( @@ -232,6 +232,29 @@ export function MouseMove( ] } +export function NetworkRequest( + type: string, + method: string, + url: string, + request: string, + response: string, + status: number, + timestamp: number, + duration: number, +): Messages.NetworkRequest { + return [ + Messages.Type.NetworkRequest, + type, + method, + url, + request, + response, + status, + timestamp, + duration, + ] +} + export function ConsoleLog( level: string, value: string, diff --git a/tracker/tracker/src/main/app/messages.ts b/tracker/tracker/src/main/app/messages.ts deleted file mode 100644 index 413d2df7e..000000000 --- a/tracker/tracker/src/main/app/messages.ts +++ /dev/null @@ -1,339 +0,0 @@ -// Auto-generated, do not edit - -import * as Messages from '../../common/messages.gen.js' -export { default } from '../../common/messages.gen.js' - -export function BatchMetadata( - version: number, - pageNo: number, - firstIndex: number, - timestamp: number, - location: string, -): Messages.BatchMetadata { - return [Messages.Type.BatchMetadata, version, pageNo, firstIndex, timestamp, location] -} - -export function PartitionedMessage(partNo: number, partTotal: number): Messages.PartitionedMessage { - return [Messages.Type.PartitionedMessage, partNo, partTotal] -} - -export function Timestamp(timestamp: number): Messages.Timestamp { - return [Messages.Type.Timestamp, timestamp] -} - -export function SetPageLocation( - url: string, - referrer: string, - navigationStart: number, -): Messages.SetPageLocation { - return [Messages.Type.SetPageLocation, url, referrer, navigationStart] -} - -export function SetViewportSize(width: number, height: number): Messages.SetViewportSize { - return [Messages.Type.SetViewportSize, width, height] -} - -export function SetViewportScroll(x: number, y: number): Messages.SetViewportScroll { - return [Messages.Type.SetViewportScroll, x, y] -} - -export function CreateDocument(): Messages.CreateDocument { - return [Messages.Type.CreateDocument] -} - -export function CreateElementNode( - id: number, - parentID: number, - index: number, - tag: string, - svg: boolean, -): Messages.CreateElementNode { - return [Messages.Type.CreateElementNode, id, parentID, index, tag, svg] -} - -export function CreateTextNode( - id: number, - parentID: number, - index: number, -): Messages.CreateTextNode { - return [Messages.Type.CreateTextNode, id, parentID, index] -} - -export function MoveNode(id: number, parentID: number, index: number): Messages.MoveNode { - return [Messages.Type.MoveNode, id, parentID, index] -} - -export function RemoveNode(id: number): Messages.RemoveNode { - return [Messages.Type.RemoveNode, id] -} - -export function SetNodeAttribute( - id: number, - name: string, - value: string, -): Messages.SetNodeAttribute { - return [Messages.Type.SetNodeAttribute, id, name, value] -} - -export function RemoveNodeAttribute(id: number, name: string): Messages.RemoveNodeAttribute { - return [Messages.Type.RemoveNodeAttribute, id, name] -} - -export function SetNodeData(id: number, data: string): Messages.SetNodeData { - return [Messages.Type.SetNodeData, id, data] -} - -export function SetNodeScroll(id: number, x: number, y: number): Messages.SetNodeScroll { - return [Messages.Type.SetNodeScroll, id, x, y] -} - -export function SetInputTarget(id: number, label: string): Messages.SetInputTarget { - return [Messages.Type.SetInputTarget, id, label] -} - -export function SetInputValue(id: number, value: string, mask: number): Messages.SetInputValue { - return [Messages.Type.SetInputValue, id, value, mask] -} - -export function SetInputChecked(id: number, checked: boolean): Messages.SetInputChecked { - return [Messages.Type.SetInputChecked, id, checked] -} - -export function MouseMove(x: number, y: number): Messages.MouseMove { - return [Messages.Type.MouseMove, x, y] -} - -export function ConsoleLog(level: string, value: string): Messages.ConsoleLog { - return [Messages.Type.ConsoleLog, level, value] -} - -export function PageLoadTiming( - requestStart: number, - responseStart: number, - responseEnd: number, - domContentLoadedEventStart: number, - domContentLoadedEventEnd: number, - loadEventStart: number, - loadEventEnd: number, - firstPaint: number, - firstContentfulPaint: number, -): Messages.PageLoadTiming { - return [ - Messages.Type.PageLoadTiming, - requestStart, - responseStart, - responseEnd, - domContentLoadedEventStart, - domContentLoadedEventEnd, - loadEventStart, - loadEventEnd, - firstPaint, - firstContentfulPaint, - ] -} - -export function PageRenderTiming( - speedIndex: number, - visuallyComplete: number, - timeToInteractive: number, -): Messages.PageRenderTiming { - return [Messages.Type.PageRenderTiming, speedIndex, visuallyComplete, timeToInteractive] -} - -export function JSException( - name: string, - message: string, - payload: string, - metadata: string, -): Messages.JSException { - return [Messages.Type.JSException, name, message, payload, metadata] -} - -export function RawCustomEvent(name: string, payload: string): Messages.CustomEvent { - return [Messages.Type.CustomEvent, name, payload] -} - -export function UserID(id: string): Messages.UserID { - return [Messages.Type.UserID, id] -} - -export function UserAnonymousID(id: string): Messages.UserAnonymousID { - return [Messages.Type.UserAnonymousID, id] -} - -export function Metadata(key: string, value: string): Messages.Metadata { - return [Messages.Type.Metadata, key, value] -} - -export function CSSInsertRule(id: number, rule: string, index: number): Messages.CSSInsertRule { - return [Messages.Type.CSSInsertRule, id, rule, index] -} - -export function CSSDeleteRule(id: number, index: number): Messages.CSSDeleteRule { - return [Messages.Type.CSSDeleteRule, id, index] -} - -export function Fetch( - method: string, - url: string, - request: string, - response: string, - status: number, - timestamp: number, - duration: number, -): Messages.Fetch { - return [Messages.Type.Fetch, method, url, request, response, status, timestamp, duration] -} - -export function Profiler( - name: string, - duration: number, - args: string, - result: string, -): Messages.Profiler { - return [Messages.Type.Profiler, name, duration, args, result] -} - -export function OTable(key: string, value: string): Messages.OTable { - return [Messages.Type.OTable, key, value] -} - -export function StateAction(type: string): Messages.StateAction { - return [Messages.Type.StateAction, type] -} - -export function Redux(action: string, state: string, duration: number): Messages.Redux { - return [Messages.Type.Redux, action, state, duration] -} - -export function Vuex(mutation: string, state: string): Messages.Vuex { - return [Messages.Type.Vuex, mutation, state] -} - -export function MobX(type: string, payload: string): Messages.MobX { - return [Messages.Type.MobX, type, payload] -} - -export function NgRx(action: string, state: string, duration: number): Messages.NgRx { - return [Messages.Type.NgRx, action, state, duration] -} - -export function GraphQL( - operationKind: string, - operationName: string, - variables: string, - response: string, -): Messages.GraphQL { - return [Messages.Type.GraphQL, operationKind, operationName, variables, response] -} - -export function PerformanceTrack( - frames: number, - ticks: number, - totalJSHeapSize: number, - usedJSHeapSize: number, -): Messages.PerformanceTrack { - return [Messages.Type.PerformanceTrack, frames, ticks, totalJSHeapSize, usedJSHeapSize] -} - -export function ResourceTiming( - timestamp: number, - duration: number, - ttfb: number, - headerSize: number, - encodedBodySize: number, - decodedBodySize: number, - url: string, - initiator: string, -): Messages.ResourceTiming { - return [ - Messages.Type.ResourceTiming, - timestamp, - duration, - ttfb, - headerSize, - encodedBodySize, - decodedBodySize, - url, - initiator, - ] -} - -export function ConnectionInformation( - downlink: number, - type: string, -): Messages.ConnectionInformation { - return [Messages.Type.ConnectionInformation, downlink, type] -} - -export function SetPageVisibility(hidden: boolean): Messages.SetPageVisibility { - return [Messages.Type.SetPageVisibility, hidden] -} - -export function LongTask( - timestamp: number, - duration: number, - context: number, - containerType: number, - containerSrc: string, - containerId: string, - containerName: string, -): Messages.LongTask { - return [ - Messages.Type.LongTask, - timestamp, - duration, - context, - containerType, - containerSrc, - containerId, - containerName, - ] -} - -export function SetNodeAttributeURLBased( - id: number, - name: string, - value: string, - baseURL: string, -): Messages.SetNodeAttributeURLBased { - return [Messages.Type.SetNodeAttributeURLBased, id, name, value, baseURL] -} - -export function SetCSSDataURLBased( - id: number, - data: string, - baseURL: string, -): Messages.SetCSSDataURLBased { - return [Messages.Type.SetCSSDataURLBased, id, data, baseURL] -} - -export function TechnicalInfo(type: string, value: string): Messages.TechnicalInfo { - return [Messages.Type.TechnicalInfo, type, value] -} - -export function CustomIssue(name: string, payload: string): Messages.CustomIssue { - return [Messages.Type.CustomIssue, name, payload] -} - -export function CSSInsertRuleURLBased( - id: number, - rule: string, - index: number, - baseURL: string, -): Messages.CSSInsertRuleURLBased { - return [Messages.Type.CSSInsertRuleURLBased, id, rule, index, baseURL] -} - -export function MouseClick( - id: number, - hesitationTime: number, - label: string, - selector: string, -): Messages.MouseClick { - return [Messages.Type.MouseClick, id, hesitationTime, label, selector] -} - -export function CreateIFrameDocument(frameID: number, id: number): Messages.CreateIFrameDocument { - return [Messages.Type.CreateIFrameDocument, frameID, id] -} diff --git a/tracker/tracker/src/main/index.ts b/tracker/tracker/src/main/index.ts index 630d7f484..b6a3019d9 100644 --- a/tracker/tracker/src/main/index.ts +++ b/tracker/tracker/src/main/index.ts @@ -22,6 +22,7 @@ import Viewport from './modules/viewport.js' import CSSRules from './modules/cssrules.js' import Focus from './modules/focus.js' import Fonts from './modules/fonts.js' +import Network from './modules/network.js' import ConstructedStyleSheets from './modules/constructedStyleSheets.js' import { IN_BROWSER, deprecationWarn, DOCS_HOST } from './utils.js' @@ -31,6 +32,7 @@ import type { Options as ExceptionOptions } from './modules/exception.js' import type { Options as InputOptions } from './modules/input.js' import type { Options as PerformanceOptions } from './modules/performance.js' import type { Options as TimingOptions } from './modules/timing.js' +import type { Options as NetworkOptions } from './modules/network.js' import type { StartOptions } from './app/index.js' //TODO: unique options init import type { StartPromiseReturn } from './app/index.js' @@ -43,6 +45,7 @@ export type Options = Partial< sessionToken?: string respectDoNotTrack?: boolean autoResetOnWindowOpen?: boolean + network?: NetworkOptions // dev only __DISABLE_SECURE_MODE?: boolean } @@ -127,6 +130,7 @@ export default class API { Scroll(app) Focus(app) Fonts(app) + Network(app, options.network) ;(window as any).__OPENREPLAY__ = this if (options.autoResetOnWindowOpen) { diff --git a/tracker/tracker/src/main/modules/network.ts b/tracker/tracker/src/main/modules/network.ts new file mode 100644 index 000000000..fdaed16c6 --- /dev/null +++ b/tracker/tracker/src/main/modules/network.ts @@ -0,0 +1,313 @@ +import type App from '../app/index.js' +import { NetworkRequest } from '../app/messages.gen.js' +import { getTimeOrigin } from '../utils.js' + +type WindowFetch = typeof window.fetch +type XHRRequestBody = Parameters[0] +type FetchRequestBody = RequestInit['body'] + +// Request: +// declare const enum BodyType { +// Blob = "Blob", +// ArrayBuffer = "ArrayBuffer", +// TypedArray = "TypedArray", +// DataView = "DataView", +// FormData = "FormData", +// URLSearchParams = "URLSearchParams", +// Document = "Document", // XHR only +// ReadableStream = "ReadableStream", // Fetch only +// Literal = "literal", +// Unknown = "unk", +// } +// XHRResponse body: ArrayBuffer, a Blob, a Document, a JavaScript Object, or a string + +// TODO: extract maximum of useful information from any type of Request/Responce bodies +// function objectifyBody(body: any): RequestBody { +// if (body instanceof Blob) { +// return { +// body: `; size: ${body.size}`, +// bodyType: BodyType.Blob, +// } +// } +// return { +// body, +// bodyType: BodyType.Literal, +// } +// } + +interface RequestData { + body: XHRRequestBody | FetchRequestBody + headers: Record +} +interface ResponseData { + body: any + headers: Record +} + +interface RequestResponseData { + readonly status: number + readonly method: string + url: string + request: RequestData + response: ResponseData +} + +interface XHRRequestData { + body: XHRRequestBody + headers: Record +} + +function getXHRRequestDataObject(xhr: XMLHttpRequest): XHRRequestData { + // @ts-ignore this is 3x faster than using Map + if (!xhr.__or_req_data__) { + // @ts-ignore + xhr.__or_req_data__ = { body: undefined, headers: {} } + } + // @ts-ignore + return xhr.__or_req_data__ +} + +function strMethod(method?: string) { + return typeof method === 'string' ? method.toUpperCase() : 'GET' +} + +type Sanitizer = (data: RequestResponseData) => RequestResponseData | null + +export interface Options { + sessionTokenHeader: string | boolean + failuresOnly: boolean + ignoreHeaders: Array | boolean + capturePayload: boolean + sanitizer?: Sanitizer +} + +export default function (app: App, opts: Partial = {}) { + const options: Options = Object.assign( + { + failuresOnly: false, + ignoreHeaders: ['Cookie', 'Set-Cookie', 'Authorization'], + capturePayload: false, + sessionTokenHeader: false, + }, + opts, + ) + + const ignoreHeaders = options.ignoreHeaders + const isHIgnored = Array.isArray(ignoreHeaders) + ? (name: string) => ignoreHeaders.includes(name) + : () => ignoreHeaders + + const stHeader = + options.sessionTokenHeader === true ? 'X-OpenReplay-SessionToken' : options.sessionTokenHeader + function setSessionTokenHeader(setRequestHeader: (name: string, value: string) => void) { + if (stHeader) { + const sessionToken = app.getSessionToken() + if (sessionToken) { + app.safe(setRequestHeader)(stHeader, sessionToken) + } + } + } + + function sanitize(reqResInfo: RequestResponseData) { + if (!options.capturePayload) { + delete reqResInfo.request.body + delete reqResInfo.response.body + } + if (options.sanitizer) { + const resBody = reqResInfo.response.body + if (typeof resBody === 'string') { + // Parse response in order to have handy view in sanitisation function + try { + reqResInfo.response.body = JSON.parse(resBody) + } catch {} + } + return options.sanitizer(reqResInfo) + } + return reqResInfo + } + + function stringify(r: RequestData | ResponseData): string { + if (r && typeof r.body !== 'string') { + try { + r.body = JSON.stringify(r.body) + } catch { + r.body = '' + app.notify.warn("Openreplay fetch couldn't stringify body:", r.body) + } + } + return JSON.stringify(r) + } + + /* ====== Fetch ====== */ + const origFetch = window.fetch.bind(window) as WindowFetch + window.fetch = (input, init = {}) => { + if (!(typeof input === 'string' || input instanceof URL) || app.isServiceURL(String(input))) { + return origFetch(input, init) + } + + setSessionTokenHeader(function (name, value) { + if (init.headers === undefined) { + init.headers = {} + } + if (init.headers instanceof Headers) { + init.headers.append(name, value) + } else if (Array.isArray(init.headers)) { + init.headers.push([name, value]) + } else { + init.headers[name] = value + } + }) + + const startTime = performance.now() + return origFetch(input, init).then((response) => { + const duration = performance.now() - startTime + if (options.failuresOnly && response.status < 400) { + return response + } + + const r = response.clone() + r.text() + .then((text) => { + const reqHs: Record = {} + const resHs: Record = {} + if (ignoreHeaders !== true) { + // request headers + const writeReqHeader = ([n, v]: [string, string]) => { + if (!isHIgnored(n)) { + reqHs[n] = v + } + } + if (init.headers instanceof Headers) { + init.headers.forEach((v, n) => writeReqHeader([n, v])) + } else if (Array.isArray(init.headers)) { + init.headers.forEach(writeReqHeader) + } else if (typeof init.headers === 'object') { + Object.entries(init.headers).forEach(writeReqHeader) + } + // response headers + r.headers.forEach((v, n) => { + if (!isHIgnored(n)) resHs[n] = v + }) + } + const method = strMethod(init.method) + + const reqResInfo = sanitize({ + url: String(input), + method, + status: r.status, + request: { + headers: reqHs, + body: init.body, + }, + response: { + headers: resHs, + body: text, + }, + }) + if (!reqResInfo) { + return + } + + app.send( + NetworkRequest( + 'fetch', + method, + String(reqResInfo.url), + stringify(reqResInfo.request), + stringify(reqResInfo.response), + r.status, + startTime + getTimeOrigin(), + duration, + ), + ) + }) + .catch((e) => app.debug.error('Could not process Fetch response:', e)) + + return response + }) + } + /* ====== <> ====== */ + + /* ====== XHR ====== */ + const nativeOpen = XMLHttpRequest.prototype.open + XMLHttpRequest.prototype.open = function (initMethod, url) { + const xhr = this + setSessionTokenHeader((name, value) => xhr.setRequestHeader(name, value)) + + let startTime = 0 + xhr.addEventListener('loadstart', (e) => { + startTime = e.timeStamp + }) + xhr.addEventListener( + 'load', + app.safe((e) => { + const { headers: reqHs, body: reqBody } = getXHRRequestDataObject(xhr) + const duration = startTime > 0 ? e.timeStamp - startTime : 0 + + const hString: string | null = ignoreHeaders ? '' : xhr.getAllResponseHeaders() // might be null (though only if no response received though) + const resHs = hString + ? hString + .split('\r\n') + .map((h) => h.split(':')) + .filter((entry) => !isHIgnored(entry[0])) + .reduce( + (hds, [name, value]) => ({ ...hds, [name]: value }), + {} as Record, + ) + : {} + + const method = strMethod(initMethod) + const reqResInfo = sanitize({ + url: String(url), + method, + status: xhr.status, + request: { + headers: reqHs, + body: reqBody, + }, + response: { + headers: resHs, + body: xhr.response, + }, + }) + if (!reqResInfo) { + return + } + + app.send( + NetworkRequest( + 'xhr', + method, + String(reqResInfo.url), + stringify(reqResInfo.request), + stringify(reqResInfo.response), + xhr.status, + startTime + getTimeOrigin(), + duration, + ), + ) + }), + ) + + //TODO: handle error (though it has no Error API nor any useful information) + //xhr.addEventListener('error', (e) => {}) + return nativeOpen.apply(this, arguments) + } + const nativeSend = XMLHttpRequest.prototype.send + XMLHttpRequest.prototype.send = function (body) { + const rdo = getXHRRequestDataObject(this) + rdo.body = body + + return nativeSend.apply(this, arguments) + } + const nativeSetRequestHeader = XMLHttpRequest.prototype.setRequestHeader + XMLHttpRequest.prototype.setRequestHeader = function (name, value) { + if (!isHIgnored(name)) { + const rdo = getXHRRequestDataObject(this) + rdo.headers[name] = value + } + + return nativeSetRequestHeader.apply(this, arguments) + } + /* ====== <> ====== */ +} diff --git a/tracker/tracker/src/main/modules/timing.ts b/tracker/tracker/src/main/modules/timing.ts index f2d2cbf11..89f26ee7c 100644 --- a/tracker/tracker/src/main/modules/timing.ts +++ b/tracker/tracker/src/main/modules/timing.ts @@ -1,6 +1,6 @@ import type App from '../app/index.js' import { hasTag } from '../app/guards.js' -import { isURL } from '../utils.js' +import { isURL, getTimeOrigin } from '../utils.js' import { ResourceTiming, PageLoadTiming, PageRenderTiming } from '../app/messages.gen.js' // Inspired by https://github.com/WPO-Foundation/RUM-SpeedIndex/blob/master/src/rum-speedindex.js @@ -110,7 +110,7 @@ export default function (app: App, opts: Partial): void { } app.send( ResourceTiming( - entry.startTime + performance.timing.navigationStart, + entry.startTime + getTimeOrigin(), entry.duration, entry.responseStart && entry.startTime ? entry.responseStart - entry.startTime : 0, entry.transferSize > entry.encodedBodySize ? entry.transferSize - entry.encodedBodySize : 0, @@ -122,9 +122,7 @@ export default function (app: App, opts: Partial): void { ) } - const observer: PerformanceObserver = new PerformanceObserver((list) => - list.getEntries().forEach(resourceTiming), - ) + const observer = new PerformanceObserver((list) => list.getEntries().forEach(resourceTiming)) let prevSessionID: string | undefined app.attachStartCallback(function ({ sessionID }) { @@ -165,6 +163,9 @@ export default function (app: App, opts: Partial): void { if (performance.timing.loadEventEnd || performance.now() > 30000) { pageLoadTimingSent = true const { + // should be ok to use here, (https://github.com/mdn/content/issues/4713) + // since it is compared with the values obtained on the page load (before any possible sleep state) + // deprecated though navigationStart, requestStart, responseStart, diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 74e51f528..d7772d54a 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -86,6 +86,10 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.uint(msg[2]) break + case Messages.Type.NetworkRequest: + return this.string(msg[1]) && this.string(msg[2]) && this.string(msg[3]) && this.string(msg[4]) && this.string(msg[5]) && this.uint(msg[6]) && this.uint(msg[7]) && this.uint(msg[8]) + break + case Messages.Type.ConsoleLog: return this.string(msg[1]) && this.string(msg[2]) break