jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(backend,mobs):replace Fetch with NetworkRequest

Browse source
This commit is contained in:
Alex Kaminskii 2022-12-09 15:48:47 +01:00
parent 6a930a433a
commit ff80386aff
19 changed files with 198 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/cmd/db/main.go
View file

@ -62,7 +62,7 @@ func main() {
messages.MsgUserID, messages.MsgUserAnonymousID, messages.MsgClickEvent,
messages.MsgIntegrationEvent, messages.MsgPerformanceTrackAggr,
messages.MsgJSException, messages.MsgResourceTiming,
messages.MsgCustomEvent, messages.MsgCustomIssue, messages.MsgFetch, messages.MsgGraphQL,
messages.MsgCustomEvent, messages.MsgCustomIssue, messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL,
messages.MsgStateAction, messages.MsgSetInputTarget, messages.MsgSetInputValue, messages.MsgCreateDocument,
messages.MsgMouseClick, messages.MsgSetPageLocation, messages.MsgPageLoadTiming, messages.MsgPageRenderTiming}

6
backend/internal/db/datasaver/fts.go
View file

@ -6,7 +6,7 @@ import (
"openreplay/backend/pkg/messages"
)
type FetchFTS struct {
type NetworkRequestFTS struct {
Method string `json:"method"`
URL string `json:"url"`
Request string `json:"request"`
@ -56,8 +56,8 @@ func (s *Saver) sendToFTS(msg messages.Message, sessionID uint64) {
switch m := msg.(type) {
// Common
case *messages.Fetch:
event, err = json.Marshal(FetchFTS{
case *messages.NetworkRequest:
event, err = json.Marshal(NetworkRequestFTS{
Method: m.Method,
URL: m.URL,
Request: m.Request,

4
backend/internal/db/datasaver/messages.go
View file

@ -38,9 +38,9 @@ func (mi *Saver) InsertMessage(msg Message) error {
case *PageEvent:
mi.sendToFTS(msg, sessionID)
return mi.pg.InsertWebPageEvent(sessionID, m)
case *Fetch:
case *NetworkRequest:
mi.sendToFTS(msg, sessionID)
return mi.pg.InsertWebFetch(sessionID, m)
return mi.pg.InsertWebNetworkRequest(sessionID, m)
case *GraphQL:
mi.sendToFTS(msg, sessionID)
return mi.pg.InsertWebGraphQL(sessionID, m)

4
backend/pkg/db/cache/messages-web.go vendored
View file

@ -99,7 +99,7 @@ func (c *PGCache) InsertSessionReferrer(sessionID uint64, referrer string) error
return c.Conn.InsertSessionReferrer(sessionID, referrer)
}
func (c *PGCache) InsertWebFetch(sessionID uint64, e *Fetch) error {
func (c *PGCache) InsertWebNetworkRequest(sessionID uint64, e *NetworkRequest) error {
session, err := c.Cache.GetSession(sessionID)
if err != nil {
return err
@ -108,7 +108,7 @@ func (c *PGCache) InsertWebFetch(sessionID uint64, e *Fetch) error {
if err != nil {
return err
}
return c.Conn.InsertWebFetch(sessionID, session.ProjectID, project.SaveRequestPayloads, e)
return c.Conn.InsertWebNetworkRequest(sessionID, session.ProjectID, project.SaveRequestPayloads, e)
}
func (c *PGCache) InsertWebGraphQL(sessionID uint64, e *GraphQL) error {

2
backend/pkg/db/postgres/messages-web.go
View file

@ -148,7 +148,7 @@ func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *typ
return
}
func (conn *Conn) InsertWebFetch(sessionID uint64, projectID uint32, savePayload bool, e *Fetch) error {
func (conn *Conn) InsertWebNetworkRequest(sessionID uint64, projectID uint32, savePayload bool, e *NetworkRequest) error {
var request, response *string
if savePayload {
request = &e.Request

4
backend/pkg/handlers/web/networkIssue.go
View file

@ -7,7 +7,7 @@ import (
/*
Handler name: NetworkIssue
Input events: ResourceTiming,
Fetch
NetworkRequest
Output event: IssueEvent
*/
@ -33,7 +33,7 @@ func (f *NetworkIssueDetector) Handle(message Message, messageID uint64, timesta
// ContextString: msg.URL,
// }
// }
case *Fetch:
case *NetworkRequest:
if msg.Status >= 400 {
return &IssueEvent{
Type: "bad_request",

11
backend/pkg/messages/legacy-message-transform.go
View file

@ -14,6 +14,17 @@ func transformDeprecated(msg Message) Message {
Timestamp: m.Timestamp,
EncryptionKey: "",
}
case *Fetch:
return &NetworkRequest{
Type: "fetch",
Method: m.Method,
URL: m.URL,
Request: m.Request,
Response: m.Response,
Status: m.Status,
Timestamp: m.Timestamp,
Duration: m.Duration,
}
}
return msg
}

36
backend/pkg/messages/messages.go
View file

@ -25,6 +25,7 @@ const (
MsgSetInputValue = 18
MsgSetInputChecked = 19
MsgMouseMove = 20
MsgNetworkRequest = 21
MsgConsoleLog = 22
MsgPageLoadTiming = 23
MsgPageRenderTiming = 24
@ -673,6 +674,41 @@ func (msg *MouseMove) TypeID() int {
return 20
}
type NetworkRequest struct {
message
Type string
Method string
URL string
Request string
Response string
Status uint64
Timestamp uint64
Duration uint64
}
func (msg *NetworkRequest) Encode() []byte {
buf := make([]byte, 81+len(msg.Type)+len(msg.Method)+len(msg.URL)+len(msg.Request)+len(msg.Response))
buf[0] = 21
p := 1
p = WriteString(msg.Type, buf, p)
p = WriteString(msg.Method, buf, p)
p = WriteString(msg.URL, buf, p)
p = WriteString(msg.Request, buf, p)
p = WriteString(msg.Response, buf, p)
p = WriteUint(msg.Status, buf, p)
p = WriteUint(msg.Timestamp, buf, p)
p = WriteUint(msg.Duration, buf, p)
return buf[:p]
}
func (msg *NetworkRequest) Decode() Message {
return msg
}
func (msg *NetworkRequest) TypeID() int {
return 21
}
type ConsoleLog struct {
message
Level string

32
backend/pkg/messages/read-message.go
View file

@ -348,6 +348,36 @@ func DecodeMouseMove(reader BytesReader) (Message, error) {
return msg, err
}
func DecodeNetworkRequest(reader BytesReader) (Message, error) {
var err error = nil
msg := &NetworkRequest{}
if msg.Type, err = reader.ReadString(); err != nil {
return nil, err
}
if msg.Method, err = reader.ReadString(); err != nil {
return nil, err
}
if msg.URL, err = reader.ReadString(); err != nil {
return nil, err
}
if msg.Request, err = reader.ReadString(); err != nil {
return nil, err
}
if msg.Response, err = reader.ReadString(); err != nil {
return nil, err
}
if msg.Status, err = reader.ReadUint(); err != nil {
return nil, err
}
if msg.Timestamp, err = reader.ReadUint(); err != nil {
return nil, err
}
if msg.Duration, err = reader.ReadUint(); err != nil {
return nil, err
}
return msg, err
}
func DecodeConsoleLog(reader BytesReader) (Message, error) {
var err error = nil
msg := &ConsoleLog{}
@ -1710,6 +1740,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) {
return DecodeSetInputChecked(reader)
case 20:
return DecodeMouseMove(reader)
case 21:
return DecodeNetworkRequest(reader)
case 22:
return DecodeConsoleLog(reader)
case 23:

4
ee/backend/internal/db/datasaver/messages.go
View file

@ -58,7 +58,7 @@ func (mi *Saver) InsertMessage(msg Message) error {
return mi.pg.InsertWebJSException(m)
case *IntegrationEvent:
return mi.pg.InsertWebIntegrationEvent(m)
case *Fetch:
case *NetworkRequest:
session, err := mi.pg.GetSession(sessionID)
if err != nil {
log.Printf("can't get session info for CH: %s", err)
@ -72,7 +72,7 @@ func (mi *Saver) InsertMessage(msg Message) error {
}
}
}
return mi.pg.InsertWebFetch(sessionID, m)
return mi.pg.InsertWebNetworkRequest(sessionID, m)
case *GraphQL:
session, err := mi.pg.GetSession(sessionID)
if err != nil {

4
ee/backend/pkg/db/clickhouse/connector.go
View file

@ -27,7 +27,7 @@ type Connector interface {
InsertWebErrorEvent(session *types.Session, msg *types.ErrorEvent) error
InsertWebPerformanceTrackAggr(session *types.Session, msg *messages.PerformanceTrackAggr) error
InsertAutocomplete(session *types.Session, msgType, msgValue string) error
InsertRequest(session *types.Session, msg *messages.Fetch, savePayload bool) error
InsertRequest(session *types.Session, msg *messages.NetworkRequest, savePayload bool) error
InsertCustom(session *types.Session, msg *messages.CustomEvent) error
InsertGraphQL(session *types.Session, msg *messages.GraphQL) error
InsertIssue(session *types.Session, msg *messages.IssueEvent) error
@ -352,7 +352,7 @@ func (c *connectorImpl) InsertAutocomplete(session *types.Session, msgType, msgV
return nil
}
func (c *connectorImpl) InsertRequest(session *types.Session, msg *messages.Fetch, savePayload bool) error {
func (c *connectorImpl) InsertRequest(session *types.Session, msg *messages.NetworkRequest, savePayload bool) error {
urlMethod := url.EnsureMethod(msg.Method)
if urlMethod == "" {
return fmt.Errorf("can't parse http method. sess: %d, method: %s", session.SessionID, msg.Method)

14
ee/connectors/msgcodec/messages.py
View file

@ -213,6 +213,20 @@ class MouseMove(Message):
self.y = y
class NetworkRequest(Message):
__id__ = 21
def __init__(self, type, method, url, request, response, status, timestamp, duration):
self.type = type
self.method = method
self.url = url
self.request = request
self.response = response
self.status = status
self.timestamp = timestamp
self.duration = duration
class ConsoleLog(Message):
__id__ = 22

12
ee/connectors/msgcodec/msgcodec.py
View file

@ -237,6 +237,18 @@ class MessageCodec(Codec):
y=self.read_uint(reader)
)
if message_id == 21:
return NetworkRequest(
type=self.read_string(reader),
method=self.read_string(reader),
url=self.read_string(reader),
request=self.read_string(reader),
response=self.read_string(reader),
status=self.read_uint(reader),
timestamp=self.read_uint(reader),
duration=self.read_uint(reader)
)
if message_id == 22:
return ConsoleLog(
level=self.read_string(reader),

22
frontend/app/player/web/messages/RawMessageReader.gen.ts
View file

@ -201,6 +201,28 @@ export default class RawMessageReader extends PrimitiveReader {
};
}
case 21: {
const type = this.readString(); if (type === null) { return resetPointer() }
const method = this.readString(); if (method === null) { return resetPointer() }
const url = this.readString(); if (url === null) { return resetPointer() }
const request = this.readString(); if (request === null) { return resetPointer() }
const response = this.readString(); if (response === null) { return resetPointer() }
const status = this.readUint(); if (status === null) { return resetPointer() }
const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() }
const duration = this.readUint(); if (duration === null) { return resetPointer() }
return {
tp: MType.NetworkRequest,
type,
method,
url,
request,
response,
status,
timestamp,
duration,
};
}
case 22: {
const level = this.readString(); if (level === null) { return resetPointer() }
const value = this.readString(); if (value === null) { return resetPointer() }

3
frontend/app/player/web/messages/message.gen.ts
View file

@ -21,6 +21,7 @@ import type {
RawSetInputValue,
RawSetInputChecked,
RawMouseMove,
RawNetworkRequest,
RawConsoleLog,
RawCssInsertRule,
RawCssDeleteRule,
@ -97,6 +98,8 @@ export type SetInputChecked = RawSetInputChecked & Timed
export type MouseMove = RawMouseMove & Timed
export type NetworkRequest = RawNetworkRequest & Timed
export type ConsoleLog = RawConsoleLog & Timed
export type CssInsertRule = RawCssInsertRule & Timed

15
frontend/app/player/web/messages/raw.gen.ts
View file

@ -19,6 +19,7 @@ export const enum MType {
SetInputValue = 18,
SetInputChecked = 19,
MouseMove = 20,
NetworkRequest = 21,
ConsoleLog = 22,
CssInsertRule = 37,
CssDeleteRule = 38,
@ -167,6 +168,18 @@ export interface RawMouseMove {
y: number,
}
export interface RawNetworkRequest {
tp: MType.NetworkRequest,
type: string,
method: string,
url: string,
request: string,
response: string,
status: number,
timestamp: number,
duration: number,
}
export interface RawConsoleLog {
tp: MType.ConsoleLog,
level: string,
@ -448,4 +461,4 @@ export interface RawIosNetworkCall {
}
export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall;
export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall;

1
frontend/app/player/web/messages/tracker-legacy.gen.ts
View file

@ -20,6 +20,7 @@ export const TP_MAP = {
18: MType.SetInputValue,
19: MType.SetInputChecked,
20: MType.MouseMove,
21: MType.NetworkRequest,
22: MType.ConsoleLog,
37: MType.CssInsertRule,
38: MType.CssDeleteRule,

28
frontend/app/player/web/messages/tracker.gen.ts
View file

@ -128,6 +128,18 @@ type TrMouseMove = [
y: number,
]
type TrNetworkRequest = [
type: 21,
type: string,
method: string,
url: string,
request: string,
response: string,
status: number,
timestamp: number,
duration: number,
]
type TrConsoleLog = [
type: 22,
level: string,
@ -412,7 +424,7 @@ type TrJSException = [
]
export type TrackerMessage = TrBatchMetadata | TrPartitionedMessage | TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrJSExceptionDeprecated | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrZustand | TrJSException
export type TrackerMessage = TrBatchMetadata | TrPartitionedMessage | TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrJSExceptionDeprecated | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrZustand | TrJSException
export default function translate(tMsg: TrackerMessage): RawMessage | null {
switch(tMsg[0]) {
@ -551,6 +563,20 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null {
}
}
case 21: {
return {
tp: MType.NetworkRequest,
type: tMsg[1],
method: tMsg[2],
url: tMsg[3],
request: tMsg[4],
response: tMsg[5],
status: tMsg[6],
timestamp: tMsg[7],
duration: tMsg[8],
}
}
case 22: {
return {
tp: MType.ConsoleLog,

12
mobs/messages.rb
View file

@ -125,7 +125,16 @@ message 20, 'MouseMove' do
uint 'X'
uint 'Y'
end
# 21
message 21, 'NetworkRequest', :replayer => :devtools do
string 'Type' # fetch/xhr/anythingElse(axios,gql,fonts,image?)
string 'Method'
string 'URL'
string 'Request'
string 'Response'
uint 'Status'
uint 'Timestamp'
uint 'Duration'
end
message 22, 'ConsoleLog', :replayer => :devtools do
string 'Level'
string 'Value'
@ -236,6 +245,7 @@ end
# string 'Name'
# string 'Payload'
# end
# deprecated since 4.0.2 in favor of AdoptedSSInsertRule + AdoptedSSAddOwner
message 37, 'CSSInsertRule' do
uint 'ID'