jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(): separate ieBuilder, peBuilder & networkIssueDeterctor from EventMapper

Browse source
This commit is contained in:
Alex Kaminskii 2022-05-11 21:27:18 +02:00
parent 6d2bfc0e77
commit 88bec7ab60
9 changed files with 309 additions and 403 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
backend/cmd/db/main.go
View file

@ -33,9 +33,9 @@ func main() {
// HandlersFabric returns the list of message handlers we want to be applied to each incoming message.
handlersFabric := func() {
return []handlers.MessageProcessor{
custom.NewMainHandler(), // TODO: separate to several handler
//custom.NewInputEventBuilder(),
//custom.NewPageEventBuilder(),
custom.EventMapper{},
custom.NewInputEventBuilder(),
custom.NewPageEventBuilder(),
}
}

1
backend/cmd/heuristics/main.go
View file

@ -33,6 +33,7 @@ func main() {
&web.CpuIssueDetector{},
&web.DeadClickDetector{},
&web.MemoryIssueDetector{},
&web.NetworkIssueDetector{},
&web.PerformanceAggregator{},
// iOS handlers
&ios.AppNotResponding{},

135
backend/internal/handlers/custom/eventMapper.go Normal file
View file

@ -0,0 +1,135 @@
package custom
import (
"net/url"
"strings"
. "openreplay/backend/pkg/messages"
)
func getURLExtention(URL string) string {
u, err := url.Parse(URL)
if err != nil {
return ""
}
i := strings.LastIndex(u.Path, ".")
return u.Path[i+1:]
}
func getResourceType(initiator string, URL string) string {
switch initiator {
case "xmlhttprequest", "fetch":
return "fetch"
case "img":
return "img"
default:
switch getURLExtention(URL) {
case "css":
return "stylesheet"
case "js":
return "script"
case "png", "gif", "jpg", "jpeg", "svg":
return "img"
case "mp4", "mkv", "ogg", "webm", "avi", "mp3":
return "media"
default:
return "other"
}
}
}
type EventMapper struct{}
func (b *EventMapper) Build() Message {
return nil
}
func (b *EventMapper) Handle(message Message, messageID uint64, timestamp uint64) Message {
switch msg := message.(type) {
case *RawErrorEvent:
// !!! This won't be handled because the Meta() timestamp emitted by `integrations` will be 0
// TODO: move to db directly
return &ErrorEvent{
MessageID: messageID,
Timestamp: msg.Timestamp,
Source: msg.Source,
Name: msg.Name,
Message: msg.Message,
Payload: msg.Payload,
}
case *MouseClick:
if msg.Label != "" {
return &ClickEvent{
MessageID: messageID,
Label: msg.Label,
HesitationTime: msg.HesitationTime,
Timestamp: timestamp,
Selector: msg.Selector,
}
}
case *JSException:
return &ErrorEvent{
MessageID: messageID,
Timestamp: timestamp,
Source: "js_exception",
Name: msg.Name,
Message: msg.Message,
Payload: msg.Payload,
}
case *ResourceTiming:
return &ResourceEvent{
MessageID: messageID,
Timestamp: msg.Timestamp,
Duration: msg.Duration,
TTFB: msg.TTFB,
HeaderSize: msg.HeaderSize,
EncodedBodySize: msg.EncodedBodySize,
DecodedBodySize: msg.DecodedBodySize,
URL: msg.URL,
Type: getResourceType(msg.Initiator, msg.URL),
Success: msg.Duration != 0,
}
case *RawCustomEvent:
return &CustomEvent{
MessageID: messageID,
Timestamp: timestamp,
Name: msg.Name,
Payload: msg.Payload,
}
case *CustomIssue:
return &IssueEvent{
Type: "custom",
Timestamp: timestamp,
MessageID: messageID,
ContextString: msg.Name,
Payload: msg.Payload,
}
case *Fetch:
return &FetchEvent{
MessageID: messageID,
Timestamp: msg.Timestamp,
Method: msg.Method,
URL: msg.URL,
Request: msg.Request,
Response: msg.Response,
Status: msg.Status,
Duration: msg.Duration,
}
case *GraphQL:
return &GraphQLEvent{
MessageID: messageID,
Timestamp: timestamp,
OperationKind: msg.OperationKind,
OperationName: msg.OperationName,
Variables: msg.Variables,
Response: msg.Response,
}
case *StateAction:
return &StateActionEvent{
MessageID: messageID,
Timestamp: timestamp,
Type: msg.Type,
}
}
return nil
}

95
backend/internal/handlers/custom/inputEventBuilder.go
View file

@ -4,6 +4,8 @@ import (
. "openreplay/backend/pkg/messages"
)
const INPUT_EVENT_TIMEOUT = 1 * 60 * 1000
type inputLabels map[uint64]string
type inputEventBuilder struct {
@ -12,78 +14,63 @@ type inputEventBuilder struct {
inputID uint64
}
func (b *inputEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message {
//TODO implement me
panic("implement me")
}
func (b *inputEventBuilder) Build() Message {
// b.build()
//TODO implement me
panic("implement me")
}
func NewInputEventBuilder() *inputEventBuilder {
ieBuilder := &inputEventBuilder{}
ieBuilder.ClearLabels()
ieBuilder.clearLabels()
return ieBuilder
}
func (b *inputEventBuilder) ClearLabels() {
func (b *inputEventBuilder) clearLabels() {
b.inputLabels = make(inputLabels)
}
func (b *inputEventBuilder) HandleSetInputTarget(msg *SetInputTarget) *InputEvent {
var inputEvent *InputEvent
if b.inputID != msg.ID {
inputEvent = b.build()
b.inputID = msg.ID
}
b.inputLabels[msg.ID] = msg.Label
return inputEvent
}
func (b *inputEventBuilder) HandleSetInputValue(msg *SetInputValue, messageID uint64, timestamp uint64) *InputEvent {
var inputEvent *InputEvent
if b.inputID != msg.ID {
inputEvent = b.build()
b.inputID = msg.ID
}
if b.inputEvent == nil {
b.inputEvent = &InputEvent{
MessageID: messageID,
Timestamp: timestamp,
Value: msg.Value,
ValueMasked: msg.Mask > 0,
func (b *inputEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message {
var inputEvent Message = nil
switch msg := message.(type) {
case *SetInputTarget:
if b.inputID != msg.ID {
inputEvent = b.Build()
b.inputID = msg.ID
}
} else {
b.inputEvent.Value = msg.Value
b.inputEvent.ValueMasked = msg.Mask > 0
b.inputLabels[msg.ID] = msg.Label
return inputEvent
case *SetInputValue:
if b.inputID != msg.ID {
inputEvent = b.Build()
b.inputID = msg.ID
}
if b.inputEvent == nil {
b.inputEvent = &InputEvent{
MessageID: messageID,
Timestamp: timestamp,
Value: msg.Value,
ValueMasked: msg.Mask > 0,
}
} else {
b.inputEvent.Value = msg.Value
b.inputEvent.ValueMasked = msg.Mask > 0
}
return inputEvent
case *CreateDocument:
inputEvent = b.Build()
b.clearLabels()
return inputEvent
case *MouseClick:
return b.Build()
}
return inputEvent
}
func (b *inputEventBuilder) HasInstance() bool {
return b.inputEvent != nil
}
func (b *inputEventBuilder) GetTimestamp() uint64 {
if b.inputEvent == nil {
return 0
if b.inputEvent != nil && b.inputEvent.Timestamp+INPUT_EVENT_TIMEOUT < timestamp {
return b.Build()
}
return b.inputEvent.Timestamp
return nil
}
func (b *inputEventBuilder) build() *InputEvent {
func (b *inputEventBuilder) Build() Message {
if b.inputEvent == nil {
return nil
}
inputEvent := b.inputEvent
label, exists := b.inputLabels[b.inputID]
if !exists {
return nil
}
inputEvent.Label = label
inputEvent.Label = b.inputLabels[b.inputID] // might be empty string
b.inputEvent = nil
return inputEvent

257
backend/internal/handlers/custom/mainHandler.go
View file

@ -1,257 +0,0 @@
package custom
import (
"net/url"
"openreplay/backend/pkg/intervals"
"strings"
"time"
. "openreplay/backend/pkg/messages"
)
func getURLExtention(URL string) string {
u, err := url.Parse(URL)
if err != nil {
return ""
}
i := strings.LastIndex(u.Path, ".")
return u.Path[i+1:]
}
func getResourceType(initiator string, URL string) string {
switch initiator {
case "xmlhttprequest", "fetch":
return "fetch"
case "img":
return "img"
default:
switch getURLExtention(URL) {
case "css":
return "stylesheet"
case "js":
return "script"
case "png", "gif", "jpg", "jpeg", "svg":
return "img"
case "mp4", "mkv", "ogg", "webm", "avi", "mp3":
return "media"
default:
return "other"
}
}
}
type builder struct {
readyMsgs []Message
timestamp uint64
lastProcessedTimestamp int64
peBuilder *pageEventBuilder
ieBuilder *inputEventBuilder
integrationsWaiting bool
sid uint64
}
func (b *builder) Build() Message {
//TODO implement me
panic("implement me")
}
func NewMainHandler() *builder {
return &builder{
peBuilder: &pageEventBuilder{},
ieBuilder: NewInputEventBuilder(),
integrationsWaiting: true,
}
}
func (b *builder) appendReadyMessage(msg Message) { // interface is never nil even if it holds nil value
b.readyMsgs = append(b.readyMsgs, msg)
}
func (b *builder) iterateReadyMessage(iter func(msg Message)) {
for _, readyMsg := range b.readyMsgs {
iter(readyMsg)
}
b.readyMsgs = nil
}
func (b *builder) buildPageEvent() {
if msg := b.peBuilder.Build(); msg != nil {
b.appendReadyMessage(msg)
}
}
func (b *builder) buildInputEvent() {
if msg := b.ieBuilder.Build(); msg != nil {
b.appendReadyMessage(msg)
}
}
func (b *builder) Handle(message Message, messageID uint64, timestamp uint64) Message {
b.timestamp = timestamp
b.lastProcessedTimestamp = time.Now().UnixMilli()
// Might happen before the first timestamp.
switch msg := message.(type) {
case *SessionStart,
*Metadata,
*UserID,
*UserAnonymousID:
b.appendReadyMessage(msg)
case *RawErrorEvent:
b.appendReadyMessage(&ErrorEvent{
MessageID: messageID,
Timestamp: msg.Timestamp,
Source: msg.Source,
Name: msg.Name,
Message: msg.Message,
Payload: msg.Payload,
})
}
if b.timestamp == 0 {
return nil
}
switch msg := message.(type) {
case *SetPageLocation:
if msg.NavigationStart == 0 {
b.appendReadyMessage(&PageEvent{
URL: msg.URL,
Referrer: msg.Referrer,
Loaded: false,
MessageID: messageID,
Timestamp: b.timestamp,
})
} else {
b.buildPageEvent()
b.buildInputEvent()
b.ieBuilder.ClearLabels()
b.peBuilder.HandleSetPageLocation(msg, messageID, b.timestamp)
}
case *PageLoadTiming:
if rm := b.peBuilder.HandlePageLoadTiming(msg); rm != nil {
b.appendReadyMessage(rm)
}
case *PageRenderTiming:
if rm := b.peBuilder.HandlePageRenderTiming(msg); rm != nil {
b.appendReadyMessage(rm)
}
case *SetInputTarget:
if rm := b.ieBuilder.HandleSetInputTarget(msg); rm != nil {
b.appendReadyMessage(rm)
}
case *SetInputValue:
if rm := b.ieBuilder.HandleSetInputValue(msg, messageID, b.timestamp); rm != nil {
b.appendReadyMessage(rm)
}
case *MouseClick:
b.buildInputEvent()
if msg.Label != "" {
b.appendReadyMessage(&ClickEvent{
MessageID: messageID,
Label: msg.Label,
HesitationTime: msg.HesitationTime,
Timestamp: b.timestamp,
Selector: msg.Selector,
})
}
case *JSException:
b.appendReadyMessage(&ErrorEvent{
MessageID: messageID,
Timestamp: b.timestamp,
Source: "js_exception",
Name: msg.Name,
Message: msg.Message,
Payload: msg.Payload,
})
case *ResourceTiming:
tp := getResourceType(msg.Initiator, msg.URL)
success := msg.Duration != 0
b.appendReadyMessage(&ResourceEvent{
MessageID: messageID,
Timestamp: msg.Timestamp,
Duration: msg.Duration,
TTFB: msg.TTFB,
HeaderSize: msg.HeaderSize,
EncodedBodySize: msg.EncodedBodySize,
DecodedBodySize: msg.DecodedBodySize,
URL: msg.URL,
Type: tp,
Success: success,
})
if !success {
issueType := "missing_resource"
if tp == "fetch" {
issueType = "bad_request"
}
b.appendReadyMessage(&IssueEvent{
Type: issueType,
MessageID: messageID,
Timestamp: msg.Timestamp,
ContextString: msg.URL,
})
}
case *RawCustomEvent:
b.appendReadyMessage(&CustomEvent{
MessageID: messageID,
Timestamp: b.timestamp,
Name: msg.Name,
Payload: msg.Payload,
})
case *CustomIssue:
b.appendReadyMessage(&IssueEvent{
Type: "custom",
Timestamp: b.timestamp,
MessageID: messageID,
ContextString: msg.Name,
Payload: msg.Payload,
})
case *Fetch:
b.appendReadyMessage(&FetchEvent{
MessageID: messageID,
Timestamp: msg.Timestamp,
Method: msg.Method,
URL: msg.URL,
Request: msg.Request,
Response: msg.Response,
Status: msg.Status,
Duration: msg.Duration,
})
if msg.Status >= 400 {
b.appendReadyMessage(&IssueEvent{
Type: "bad_request",
MessageID: messageID,
Timestamp: msg.Timestamp,
ContextString: msg.URL,
})
}
case *GraphQL:
b.appendReadyMessage(&GraphQLEvent{
MessageID: messageID,
Timestamp: b.timestamp,
OperationKind: msg.OperationKind,
OperationName: msg.OperationName,
Variables: msg.Variables,
Response: msg.Response,
})
case *StateAction:
b.appendReadyMessage(&StateActionEvent{
MessageID: messageID,
Timestamp: b.timestamp,
Type: msg.Type,
})
}
return nil
}
func (b *builder) checkTimeouts(ts int64) bool {
if b.timestamp == 0 {
return false // There was no timestamp events yet
}
if b.peBuilder.HasInstance() && int64(b.peBuilder.GetTimestamp())+intervals.EVENTS_PAGE_EVENT_TIMEOUT < ts {
b.buildPageEvent()
}
if b.ieBuilder.HasInstance() && int64(b.ieBuilder.GetTimestamp())+intervals.EVENTS_INPUT_EVENT_TIMEOUT < ts {
b.buildInputEvent()
}
return false
}

159
backend/internal/handlers/custom/pageEventBuilder.go
View file

@ -4,104 +4,103 @@ import (
. "openreplay/backend/pkg/messages"
)
const PAGE_EVENT_TIMEOUT = 1 * 60 * 1000
type pageEventBuilder struct {
pageEvent *PageEvent
firstTimingHandled bool
}
func (b *pageEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message {
//TODO implement me
panic("implement me")
}
func (b *pageEventBuilder) Build() Message {
// b.build()
//TODO implement me
panic("implement me")
}
func NewPageEventBuilder() *pageEventBuilder {
ieBuilder := &pageEventBuilder{}
return ieBuilder
}
func (b *pageEventBuilder) buildIfTimingsComplete() *PageEvent {
if b.firstTimingHandled {
return b.build()
func (b *pageEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message {
switch msg := message.(type) {
case *SetPageLocation:
if msg.NavigationStart == 0 { // routing without new page loading
return &PageEvent{
URL: msg.URL,
Referrer: msg.Referrer,
Loaded: false,
MessageID: messageID,
Timestamp: timestamp,
}
} else {
pageEvent := b.Build()
b.pageEvent = &PageEvent{
URL: msg.URL,
Referrer: msg.Referrer,
Loaded: true,
MessageID: messageID,
Timestamp: timestamp,
}
return pageEvent
}
case *PageLoadTiming:
if b.pageEvent == nil {
break
}
if msg.RequestStart <= 30000 {
b.pageEvent.RequestStart = msg.RequestStart
}
if msg.ResponseStart <= 30000 {
b.pageEvent.ResponseStart = msg.ResponseStart
}
if msg.ResponseEnd <= 30000 {
b.pageEvent.ResponseEnd = msg.ResponseEnd
}
if msg.DomContentLoadedEventStart <= 30000 {
b.pageEvent.DomContentLoadedEventStart = msg.DomContentLoadedEventStart
}
if msg.DomContentLoadedEventEnd <= 30000 {
b.pageEvent.DomContentLoadedEventEnd = msg.DomContentLoadedEventEnd
}
if msg.LoadEventStart <= 30000 {
b.pageEvent.LoadEventStart = msg.LoadEventStart
}
if msg.LoadEventEnd <= 30000 {
b.pageEvent.LoadEventEnd = msg.LoadEventEnd
}
if msg.FirstPaint <= 30000 {
b.pageEvent.FirstPaint = msg.FirstPaint
}
if msg.FirstContentfulPaint <= 30000 {
b.pageEvent.FirstContentfulPaint = msg.FirstContentfulPaint
}
return b.buildIfTimingsComplete()
case *PageRenderTiming:
if b.pageEvent == nil {
break
}
b.pageEvent.SpeedIndex = msg.SpeedIndex
b.pageEvent.VisuallyComplete = msg.VisuallyComplete
b.pageEvent.TimeToInteractive = msg.TimeToInteractive
return b.buildIfTimingsComplete()
}
if b.pageEvent != nil && b.pageEvent.Timestamp+PAGE_EVENT_TIMEOUT < timestamp {
return b.Build()
}
b.firstTimingHandled = true
return nil
}
// Only for Loaded: true
func (b *pageEventBuilder) HandleSetPageLocation(msg *SetPageLocation, messageID uint64, timestamp uint64) {
b.pageEvent = &PageEvent{
URL: msg.URL,
Referrer: msg.Referrer,
Loaded: true,
MessageID: messageID,
Timestamp: timestamp,
}
}
func (b *pageEventBuilder) HandlePageLoadTiming(msg *PageLoadTiming) *PageEvent {
if !b.HasInstance() {
return nil
}
if msg.RequestStart <= 30000 {
b.pageEvent.RequestStart = msg.RequestStart
}
if msg.ResponseStart <= 30000 {
b.pageEvent.ResponseStart = msg.ResponseStart
}
if msg.ResponseEnd <= 30000 {
b.pageEvent.ResponseEnd = msg.ResponseEnd
}
if msg.DomContentLoadedEventStart <= 30000 {
b.pageEvent.DomContentLoadedEventStart = msg.DomContentLoadedEventStart
}
if msg.DomContentLoadedEventEnd <= 30000 {
b.pageEvent.DomContentLoadedEventEnd = msg.DomContentLoadedEventEnd
}
if msg.LoadEventStart <= 30000 {
b.pageEvent.LoadEventStart = msg.LoadEventStart
}
if msg.LoadEventEnd <= 30000 {
b.pageEvent.LoadEventEnd = msg.LoadEventEnd
}
if msg.FirstPaint <= 30000 {
b.pageEvent.FirstPaint = msg.FirstPaint
}
if msg.FirstContentfulPaint <= 30000 {
b.pageEvent.FirstContentfulPaint = msg.FirstContentfulPaint
}
return b.buildIfTimingsComplete()
}
func (b *pageEventBuilder) HandlePageRenderTiming(msg *PageRenderTiming) *PageEvent {
if !b.HasInstance() {
return nil
}
b.pageEvent.SpeedIndex = msg.SpeedIndex
b.pageEvent.VisuallyComplete = msg.VisuallyComplete
b.pageEvent.TimeToInteractive = msg.TimeToInteractive
return b.buildIfTimingsComplete()
}
func (b *pageEventBuilder) HasInstance() bool {
return b.pageEvent != nil
}
func (b *pageEventBuilder) GetTimestamp() uint64 {
func (b *pageEventBuilder) Build() Message {
if b.pageEvent == nil {
return 0
return nil
}
return b.pageEvent.Timestamp
}
func (b *pageEventBuilder) build() *PageEvent {
pageEvent := b.pageEvent
b.pageEvent = nil
b.firstTimingHandled = false
return pageEvent
}
func (b *pageEventBuilder) buildIfTimingsComplete() Message {
if b.firstTimingHandled {
return b.Build()
}
b.firstTimingHandled = true
return nil
}

10
backend/internal/handlers/ios/performanceAggregator.go
View file

@ -48,7 +48,7 @@ func (h *PerformanceAggregator) Handle(message Message, messageID uint64, timest
h.pa.TimestampStart = m.Timestamp
}
if h.pa.TimestampStart+AGGR_TIME <= m.Timestamp {
event = h.build(m.Timestamp)
event = h.Build()
}
switch m.Name {
case "fps":
@ -89,21 +89,17 @@ func (h *PerformanceAggregator) Handle(message Message, messageID uint64, timest
}
}
case *IOSSessionEnd:
event = h.build(m.Timestamp)
event = h.Build()
}
return event
}
func (h *PerformanceAggregator) Build() Message {
return h.build(h.lastTimestamp)
}
func (h *PerformanceAggregator) build(timestamp uint64) Message {
if h.pa == nil {
return nil
}
h.pa.TimestampEnd = timestamp
h.pa.TimestampEnd = h.lastTimestamp
h.pa.AvgFPS = h.fps.aggregate()
h.pa.AvgCPU = h.cpu.aggregate()
h.pa.AvgMemory = h.memory.aggregate()

47
backend/internal/handlers/web/networkIssue.go Normal file
View file

@ -0,0 +1,47 @@
package web
import (
. "openreplay/backend/pkg/messages"
)
/*
Handler name: NetworkIssue
Input events: ResourceTiming,
Fetch
Output event: IssueEvent
*/
type NetworkIssueDetector struct{}
func (f *NetworkIssueDetector) Build() Message {
return nil
}
func (f *NetworkIssueDetector) Handle(message Message, messageID uint64, timestamp uint64) Message {
switch msg := message.(type) {
case *ResourceTiming:
success := msg.Duration != 0 // The only available way here
if !success {
issueType := "missing_resource"
if msg.Initiator == "fetch" || msg.Initiator == "xmlhttprequest" {
issueType = "bad_request"
}
return &IssueEvent{
Type: issueType,
MessageID: messageID,
Timestamp: msg.Timestamp,
ContextString: msg.URL,
}
}
case *Fetch:
if msg.Status >= 400 {
return &IssueEvent{
Type: "bad_request",
MessageID: messageID,
Timestamp: msg.Timestamp,
ContextString: msg.URL,
}
}
}
return nil
}

2
backend/pkg/intervals/intervals.go
View file

@ -3,8 +3,6 @@ package intervals
const EVENTS_COMMIT_INTERVAL = 30 * 1000 // как часто комитим сообщения в кафке (ender)
const HEARTBEAT_INTERVAL = 2 * 60 * 1000 // максимальный таймаут от трекера в рамках сессии
const INTEGRATIONS_REQUEST_INTERVAL = 1 * 60 * 1000 // интеграции
const EVENTS_PAGE_EVENT_TIMEOUT = 2 * 60 * 1000 // таймаут пейдж ивента
const EVENTS_INPUT_EVENT_TIMEOUT = 2 * 60 * 1000 //
const EVENTS_SESSION_END_TIMEOUT = HEARTBEAT_INTERVAL + 30*1000
const EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS = HEARTBEAT_INTERVAL + 3*60*1000
const EVENTS_BACK_COMMIT_GAP = EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS + 1*60*1000 // для бэк коммита