jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[DB] added url field for click rage issues (#925)

Browse source 
* feat(backend): added url field for click rage issues
This commit is contained in:
Alexander 2023-01-18 13:38:10 +01:00 • committed by GitHub
parent a6e4779652
commit da7d2038ef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 1858 additions and 1731 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
backend/pkg/handlers/web/clickRage.go
View file

@ -22,6 +22,7 @@ type ClickRageDetector struct {
firstInARawTimestamp uint64
firstInARawMessageId uint64
countsInARow int
url string
}
func (crd *ClickRageDetector) reset() {
@ -30,6 +31,7 @@ func (crd *ClickRageDetector) reset() {
crd.firstInARawTimestamp = 0
crd.firstInARawMessageId = 0
crd.countsInARow = 0
crd.url = ""
}
func (crd *ClickRageDetector) Build() Message {
@ -45,6 +47,7 @@ func (crd *ClickRageDetector) Build() Message {
Payload: string(payload),
Timestamp: crd.firstInARawTimestamp,
MessageID: crd.firstInARawMessageId,
URL: crd.url,
}
return event
}
@ -54,6 +57,9 @@ func (crd *ClickRageDetector) Build() Message {
func (crd *ClickRageDetector) Handle(message Message, messageID uint64, timestamp uint64) Message {
switch msg := message.(type) {
case *MouseClick:
if crd.url == "" && msg.Url != "" {
crd.url = msg.Url
}
// TODO: check if we it is ok to capture clickRage event without the connected ClickEvent in db.
if msg.Label == "" {
return crd.Build()
@ -69,6 +75,9 @@ func (crd *ClickRageDetector) Handle(message Message, messageID uint64, timestam
crd.firstInARawTimestamp = timestamp
crd.firstInARawMessageId = messageID
crd.countsInARow = 1
if crd.url == "" && msg.Url != "" {
crd.url = msg.Url
}
return event
}
return nil

4
backend/pkg/messages/filters.go
View file

@ -2,7 +2,7 @@
package messages
func IsReplayerType(id int) bool {
return 80 != id && 81 != id && 82 != id && 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id
return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id
}
func IsIOSType(id int) bool {
@ -11,4 +11,4 @@ func IsIOSType(id int) bool {
func IsDOMType(id int) bool {
return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id
}
}

10
backend/pkg/messages/legacy-message-transform.go
View file

@ -25,6 +25,16 @@ func transformDeprecated(msg Message) Message {
Timestamp: m.Timestamp,
Duration: m.Duration,
}
case *IssueEventDeprecated:
return &IssueEvent{
MessageID: m.MessageID,
Timestamp: m.Timestamp,
Type: m.Type,
ContextString: m.ContextString,
Context: m.Context,
Payload: m.Payload,
URL: "",
}
}
return msg
}

897
backend/pkg/messages/messages.go
View file

File diff suppressed because it is too large Load diff

2325
backend/pkg/messages/read-message.go
View file

File diff suppressed because it is too large Load diff

7
backend/pkg/messages/session-iterator.go
View file

@ -40,6 +40,13 @@ func SplitMessages(data []byte) ([]*msgInfo, error) {
return nil, fmt.Errorf("read message type err: %s", err)
}
if msgType == MsgRedux {
log.Printf("redux")
}
if msgType == MsgFetch {
log.Printf("fetch")
}
// Read message body
_, err = ReadMessage(msgType, reader)
if err != nil {

3
ee/backend/pkg/db/clickhouse/connector.go
View file

@ -87,7 +87,7 @@ var batches = map[string]string{
"requests": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_body, response_body, status, method, duration, success, event_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"custom": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, name, payload, event_type) VALUES (?, ?, ?, ?, ?, ?, ?)",
"graphql": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, name, request_body, response_body, event_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
"issuesEvents": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, issue_id, issue_type, event_type) VALUES (?, ?, ?, ?, ?, ?, ?)",
"issuesEvents": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, issue_id, issue_type, event_type, url) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
"issues": "INSERT INTO experimental.issues (project_id, issue_id, type, context_string) VALUES (?, ?, ?, ?)",
}
@ -132,6 +132,7 @@ func (c *connectorImpl) InsertIssue(session *types.Session, msg *messages.IssueE
issueID,
msg.Type,
"ISSUE",
msg.URL,
); err != nil {
c.checkError("issuesEvents", err)
return fmt.Errorf("can't append to issuesEvents batch: %s", err)

71
ee/connectors/msgcodec/messages.py
View file

@ -6,34 +6,6 @@ class Message(ABC):
pass
class BatchMeta(Message):
__id__ = 80
def __init__(self, page_no, first_index, timestamp):
self.page_no = page_no
self.first_index = first_index
self.timestamp = timestamp
class BatchMetadata(Message):
__id__ = 81
def __init__(self, version, page_no, first_index, timestamp, location):
self.version = version
self.page_no = page_no
self.first_index = first_index
self.timestamp = timestamp
self.location = location
class PartitionedMessage(Message):
__id__ = 82
def __init__(self, part_no, part_total):
self.part_no = part_no
self.part_total = part_total
class Timestamp(Message):
__id__ = 0
@ -586,7 +558,7 @@ class SetCSSDataURLBased(Message):
self.base_url = base_url
class IssueEvent(Message):
class IssueEventDeprecated(Message):
__id__ = 62
def __init__(self, message_id, timestamp, type, context_string, context, payload):
@ -727,6 +699,47 @@ class JSException(Message):
self.metadata = metadata
class BatchMeta(Message):
__id__ = 80
def __init__(self, page_no, first_index, timestamp):
self.page_no = page_no
self.first_index = first_index
self.timestamp = timestamp
class BatchMetadata(Message):
__id__ = 81
def __init__(self, version, page_no, first_index, timestamp, location):
self.version = version
self.page_no = page_no
self.first_index = first_index
self.timestamp = timestamp
self.location = location
class PartitionedMessage(Message):
__id__ = 82
def __init__(self, part_no, part_total):
self.part_no = part_no
self.part_total = part_total
class IssueEvent(Message):
__id__ = 125
def __init__(self, message_id, timestamp, type, context_string, context, payload, url):
self.message_id = message_id
self.timestamp = timestamp
self.type = type
self.context_string = context_string
self.context = context
self.payload = payload
self.url = url
class SessionEnd(Message):
__id__ = 126

57
ee/connectors/msgcodec/msgcodec.py
View file

@ -76,28 +76,6 @@ class MessageCodec(Codec):
def read_head_message(self, reader: io.BytesIO, message_id) -> Message:
if message_id == 80:
return BatchMeta(
page_no=self.read_uint(reader),
first_index=self.read_uint(reader),
timestamp=self.read_int(reader)
)
if message_id == 81:
return BatchMetadata(
version=self.read_uint(reader),
page_no=self.read_uint(reader),
first_index=self.read_uint(reader),
timestamp=self.read_int(reader),
location=self.read_string(reader)
)
if message_id == 82:
return PartitionedMessage(
part_no=self.read_uint(reader),
part_total=self.read_uint(reader)
)
if message_id == 0:
return Timestamp(
timestamp=self.read_uint(reader)
@ -539,7 +517,7 @@ class MessageCodec(Codec):
)
if message_id == 62:
return IssueEvent(
return IssueEventDeprecated(
message_id=self.read_uint(reader),
timestamp=self.read_uint(reader),
type=self.read_string(reader),
@ -647,6 +625,39 @@ class MessageCodec(Codec):
metadata=self.read_string(reader)
)
if message_id == 80:
return BatchMeta(
page_no=self.read_uint(reader),
first_index=self.read_uint(reader),
timestamp=self.read_int(reader)
)
if message_id == 81:
return BatchMetadata(
version=self.read_uint(reader),
page_no=self.read_uint(reader),
first_index=self.read_uint(reader),
timestamp=self.read_int(reader),
location=self.read_string(reader)
)
if message_id == 82:
return PartitionedMessage(
part_no=self.read_uint(reader),
part_total=self.read_uint(reader)
)
if message_id == 125:
return IssueEvent(
message_id=self.read_uint(reader),
timestamp=self.read_uint(reader),
type=self.read_string(reader),
context_string=self.read_string(reader),
context=self.read_string(reader),
payload=self.read_string(reader),
url=self.read_string(reader)
)
if message_id == 126:
return SessionEnd(
timestamp=self.read_uint(reader),

32
frontend/app/player/web/messages/tracker.gen.ts
View file

@ -5,21 +5,6 @@ import type { RawMessage } from './raw.gen'
import { MType } from './raw.gen'
type TrBatchMetadata = [
type: 81,
version: number,
pageNo: number,
firstIndex: number,
timestamp: number,
location: string,
]
type TrPartitionedMessage = [
type: 82,
partNo: number,
partTotal: number,
]
type TrTimestamp = [
type: 0,
timestamp: number,
@ -423,8 +408,23 @@ type TrJSException = [
metadata: string,
]
type TrBatchMetadata = [
type: 81,
version: number,
pageNo: number,
firstIndex: number,
timestamp: number,
location: string,
]
export type TrackerMessage = TrBatchMetadata | TrPartitionedMessage | TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrJSExceptionDeprecated | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrZustand | TrJSException
type TrPartitionedMessage = [
type: 82,
partNo: number,
partTotal: number,
]
export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrJSExceptionDeprecated | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrZustand | TrJSException | TrBatchMetadata | TrPartitionedMessage
export default function translate(tMsg: TrackerMessage): RawMessage | null {
switch(tMsg[0]) {

66
mobs/messages.rb
View file

@ -1,27 +1,4 @@
# Special one for Batch Metadata. Message id could define the version
# Deprecated since tracker 3.6.0 in favor of BatchMetadata
message 80, 'BatchMeta', :replayer => false, :tracker => false do
uint 'PageNo'
uint 'FirstIndex'
int 'Timestamp'
end
# since tracker 3.6.0 TODO: for webworker only
message 81, 'BatchMetadata', :replayer => false do
uint 'Version'
uint 'PageNo'
uint 'FirstIndex'
int 'Timestamp'
string 'Location'
end
# since tracker 3.6.0
message 82, 'PartitionedMessage', :replayer => false do
uint 'PartNo'
uint 'PartTotal'
end
# OpenReplay messages definition
message 0, 'Timestamp' do
uint 'Timestamp'
@ -408,7 +385,7 @@ message 61, 'SetCSSDataURLBased' do
string 'Data'
string 'BaseURL'
end
message 62, 'IssueEvent', :replayer => false, :tracker => false do
message 62, 'IssueEventDeprecated', :replayer => false, :tracker => false do
uint 'MessageID'
uint 'Timestamp'
string 'Type'
@ -493,15 +470,48 @@ message 78, 'JSException', :replayer => false do
string 'Metadata'
end
# 80 -- 90 reserved
# Special one for Batch Metadata. Message id could define the version
# Deprecated since tracker 3.6.0 in favor of BatchMetadata
message 80, 'BatchMeta', :replayer => false, :tracker => false do
uint 'PageNo'
uint 'FirstIndex'
int 'Timestamp'
end
# since tracker 3.6.0 TODO: for webworker only
message 81, 'BatchMetadata', :replayer => false do
uint 'Version'
uint 'PageNo'
uint 'FirstIndex'
int 'Timestamp'
string 'Location'
end
# since tracker 3.6.0
message 82, 'PartitionedMessage', :replayer => false do
uint 'PartNo'
uint 'PartTotal'
end
message 125, 'IssueEvent', :replayer => false, :tracker => false do
uint 'MessageID'
uint 'Timestamp'
string 'Type'
string 'ContextString'
string 'Context'
string 'Payload'
string 'URL'
end
message 126, 'SessionEnd', :tracker => false, :replayer => false do
uint 'Timestamp'
string 'EncryptionKey'
end
message 127, 'SessionSearch', :tracker => false, :replayer => false do
uint 'Timestamp'
uint 'Partition'
end
# 80 -- 90 reserved

36
tracker/tracker/src/common/messages.gen.ts
View file

@ -2,8 +2,6 @@
/* eslint-disable */
export declare const enum Type {
BatchMetadata = 81,
PartitionedMessage = 82,
Timestamp = 0,
SetPageLocation = 4,
SetViewportSize = 5,
@ -62,24 +60,11 @@ export declare const enum Type {
AdoptedSSRemoveOwner = 77,
Zustand = 79,
JSException = 78,
BatchMetadata = 81,
PartitionedMessage = 82,
}
export type BatchMetadata = [
/*type:*/ Type.BatchMetadata,
/*version:*/ number,
/*pageNo:*/ number,
/*firstIndex:*/ number,
/*timestamp:*/ number,
/*location:*/ string,
]
export type PartitionedMessage = [
/*type:*/ Type.PartitionedMessage,
/*partNo:*/ number,
/*partTotal:*/ number,
]
export type Timestamp = [
/*type:*/ Type.Timestamp,
/*timestamp:*/ number,
@ -483,6 +468,21 @@ export type JSException = [
/*metadata:*/ string,
]
export type BatchMetadata = [
/*type:*/ Type.BatchMetadata,
/*version:*/ number,
/*pageNo:*/ number,
/*firstIndex:*/ number,
/*timestamp:*/ number,
/*location:*/ string,
]
type Message = BatchMetadata | PartitionedMessage | Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | JSExceptionDeprecated | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | Zustand | JSException
export type PartitionedMessage = [
/*type:*/ Type.PartitionedMessage,
/*partNo:*/ number,
/*partTotal:*/ number,
]
type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | JSExceptionDeprecated | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | Zustand | JSException | BatchMetadata | PartitionedMessage
export default Message

56
tracker/tracker/src/main/app/messages.gen.ts
View file

@ -5,34 +5,6 @@ import * as Messages from '../../common/messages.gen.js'
export { default, Type } from '../../common/messages.gen.js'
export function BatchMetadata(
version: number,
pageNo: number,
firstIndex: number,
timestamp: number,
location: string,
): Messages.BatchMetadata {
return [
Messages.Type.BatchMetadata,
version,
pageNo,
firstIndex,
timestamp,
location,
]
}
export function PartitionedMessage(
partNo: number,
partTotal: number,
): Messages.PartitionedMessage {
return [
Messages.Type.PartitionedMessage,
partNo,
partTotal,
]
}
export function Timestamp(
timestamp: number,
): Messages.Timestamp {
@ -781,3 +753,31 @@ export function JSException(
]
}
export function BatchMetadata(
version: number,
pageNo: number,
firstIndex: number,
timestamp: number,
location: string,
): Messages.BatchMetadata {
return [
Messages.Type.BatchMetadata,
version,
pageNo,
firstIndex,
timestamp,
location,
]
}
export function PartitionedMessage(
partNo: number,
partTotal: number,
): Messages.PartitionedMessage {
return [
Messages.Type.PartitionedMessage,
partNo,
partTotal,
]
}

16
tracker/tracker/src/webworker/MessageEncoder.gen.ts
View file

@ -10,14 +10,6 @@ export default class MessageEncoder extends PrimitiveEncoder {
encode(msg: Message): boolean {
switch(msg[0]) {
case Messages.Type.BatchMetadata:
return this.uint(msg[1]) && this.uint(msg[2]) && this.uint(msg[3]) && this.int(msg[4]) && this.string(msg[5])
break
case Messages.Type.PartitionedMessage:
return this.uint(msg[1]) && this.uint(msg[2])
break
case Messages.Type.Timestamp:
return this.uint(msg[1])
break
@ -250,6 +242,14 @@ export default class MessageEncoder extends PrimitiveEncoder {
return this.string(msg[1]) && this.string(msg[2]) && this.string(msg[3]) && this.string(msg[4])
break
case Messages.Type.BatchMetadata:
return this.uint(msg[1]) && this.uint(msg[2]) && this.uint(msg[3]) && this.int(msg[4]) && this.string(msg[5])
break
case Messages.Type.PartitionedMessage:
return this.uint(msg[1]) && this.uint(msg[2])
break
}
}