jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #966 from openreplay/attr-string-dict

Browse source 
string dict for attributes
This commit is contained in:
Alex K 2023-02-07 14:47:23 +01:00 • committed by GitHub
commit dbb0e27c0c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 150 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
backend/pkg/messages/messages.go
View file

@ -1320,15 +1320,15 @@ func (msg *PerformanceTrack) TypeID() int {
type StringDict struct {
message
Key string
Key uint64
Value string
}
func (msg *StringDict) Encode() []byte {
buf := make([]byte, 21+len(msg.Key)+len(msg.Value))
buf := make([]byte, 21+len(msg.Value))
buf[0] = 50
p := 1
p = WriteString(msg.Key, buf, p)
p = WriteUint(msg.Key, buf, p)
p = WriteString(msg.Value, buf, p)
return buf[:p]
}
@ -1343,18 +1343,18 @@ func (msg *StringDict) TypeID() int {
type SetNodeAttributeDict struct {
message
ID uint64
Name string
Value string
ID uint64
NameKey uint64
ValueKey uint64
}
func (msg *SetNodeAttributeDict) Encode() []byte {
buf := make([]byte, 31+len(msg.Name)+len(msg.Value))
buf := make([]byte, 31)
buf[0] = 51
p := 1
p = WriteUint(msg.ID, buf, p)
p = WriteString(msg.Name, buf, p)
p = WriteString(msg.Value, buf, p)
p = WriteUint(msg.NameKey, buf, p)
p = WriteUint(msg.ValueKey, buf, p)
return buf[:p]
}

6
backend/pkg/messages/read-message.go
View file

@ -795,7 +795,7 @@ func DecodePerformanceTrack(reader BytesReader) (Message, error) {
func DecodeStringDict(reader BytesReader) (Message, error) {
var err error = nil
msg := &StringDict{}
if msg.Key, err = reader.ReadString(); err != nil {
if msg.Key, err = reader.ReadUint(); err != nil {
return nil, err
}
if msg.Value, err = reader.ReadString(); err != nil {
@ -810,10 +810,10 @@ func DecodeSetNodeAttributeDict(reader BytesReader) (Message, error) {
if msg.ID, err = reader.ReadUint(); err != nil {
return nil, err
}
if msg.Name, err = reader.ReadString(); err != nil {
if msg.NameKey, err = reader.ReadUint(); err != nil {
return nil, err
}
if msg.Value, err = reader.ReadString(); err != nil {
if msg.ValueKey, err = reader.ReadUint(); err != nil {
return nil, err
}
return msg, err

6
ee/connectors/msgcodec/messages.py
View file

@ -464,10 +464,10 @@ class StringDict(Message):
class SetNodeAttributeDict(Message):
__id__ = 51
def __init__(self, id, name, value):
def __init__(self, id, name_key, value_key):
self.id = id
self.name = name
self.value = value
self.name_key = name_key
self.value_key = value_key
class DOMDrop(Message):

6
ee/connectors/msgcodec/msgcodec.py
View file

@ -433,15 +433,15 @@ class MessageCodec(Codec):
if message_id == 50:
return StringDict(
key=self.read_string(reader),
key=self.read_uint(reader),
value=self.read_string(reader)
)
if message_id == 51:
return SetNodeAttributeDict(
id=self.read_uint(reader),
name=self.read_string(reader),
value=self.read_string(reader)
name_key=self.read_uint(reader),
value_key=self.read_uint(reader)
)
if message_id == 52:

65
frontend/app/player/web/managers/DOM/DOMManager.ts
View file

@ -43,6 +43,7 @@ export default class DOMManager extends ListWalker<Message> {
private activeIframeRoots: Map<number, number> = new Map()
private styleSheets: Map<number, CSSStyleSheet> = new Map()
private ppStyleSheets: Map<number, PostponedStyleSheet> = new Map()
private stringDict: Record<number,string> = {}
private upperBodyId: number = -1;
@ -134,6 +135,31 @@ export default class DOMManager extends ListWalker<Message> {
parent.insertChildAt(child, index)
}
private setNodeAttribute(msg: { id: number, name: string, value: string }) {
let { name, value } = msg;
const vn = this.vElements.get(msg.id)
if (!vn) { logger.error("Node not found", msg); return }
if (vn.node.tagName === "INPUT" && name === "name") {
// Otherwise binds local autocomplete values (maybe should ignore on the tracker level)
return
}
if (name === "href" && vn.node.tagName === "LINK") {
// @ts-ignore ?global ENV type // It've been done on backend (remove after testing in saas)
// if (value.startsWith(window.env.ASSETS_HOST || window.location.origin + '/assets')) {
// value = value.replace("?", "%3F");
// }
if (!value.startsWith("http")) { return }
// blob:... value happened here. https://foss.openreplay.com/3/session/7013553567419137
// that resulted in that link being unable to load and having 4sec timeout in the below function.
this.stylesManager.setStyleHandlers(vn.node as HTMLLinkElement, value);
}
if (vn.node.namespaceURI === 'http://www.w3.org/2000/svg' && value.startsWith("url(")) {
value = "url(#" + (value.split("#")[1] ||")")
}
vn.setAttribute(name, value)
this.removeBodyScroll(msg.id, vn)
}
private applyMessage = (msg: Message): Promise<any> | undefined => {
let node: Node | undefined
let vn: VNode | undefined
@ -160,10 +186,11 @@ export default class DOMManager extends ListWalker<Message> {
this.vRoots.clear()
this.vRoots.set(0, vDoc) // watchout: id==0 for both Document and documentElement
// this is done for the AdoptedCSS logic
// todo: start from 0 (sync logic with tracker)
// todo: start from 0-node (sync logic with tracker)
this.vTexts.clear()
this.stylesManager.reset()
this.activeIframeRoots.clear()
this.stringDict = {}
return
case MType.CreateTextNode:
vn = new VText()
@ -200,28 +227,20 @@ export default class DOMManager extends ListWalker<Message> {
vn.parentNode.removeChild(vn)
return
case MType.SetNodeAttribute:
let { name, value } = msg;
vn = this.vElements.get(msg.id)
if (!vn) { logger.error("Node not found", msg); return }
if (vn.node.tagName === "INPUT" && name === "name") {
// Otherwise binds local autocomplete values (maybe should ignore on the tracker level)
return
}
if (name === "href" && vn.node.tagName === "LINK") {
// @ts-ignore ?global ENV type // It've been done on backend (remove after testing in saas)
// if (value.startsWith(window.env.ASSETS_HOST || window.location.origin + '/assets')) {
// value = value.replace("?", "%3F");
// }
if (!value.startsWith("http")) { return }
// blob:... value happened here. https://foss.openreplay.com/3/session/7013553567419137
// that resulted in that link being unable to load and having 4sec timeout in the below function.
this.stylesManager.setStyleHandlers(vn.node as HTMLLinkElement, value);
}
if (vn.node.namespaceURI === 'http://www.w3.org/2000/svg' && value.startsWith("url(")) {
value = "url(#" + (value.split("#")[1] ||")")
}
vn.setAttribute(name, value)
this.removeBodyScroll(msg.id, vn)
this.setNodeAttribute(msg)
return
case MType.StringDict:
this.stringDict[msg.key] = msg.value
return
case MType.SetNodeAttributeDict:
this.stringDict[msg.nameKey] === undefined && logger.error("No dictionary key for msg 'name': ", msg)
this.stringDict[msg.valueKey] === undefined && logger.error("No dictionary key for msg 'value': ", msg)
if (this.stringDict[msg.nameKey] === undefined || this.stringDict[msg.valueKey] === undefined ) { return }
this.setNodeAttribute({
id: msg.id,
name: this.stringDict[msg.nameKey],
value: this.stringDict[msg.valueKey],
})
return
case MType.RemoveNodeAttribute:
vn = this.vElements.get(msg.id)

10
frontend/app/player/web/messages/RawMessageReader.gen.ts
View file

@ -372,7 +372,7 @@ export default class RawMessageReader extends PrimitiveReader {
}
case 50: {
const key = this.readString(); if (key === null) { return resetPointer() }
const key = this.readUint(); if (key === null) { return resetPointer() }
const value = this.readString(); if (value === null) { return resetPointer() }
return {
tp: MType.StringDict,
@ -383,13 +383,13 @@ export default class RawMessageReader extends PrimitiveReader {
case 51: {
const id = this.readUint(); if (id === null) { return resetPointer() }
const name = this.readString(); if (name === null) { return resetPointer() }
const value = this.readString(); if (value === null) { return resetPointer() }
const nameKey = this.readUint(); if (nameKey === null) { return resetPointer() }
const valueKey = this.readUint(); if (valueKey === null) { return resetPointer() }
return {
tp: MType.SetNodeAttributeDict,
id,
name,
value,
nameKey,
valueKey,
};
}

6
frontend/app/player/web/messages/raw.gen.ts
View file

@ -271,15 +271,15 @@ export interface RawPerformanceTrack {
export interface RawStringDict {
tp: MType.StringDict,
key: string,
key: number,
value: string,
}
export interface RawSetNodeAttributeDict {
tp: MType.SetNodeAttributeDict,
id: number,
name: string,
value: string,
nameKey: number,
valueKey: number,
}
export interface RawResourceTiming {

10
frontend/app/player/web/messages/tracker.gen.ts
View file

@ -260,15 +260,15 @@ type TrPerformanceTrack = [
type TrStringDict = [
type: 50,
key: string,
key: number,
value: string,
]
type TrSetNodeAttributeDict = [
type: 51,
id: number,
name: string,
value: string,
nameKey: number,
valueKey: number,
]
type TrResourceTiming = [
@ -705,8 +705,8 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null {
return {
tp: MType.SetNodeAttributeDict,
id: tMsg[1],
name: tMsg[2],
value: tMsg[3],
nameKey: tMsg[2],
valueKey: tMsg[3],
}
}

6
mobs/messages.rb
View file

@ -279,14 +279,14 @@ message 49, 'PerformanceTrack' do #, :replayer => :devtools --> requires player
end
# since 4.1.9
message 50, "StringDict" do
string "Key"
uint "Key"
string "Value"
end
# since 4.1.9
message 51, "SetNodeAttributeDict" do
uint 'ID'
string 'Name'
string 'Value'
uint 'NameKey'
uint 'ValueKey'
end
## 50,51

6
tracker/tracker/src/common/messages.gen.ts
View file

@ -321,15 +321,15 @@ export type PerformanceTrack = [
export type StringDict = [
/*type:*/ Type.StringDict,
/*key:*/ string,
/*key:*/ number,
/*value:*/ string,
]
export type SetNodeAttributeDict = [
/*type:*/ Type.SetNodeAttributeDict,
/*id:*/ number,
/*name:*/ string,
/*value:*/ string,
/*nameKey:*/ number,
/*valueKey:*/ number,
]
export type ResourceTiming = [

10
tracker/tracker/src/main/app/messages.gen.ts
View file

@ -475,7 +475,7 @@ export function PerformanceTrack(
}
export function StringDict(
key: string,
key: number,
value: string,
): Messages.StringDict {
return [
@ -487,14 +487,14 @@ export function StringDict(
export function SetNodeAttributeDict(
id: number,
name: string,
value: string,
nameKey: number,
valueKey: number,
): Messages.SetNodeAttributeDict {
return [
Messages.Type.SetNodeAttributeDict,
id,
name,
value,
nameKey,
valueKey,
]
}

18
tracker/tracker/src/webworker/BatchWriter.ts
View file

@ -1,6 +1,7 @@
import type Message from '../common/messages.gen.js'
import * as Messages from '../common/messages.gen.js'
import MessageEncoder from './MessageEncoder.gen.js'
import StringDictionary from './StringDictionary.js'
const SIZE_BYTES = 3
const MAX_M_SIZE = (1 << (SIZE_BYTES * 8)) - 1
@ -9,6 +10,7 @@ export default class BatchWriter {
private nextIndex = 0
private beaconSize = 2 * 1e5 // Default 200kB
private encoder = new MessageEncoder(this.beaconSize)
private strDict = new StringDictionary()
private readonly sizeBuffer = new Uint8Array(SIZE_BYTES)
private isEmpty = true
@ -84,6 +86,14 @@ export default class BatchWriter {
this.beaconSizeLimit = limit
}
private applyDict(str: string): number {
const [key, isNew] = this.strDict.getKey(str)
if (isNew) {
this.writeMessage([Messages.Type.StringDict, key, str])
}
return key
}
writeMessage(message: Message) {
if (message[0] === Messages.Type.Timestamp) {
this.timestamp = message[1] // .timestamp
@ -91,6 +101,14 @@ export default class BatchWriter {
if (message[0] === Messages.Type.SetPageLocation) {
this.url = message[1] // .url
}
if (message[0] === Messages.Type.SetNodeAttribute) {
message = [
Messages.Type.SetNodeAttributeDict,
message[1],
this.applyDict(message[2]),
this.applyDict(message[3]),
] as Messages.SetNodeAttributeDict
}
if (this.writeWithSize(message)) {
return
}

4
tracker/tracker/src/webworker/MessageEncoder.gen.ts
View file

@ -159,11 +159,11 @@ export default class MessageEncoder extends PrimitiveEncoder {
break
case Messages.Type.StringDict:
return this.string(msg[1]) && this.string(msg[2])
return this.uint(msg[1]) && this.string(msg[2])
break
case Messages.Type.SetNodeAttributeDict:
return this.uint(msg[1]) && this.string(msg[2]) && this.string(msg[3])
return this.uint(msg[1]) && this.uint(msg[2]) && this.uint(msg[3])
break
case Messages.Type.ResourceTiming:

13
tracker/tracker/src/webworker/StringDictionary.ts Normal file
View file

@ -0,0 +1,13 @@
export default class StringDictionary {
private idx = 1
private backDict: Record<string, number> = {}
getKey(str: string): [number, boolean] {
let isNew = false
if (!this.backDict[str]) {
isNew = true
this.backDict[str] = this.idx++
}
return [this.backDict[str], isNew]
}
}

33
tracker/tracker/src/webworker/StringDictionary.unit.test.ts Normal file
View file

@ -0,0 +1,33 @@
import { describe, expect, test, jest, beforeEach, afterEach } from '@jest/globals'
import StringDictionary from './StringDictionary.js'
describe('StringDictionary', () => {
test('key is non-zero', () => {
const dict = new StringDictionary()
const [key, isNew] = dict.getKey('We are Asayer')
expect(key).not.toBe(0)
expect(isNew).toBe(true)
})
test('Different strings have different keys', () => {
const dict = new StringDictionary()
const [key1, isNew1] = dict.getKey('Datadog')
const [key2, isNew2] = dict.getKey('PostHog')
expect(key1).not.toBe(key2)
expect(isNew2).toBe(true)
})
test('Similar strings have similar keys', () => {
const dict = new StringDictionary()
const [key1, isNew1] = dict.getKey("What's up?")
const [key2, isNew2] = dict.getKey("What's up?")
expect(key1).toBe(key2)
expect(isNew2).toBe(false)
})
})