diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index e778f812c..a96f98de8 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -1320,15 +1320,15 @@ func (msg *PerformanceTrack) TypeID() int { type StringDict struct { message - Key string + Key uint64 Value string } func (msg *StringDict) Encode() []byte { - buf := make([]byte, 21+len(msg.Key)+len(msg.Value)) + buf := make([]byte, 21+len(msg.Value)) buf[0] = 50 p := 1 - p = WriteString(msg.Key, buf, p) + p = WriteUint(msg.Key, buf, p) p = WriteString(msg.Value, buf, p) return buf[:p] } @@ -1343,18 +1343,18 @@ func (msg *StringDict) TypeID() int { type SetNodeAttributeDict struct { message - ID uint64 - Name string - Value string + ID uint64 + NameKey uint64 + ValueKey uint64 } func (msg *SetNodeAttributeDict) Encode() []byte { - buf := make([]byte, 31+len(msg.Name)+len(msg.Value)) + buf := make([]byte, 31) buf[0] = 51 p := 1 p = WriteUint(msg.ID, buf, p) - p = WriteString(msg.Name, buf, p) - p = WriteString(msg.Value, buf, p) + p = WriteUint(msg.NameKey, buf, p) + p = WriteUint(msg.ValueKey, buf, p) return buf[:p] } diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 0531c7d1c..ecc00183f 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -795,7 +795,7 @@ func DecodePerformanceTrack(reader BytesReader) (Message, error) { func DecodeStringDict(reader BytesReader) (Message, error) { var err error = nil msg := &StringDict{} - if msg.Key, err = reader.ReadString(); err != nil { + if msg.Key, err = reader.ReadUint(); err != nil { return nil, err } if msg.Value, err = reader.ReadString(); err != nil { @@ -810,10 +810,10 @@ func DecodeSetNodeAttributeDict(reader BytesReader) (Message, error) { if msg.ID, err = reader.ReadUint(); err != nil { return nil, err } - if msg.Name, err = reader.ReadString(); err != nil { + if msg.NameKey, err = reader.ReadUint(); err != nil { return nil, err } - if msg.Value, err = reader.ReadString(); err != nil { + if msg.ValueKey, err = reader.ReadUint(); err != nil { return nil, err } return msg, err diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 1df1b06a8..54f8df955 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -464,10 +464,10 @@ class StringDict(Message): class SetNodeAttributeDict(Message): __id__ = 51 - def __init__(self, id, name, value): + def __init__(self, id, name_key, value_key): self.id = id - self.name = name - self.value = value + self.name_key = name_key + self.value_key = value_key class DOMDrop(Message): diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 36d297998..0ba21ea12 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -433,15 +433,15 @@ class MessageCodec(Codec): if message_id == 50: return StringDict( - key=self.read_string(reader), + key=self.read_uint(reader), value=self.read_string(reader) ) if message_id == 51: return SetNodeAttributeDict( id=self.read_uint(reader), - name=self.read_string(reader), - value=self.read_string(reader) + name_key=self.read_uint(reader), + value_key=self.read_uint(reader) ) if message_id == 52: diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index 198c096ad..bb4c999ae 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -43,6 +43,7 @@ export default class DOMManager extends ListWalker { private activeIframeRoots: Map = new Map() private styleSheets: Map = new Map() private ppStyleSheets: Map = new Map() + private stringDict: Record = {} private upperBodyId: number = -1; @@ -134,6 +135,31 @@ export default class DOMManager extends ListWalker { parent.insertChildAt(child, index) } + private setNodeAttribute(msg: { id: number, name: string, value: string }) { + let { name, value } = msg; + const vn = this.vElements.get(msg.id) + if (!vn) { logger.error("Node not found", msg); return } + if (vn.node.tagName === "INPUT" && name === "name") { + // Otherwise binds local autocomplete values (maybe should ignore on the tracker level) + return + } + if (name === "href" && vn.node.tagName === "LINK") { + // @ts-ignore ?global ENV type // It've been done on backend (remove after testing in saas) + // if (value.startsWith(window.env.ASSETS_HOST || window.location.origin + '/assets')) { + // value = value.replace("?", "%3F"); + // } + if (!value.startsWith("http")) { return } + // blob:... value happened here. https://foss.openreplay.com/3/session/7013553567419137 + // that resulted in that link being unable to load and having 4sec timeout in the below function. + this.stylesManager.setStyleHandlers(vn.node as HTMLLinkElement, value); + } + if (vn.node.namespaceURI === 'http://www.w3.org/2000/svg' && value.startsWith("url(")) { + value = "url(#" + (value.split("#")[1] ||")") + } + vn.setAttribute(name, value) + this.removeBodyScroll(msg.id, vn) + } + private applyMessage = (msg: Message): Promise | undefined => { let node: Node | undefined let vn: VNode | undefined @@ -160,10 +186,11 @@ export default class DOMManager extends ListWalker { this.vRoots.clear() this.vRoots.set(0, vDoc) // watchout: id==0 for both Document and documentElement // this is done for the AdoptedCSS logic - // todo: start from 0 (sync logic with tracker) + // todo: start from 0-node (sync logic with tracker) this.vTexts.clear() this.stylesManager.reset() this.activeIframeRoots.clear() + this.stringDict = {} return case MType.CreateTextNode: vn = new VText() @@ -200,28 +227,20 @@ export default class DOMManager extends ListWalker { vn.parentNode.removeChild(vn) return case MType.SetNodeAttribute: - let { name, value } = msg; - vn = this.vElements.get(msg.id) - if (!vn) { logger.error("Node not found", msg); return } - if (vn.node.tagName === "INPUT" && name === "name") { - // Otherwise binds local autocomplete values (maybe should ignore on the tracker level) - return - } - if (name === "href" && vn.node.tagName === "LINK") { - // @ts-ignore ?global ENV type // It've been done on backend (remove after testing in saas) - // if (value.startsWith(window.env.ASSETS_HOST || window.location.origin + '/assets')) { - // value = value.replace("?", "%3F"); - // } - if (!value.startsWith("http")) { return } - // blob:... value happened here. https://foss.openreplay.com/3/session/7013553567419137 - // that resulted in that link being unable to load and having 4sec timeout in the below function. - this.stylesManager.setStyleHandlers(vn.node as HTMLLinkElement, value); - } - if (vn.node.namespaceURI === 'http://www.w3.org/2000/svg' && value.startsWith("url(")) { - value = "url(#" + (value.split("#")[1] ||")") - } - vn.setAttribute(name, value) - this.removeBodyScroll(msg.id, vn) + this.setNodeAttribute(msg) + return + case MType.StringDict: + this.stringDict[msg.key] = msg.value + return + case MType.SetNodeAttributeDict: + this.stringDict[msg.nameKey] === undefined && logger.error("No dictionary key for msg 'name': ", msg) + this.stringDict[msg.valueKey] === undefined && logger.error("No dictionary key for msg 'value': ", msg) + if (this.stringDict[msg.nameKey] === undefined || this.stringDict[msg.valueKey] === undefined ) { return } + this.setNodeAttribute({ + id: msg.id, + name: this.stringDict[msg.nameKey], + value: this.stringDict[msg.valueKey], + }) return case MType.RemoveNodeAttribute: vn = this.vElements.get(msg.id) diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index f76f9fb36..793f609f5 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -372,7 +372,7 @@ export default class RawMessageReader extends PrimitiveReader { } case 50: { - const key = this.readString(); if (key === null) { return resetPointer() } + const key = this.readUint(); if (key === null) { return resetPointer() } const value = this.readString(); if (value === null) { return resetPointer() } return { tp: MType.StringDict, @@ -383,13 +383,13 @@ export default class RawMessageReader extends PrimitiveReader { case 51: { const id = this.readUint(); if (id === null) { return resetPointer() } - const name = this.readString(); if (name === null) { return resetPointer() } - const value = this.readString(); if (value === null) { return resetPointer() } + const nameKey = this.readUint(); if (nameKey === null) { return resetPointer() } + const valueKey = this.readUint(); if (valueKey === null) { return resetPointer() } return { tp: MType.SetNodeAttributeDict, id, - name, - value, + nameKey, + valueKey, }; } diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index 517bdc834..b51edb40e 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -271,15 +271,15 @@ export interface RawPerformanceTrack { export interface RawStringDict { tp: MType.StringDict, - key: string, + key: number, value: string, } export interface RawSetNodeAttributeDict { tp: MType.SetNodeAttributeDict, id: number, - name: string, - value: string, + nameKey: number, + valueKey: number, } export interface RawResourceTiming { diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index 220c4f3da..2c171011c 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -260,15 +260,15 @@ type TrPerformanceTrack = [ type TrStringDict = [ type: 50, - key: string, + key: number, value: string, ] type TrSetNodeAttributeDict = [ type: 51, id: number, - name: string, - value: string, + nameKey: number, + valueKey: number, ] type TrResourceTiming = [ @@ -705,8 +705,8 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { return { tp: MType.SetNodeAttributeDict, id: tMsg[1], - name: tMsg[2], - value: tMsg[3], + nameKey: tMsg[2], + valueKey: tMsg[3], } } diff --git a/mobs/messages.rb b/mobs/messages.rb index ab1b29c00..ef36ebfa7 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -279,14 +279,14 @@ message 49, 'PerformanceTrack' do #, :replayer => :devtools --> requires player end # since 4.1.9 message 50, "StringDict" do - string "Key" + uint "Key" string "Value" end # since 4.1.9 message 51, "SetNodeAttributeDict" do uint 'ID' - string 'Name' - string 'Value' + uint 'NameKey' + uint 'ValueKey' end ## 50,51 diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 66a1c5c7b..8f61ab917 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -321,15 +321,15 @@ export type PerformanceTrack = [ export type StringDict = [ /*type:*/ Type.StringDict, - /*key:*/ string, + /*key:*/ number, /*value:*/ string, ] export type SetNodeAttributeDict = [ /*type:*/ Type.SetNodeAttributeDict, /*id:*/ number, - /*name:*/ string, - /*value:*/ string, + /*nameKey:*/ number, + /*valueKey:*/ number, ] export type ResourceTiming = [ diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index d3a7c3b08..bdec57a0f 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -475,7 +475,7 @@ export function PerformanceTrack( } export function StringDict( - key: string, + key: number, value: string, ): Messages.StringDict { return [ @@ -487,14 +487,14 @@ export function StringDict( export function SetNodeAttributeDict( id: number, - name: string, - value: string, + nameKey: number, + valueKey: number, ): Messages.SetNodeAttributeDict { return [ Messages.Type.SetNodeAttributeDict, id, - name, - value, + nameKey, + valueKey, ] } diff --git a/tracker/tracker/src/webworker/BatchWriter.ts b/tracker/tracker/src/webworker/BatchWriter.ts index 167c1aff6..d9275ada3 100644 --- a/tracker/tracker/src/webworker/BatchWriter.ts +++ b/tracker/tracker/src/webworker/BatchWriter.ts @@ -1,6 +1,7 @@ import type Message from '../common/messages.gen.js' import * as Messages from '../common/messages.gen.js' import MessageEncoder from './MessageEncoder.gen.js' +import StringDictionary from './StringDictionary.js' const SIZE_BYTES = 3 const MAX_M_SIZE = (1 << (SIZE_BYTES * 8)) - 1 @@ -9,6 +10,7 @@ export default class BatchWriter { private nextIndex = 0 private beaconSize = 2 * 1e5 // Default 200kB private encoder = new MessageEncoder(this.beaconSize) + private strDict = new StringDictionary() private readonly sizeBuffer = new Uint8Array(SIZE_BYTES) private isEmpty = true @@ -84,6 +86,14 @@ export default class BatchWriter { this.beaconSizeLimit = limit } + private applyDict(str: string): number { + const [key, isNew] = this.strDict.getKey(str) + if (isNew) { + this.writeMessage([Messages.Type.StringDict, key, str]) + } + return key + } + writeMessage(message: Message) { if (message[0] === Messages.Type.Timestamp) { this.timestamp = message[1] // .timestamp @@ -91,6 +101,14 @@ export default class BatchWriter { if (message[0] === Messages.Type.SetPageLocation) { this.url = message[1] // .url } + if (message[0] === Messages.Type.SetNodeAttribute) { + message = [ + Messages.Type.SetNodeAttributeDict, + message[1], + this.applyDict(message[2]), + this.applyDict(message[3]), + ] as Messages.SetNodeAttributeDict + } if (this.writeWithSize(message)) { return } diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 5bd3f6a01..e6e522dd4 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -159,11 +159,11 @@ export default class MessageEncoder extends PrimitiveEncoder { break case Messages.Type.StringDict: - return this.string(msg[1]) && this.string(msg[2]) + return this.uint(msg[1]) && this.string(msg[2]) break case Messages.Type.SetNodeAttributeDict: - return this.uint(msg[1]) && this.string(msg[2]) && this.string(msg[3]) + return this.uint(msg[1]) && this.uint(msg[2]) && this.uint(msg[3]) break case Messages.Type.ResourceTiming: diff --git a/tracker/tracker/src/webworker/StringDictionary.ts b/tracker/tracker/src/webworker/StringDictionary.ts new file mode 100644 index 000000000..b183ce862 --- /dev/null +++ b/tracker/tracker/src/webworker/StringDictionary.ts @@ -0,0 +1,13 @@ +export default class StringDictionary { + private idx = 1 + private backDict: Record = {} + + getKey(str: string): [number, boolean] { + let isNew = false + if (!this.backDict[str]) { + isNew = true + this.backDict[str] = this.idx++ + } + return [this.backDict[str], isNew] + } +} diff --git a/tracker/tracker/src/webworker/StringDictionary.unit.test.ts b/tracker/tracker/src/webworker/StringDictionary.unit.test.ts new file mode 100644 index 000000000..8bf17e14b --- /dev/null +++ b/tracker/tracker/src/webworker/StringDictionary.unit.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, test, jest, beforeEach, afterEach } from '@jest/globals' +import StringDictionary from './StringDictionary.js' + +describe('StringDictionary', () => { + test('key is non-zero', () => { + const dict = new StringDictionary() + + const [key, isNew] = dict.getKey('We are Asayer') + + expect(key).not.toBe(0) + expect(isNew).toBe(true) + }) + + test('Different strings have different keys', () => { + const dict = new StringDictionary() + + const [key1, isNew1] = dict.getKey('Datadog') + const [key2, isNew2] = dict.getKey('PostHog') + + expect(key1).not.toBe(key2) + expect(isNew2).toBe(true) + }) + + test('Similar strings have similar keys', () => { + const dict = new StringDictionary() + + const [key1, isNew1] = dict.getKey("What's up?") + const [key2, isNew2] = dict.getKey("What's up?") + + expect(key1).toBe(key2) + expect(isNew2).toBe(false) + }) +})