From ab279697a43699221e1615f2c8862105d5ffcfa5 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 6 Apr 2023 14:53:43 +0200 Subject: [PATCH] change(tracker): configure per message compression --- ...er~src~webworker~MessageEncoder.gen.ts.erb | 11 +- tracker/tracker/package.json | 3 +- tracker/tracker/src/main/app/index.ts | 9 +- .../src/webworker/MessageEncoder.gen.ts | 272 +++++++++++++++++- .../tracker/src/webworker/PrimitiveEncoder.ts | 10 + tracker/tracker/src/webworker/QueueSender.ts | 2 +- 6 files changed, 302 insertions(+), 5 deletions(-) diff --git a/mobs/templates/tracker~tracker~src~webworker~MessageEncoder.gen.ts.erb b/mobs/templates/tracker~tracker~src~webworker~MessageEncoder.gen.ts.erb index 337fc914e..636f73563 100644 --- a/mobs/templates/tracker~tracker~src~webworker~MessageEncoder.gen.ts.erb +++ b/mobs/templates/tracker~tracker~src~webworker~MessageEncoder.gen.ts.erb @@ -7,7 +7,16 @@ import PrimitiveEncoder from './PrimitiveEncoder.js' export default class MessageEncoder extends PrimitiveEncoder { - encode(msg: Message): boolean { + encode(msg: Message, isCompressed: boolean): boolean { + if (isCompressed) { + switch(msg[0]) { + <% $messages.select { |msg| msg.tracker }.each do |msg| %> + case Messages.Type.<%= msg.name %>: + return <% if msg.attributes.size == 0 %> true <% else %> <%= msg.attributes.map.with_index { |attr, index| "this.encodeCompressed(msg[#{index+1}])" }.join " && " %><% end %> + break + <% end %> + } + } switch(msg[0]) { <% $messages.select { |msg| msg.tracker }.each do |msg| %> case Messages.Type.<%= msg.name %>: diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 636582b10..67ffa66ae 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -48,7 +48,8 @@ }, "dependencies": { "@medv/finder": "^3.0.0", - "error-stack-parser": "^2.0.6" + "error-stack-parser": "^2.0.6", + "fflate": "^0.7.4" }, "engines": { "node": ">=14.0" diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 4c770f664..24a385a60 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -7,7 +7,7 @@ import Sanitizer from './sanitizer.js' import Ticker from './ticker.js' import Logger, { LogLevel } from './logger.js' import Session from './session.js' - +import { gzip, strToU8 } from 'fflate' import { deviceMemory, jsHeapSizeLimit } from '../modules/performance.js' import type { Options as ObserverOptions } from './observer/top_observer.js' @@ -208,6 +208,13 @@ export default class App { private _usingOldFetchPlugin = false send(message: Message, urgent = false): void { + // @ts-ignore + const compressedMessage = message.reduce((acc, curr, index) => { + if (index === 0) return curr + // @ts-ignore + return gzip(strToU8(curr.toString()), (err, data) => data) + }, []) + console.log(message, compressedMessage) if (this.activityState === ActivityState.NotActive) { return } diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 69fc7b35f..21773dbad 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -1,4 +1,5 @@ // Auto-generated, do not edit +// @ts-nocheck /* eslint-disable */ import * as Messages from '../common/messages.gen.js' @@ -7,7 +8,276 @@ import PrimitiveEncoder from './PrimitiveEncoder.js' export default class MessageEncoder extends PrimitiveEncoder { - encode(msg: Message): boolean { + encode(msg: Message, isCompressed?: boolean): boolean { + if (isCompressed) { + switch(msg[0]) { + + case Messages.Type.Timestamp: + return this.encodeCompressed(msg[1]) + break + + case Messages.Type.SetPageLocation: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.SetViewportSize: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.SetViewportScroll: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.CreateDocument: + return true + break + + case Messages.Type.CreateElementNode: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) && this.encodeCompressed(msg[5]) + break + + case Messages.Type.CreateTextNode: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.MoveNode: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.RemoveNode: + return this.encodeCompressed(msg[1]) + break + + case Messages.Type.SetNodeAttribute: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.RemoveNodeAttribute: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.SetNodeData: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.SetNodeScroll: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.SetInputTarget: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.SetInputValue: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.SetInputChecked: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.MouseMove: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.NetworkRequest: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) && this.encodeCompressed(msg[5]) && this.encodeCompressed(msg[6]) && this.encodeCompressed(msg[7]) && this.encodeCompressed(msg[8]) + break + + case Messages.Type.ConsoleLog: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.PageLoadTiming: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) && this.encodeCompressed(msg[5]) && this.encodeCompressed(msg[6]) && this.encodeCompressed(msg[7]) && this.encodeCompressed(msg[8]) && this.encodeCompressed(msg[9]) + break + + case Messages.Type.PageRenderTiming: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.CustomEvent: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.UserID: + return this.encodeCompressed(msg[1]) + break + + case Messages.Type.UserAnonymousID: + return this.encodeCompressed(msg[1]) + break + + case Messages.Type.Metadata: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.CSSInsertRule: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.CSSDeleteRule: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.Fetch: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) && this.encodeCompressed(msg[5]) && this.encodeCompressed(msg[6]) && this.encodeCompressed(msg[7]) + break + + case Messages.Type.Profiler: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) + break + + case Messages.Type.OTable: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.StateAction: + return this.encodeCompressed(msg[1]) + break + + case Messages.Type.Redux: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.Vuex: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.MobX: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.NgRx: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.GraphQL: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) + break + + case Messages.Type.PerformanceTrack: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) + break + + case Messages.Type.StringDict: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.SetNodeAttributeDict: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.ResourceTimingDeprecated: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) && this.encodeCompressed(msg[5]) && this.encodeCompressed(msg[6]) && this.encodeCompressed(msg[7]) && this.encodeCompressed(msg[8]) + break + + case Messages.Type.ConnectionInformation: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.SetPageVisibility: + return this.encodeCompressed(msg[1]) + break + + case Messages.Type.LoadFontFace: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) + break + + case Messages.Type.SetNodeFocus: + return this.encodeCompressed(msg[1]) + break + + case Messages.Type.LongTask: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) && this.encodeCompressed(msg[5]) && this.encodeCompressed(msg[6]) && this.encodeCompressed(msg[7]) + break + + case Messages.Type.SetNodeAttributeURLBased: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) + break + + case Messages.Type.SetCSSDataURLBased: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.TechnicalInfo: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.CustomIssue: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.CSSInsertRuleURLBased: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) + break + + case Messages.Type.MouseClick: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) + break + + case Messages.Type.CreateIFrameDocument: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.AdoptedSSReplaceURLBased: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.AdoptedSSInsertRuleURLBased: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) + break + + case Messages.Type.AdoptedSSDeleteRule: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.AdoptedSSAddOwner: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.AdoptedSSRemoveOwner: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.JSException: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) + break + + case Messages.Type.Zustand: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.BatchMetadata: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) && this.encodeCompressed(msg[5]) + break + + case Messages.Type.PartitionedMessage: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) + break + + case Messages.Type.InputChange: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) && this.encodeCompressed(msg[5]) && this.encodeCompressed(msg[6]) + break + + case Messages.Type.SelectionChange: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) + break + + case Messages.Type.MouseThrashing: + return this.encodeCompressed(msg[1]) + break + + case Messages.Type.UnbindNodes: + return this.encodeCompressed(msg[1]) + break + + case Messages.Type.ResourceTiming: + return this.encodeCompressed(msg[1]) && this.encodeCompressed(msg[2]) && this.encodeCompressed(msg[3]) && this.encodeCompressed(msg[4]) && this.encodeCompressed(msg[5]) && this.encodeCompressed(msg[6]) && this.encodeCompressed(msg[7]) && this.encodeCompressed(msg[8]) && this.encodeCompressed(msg[9]) && this.encodeCompressed(msg[10]) + break + + } + } switch(msg[0]) { case Messages.Type.Timestamp: diff --git a/tracker/tracker/src/webworker/PrimitiveEncoder.ts b/tracker/tracker/src/webworker/PrimitiveEncoder.ts index ac3e98a03..caf0b2db7 100644 --- a/tracker/tracker/src/webworker/PrimitiveEncoder.ts +++ b/tracker/tracker/src/webworker/PrimitiveEncoder.ts @@ -114,4 +114,14 @@ export default class PrimitiveEncoder { this.reset() return data } + + encodeCompressed(message: Uint8Array) { + const length = message.byteLength + if (!this.uint(length) || this.offset + length > this.size) { + return false + } + this.data.set(message, this.offset) + this.offset += length + return true + } } diff --git a/tracker/tracker/src/webworker/QueueSender.ts b/tracker/tracker/src/webworker/QueueSender.ts index d37d7c313..337aba2df 100644 --- a/tracker/tracker/src/webworker/QueueSender.ts +++ b/tracker/tracker/src/webworker/QueueSender.ts @@ -62,7 +62,7 @@ export default class QueueSender { body: batch, method: 'POST', headers: { - Authorization: 'Bearer ' + this.token, + Authorization: `Bearer ${this.token as string}`, //"Content-Type": "", }, keepalive: batch.length < KEEPALIVE_SIZE_LIMIT,