fix (tracker): 3.0.5: wbworker beacon size; handle overflow

tracker/tracker/package-lock.json

@@ -1,6 +1,6 @@
 {
-  "name": "@asayerio/tracker",
-  "version": "5.6.5",
+  "name": "@openreplay/tracker",
+  "version": "3.1.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -4513,9 +4513,9 @@
       "dev": true
     },
     "typescript": {
-      "version": "3.9.5",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.5.tgz",
-      "integrity": "sha512-hSAifV3k+i6lEoCJ2k6R2Z/rp/H3+8sdmcn5NrS3/3kE7+RyZXm9aqvxWqjEXHAd8b0pShatpcdMTvEdvAJltQ==",
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz",
+      "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==",
       "dev": true
     },
     "unc-path-regex": {

tracker/tracker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openreplay/tracker",
   "description": "The OpenReplay tracker main package",
-  "version": "3.0.4",
+  "version": "3.0.5",
   "keywords": [
     "logging",
     "replay"
@@ -18,9 +18,9 @@
     "clean": "rm -Rf build && rm -Rf lib && rm -Rf cjs",
     "tsc": "tsc -b src/main && tsc -b src/webworker && tsc --project src/main/tsconfig-cjs.json",
     "rollup": "rollup --config rollup.config.js",
-    "compile": "node --experimental-modules --experimental-json-modules compile.js",
+    "compile": "node --experimental-modules --experimental-json-modules scripts/compile.js",
     "build": "npm run clean && npm run tsc && npm run rollup && npm run compile",
-    "prepare": "node checkver.cjs && npm run build"
+    "prepare": "node scripts/checkver.cjs && npm run build"
   },
   "devDependencies": {
     "@babel/core": "^7.10.2",
@@ -38,7 +38,7 @@
     "rollup": "^2.17.0",
     "rollup-plugin-terser": "^6.1.0",
     "semver": "^6.3.0",
-    "typescript": "^3.9.5"
+    "typescript": "^4.3.4"
   },
   "dependencies": {
     "error-stack-parser": "^2.0.6"

tracker/tracker/src/main/modules/performance.ts

@@ -2,13 +2,19 @@ import App from '../app';
 import { IN_BROWSER } from '../utils';
 import { PerformanceTrack } from '../../messages';
 
-const perf: {
-  memory: {
-    jsHeapSizeLimit?: number;
-    totalJSHeapSize?: number;
-    usedJSHeapSize?: number;
-  };
-} = IN_BROWSER && 'memory' in performance ? performance : { memory: {} };
+
+type Perf = {
+  memory: { 
+    totalJSHeapSize?: number,
+    usedJSHeapSize?: number,
+    jsHeapSizeLimit?: number,
+  }
+}
+
+const perf: Perf = IN_BROWSER && 'memory' in performance // works in Chrome only
+  ? performance as any
+  : { memory: {} }
+
 
 export const deviceMemory = IN_BROWSER ? ((navigator as any).deviceMemory || 0) * 1024 : 0;
 export const jsHeapSizeLimit = perf.memory.jsHeapSizeLimit || 0;

tracker/tracker/src/messages/webworker.ts

@@ -1,6 +1,8 @@
+// TODO: "common" folder instead of "messages". (better file structure)
 export interface Options {
   connAttemptCount?: number;
   connAttemptGap?: number;
+  beaconSize?: number;
 }
 
 type Settings = {
@@ -11,4 +13,4 @@ type Settings = {
   timeAdjustment?: number;
 } & Partial<Options>;
 
-export type MessageData = null | "stop" | Settings | Array<{ _id: number }>;
+export type WorkerMessageData = null | "stop" | Settings | Array<{ _id: number }>;

tracker/tracker/src/webworker/index.ts

@@ -2,13 +2,15 @@ import { classes, BatchMeta, Timestamp, SetPageVisibility, CreateDocument } from
 import Message from '../messages/message';
 import Writer from '../messages/writer';
 
-import type { MessageData } from './types';
+import type { WorkerMessageData } from '../messages/webworker';
+
 
-// TODO: what if on message overflows? (maybe one option)
-const MAX_BATCH_SIZE = 4 * 1e5; // Max 400kB  
 const SEND_INTERVAL = 20 * 1000;
+const BEACON_SIZE_LIMIT = 1e6 // Limit is set in the backend/services/http
+let beaconSize = 4 * 1e5; // Default 400kB
 
-const writer: Writer = new Writer(MAX_BATCH_SIZE);
+
+let writer: Writer = new Writer(beaconSize);
 
 let ingestPoint: string = "";
 let token: string = "";
@@ -31,9 +33,12 @@ let busy = false;
 let attemptsCount = 0;
 let ATTEMPT_TIMEOUT = 8000;
 let MAX_ATTEMPTS_COUNT = 10;
+
+// TODO?: exploit https://developer.mozilla.org/en-US/docs/Web/API/Navigator/sendBeacon
 function sendBatch(batch: Uint8Array):void {
   const req = new XMLHttpRequest();
-  req.open("POST", ingestPoint + "/v1/web/i");  // TODO opaque request?
+  // TODO:  async=false (3d param) instead of sendQueue array ?
+  req.open("POST", ingestPoint + "/v1/web/i", false);  // TODO opaque request?
   req.setRequestHeader("Authorization", "Bearer " + token);
   // req.setRequestHeader("Content-Type", "");
   req.onreadystatechange = function() {
@@ -41,7 +46,7 @@ function sendBatch(batch: Uint8Array):void {
       if (this.status == 0) {
         return; // happens simultaneously with onerror TODO: clear codeflow
       }
-      if (this.status >= 400) {
+      if (this.status >= 400) { // TODO: test workflow. After 400+ it calls /start for some reason
         reset();
         sendQueue.length = 0;
         if (this.status === 403) { // Unauthorised (Token expired)
@@ -69,7 +74,7 @@ function sendBatch(batch: Uint8Array):void {
     attemptsCount++;
     setTimeout(() => sendBatch(batch), ATTEMPT_TIMEOUT); 
   }
-  req.send(batch);
+  req.send(batch.buffer);
 }
 
 function send(): void {
@@ -100,7 +105,7 @@ function hasTimestamp(msg: any): msg is { timestamp: number } {
   return typeof msg === 'object' && typeof msg.timestamp === 'number';
 }
 
-self.onmessage = ({ data }: MessageEvent) => {
+self.onmessage = ({ data }: MessageEvent<WorkerMessageData>) => {
   if (data === null) {
     send();
     return;
@@ -118,6 +123,7 @@ self.onmessage = ({ data }: MessageEvent) => {
     timeAdjustment = data.timeAdjustment || timeAdjustment;
     MAX_ATTEMPTS_COUNT = data.connAttemptCount || MAX_ATTEMPTS_COUNT;
     ATTEMPT_TIMEOUT = data.connAttemptGap || ATTEMPT_TIMEOUT;
+    beaconSize = Math.min(BEACON_SIZE_LIMIT, data.beaconSize || beaconSize);
     if (writer.isEmpty()) {
       writeBatchMeta();
     }
@@ -126,7 +132,7 @@ self.onmessage = ({ data }: MessageEvent) => {
     }
     return;
   }
-  data.forEach((data: any) => {
+  data.forEach((data) => {
     const message: Message = new (<any>classes.get(data._id))();
     Object.assign(message, data);
 
@@ -140,20 +146,26 @@ self.onmessage = ({ data }: MessageEvent) => {
       }
     }
 
-    writer.checkpoint();
-    nextIndex++;
-    if (message.encode(writer)) {
-      isEmpty = false; 
-    } else {
+    writer.checkpoint(); // TODO: incapsulate in writer
+    if (!message.encode(writer)) {
       send();
-      if (message.encode(writer)) {
-        isEmpty = false;
-      } else {
-        // MAX_BATCH_SIZE overflow by one message
-        // TODO: correct handle
-        nextIndex--;
-        return;
-      }
+      // writer.reset(); // TODO: sematically clear code
+      if (!message.encode(writer)) { // Try to encode within empty state
+        // MBTODO: tempWriter for one message?
+        while (!message.encode(writer)) {
+          if (beaconSize === BEACON_SIZE_LIMIT) {
+            console.warn("OpenReplay: beacon size overflow.");
+            writer.reset();
+            writeBatchMeta();
+            return
+          }
+          beaconSize = Math.min(beaconSize*2, BEACON_SIZE_LIMIT);
+          writer = new Writer(beaconSize);
+          writeBatchMeta();
+        }
+      } 
     };
+    nextIndex++; // TODO: incapsulate in writer
+    isEmpty = false;
   });
 };

tracker/tracker/src/webworker/transformer.js

@@ -1,21 +0,0 @@
-import Message from '../messages/message';
-
-
-
-
-class MessageTransformer {
-	private urlRewriter?: URLRewriter
-
-	constructor() {
-
-	}
-
-  transform(m: Message): Message {
-		if (m instanceof SetNodeAttribute) {
-			if (m.name == "src" || m.name == "href") {
-				sendAssetForCache
-			}
-		}
-	}
-
-}