fix tracker speed up redux plugin hashing (#2027)

* feat tracker update redux plugin to speed up state hashing

* feat tracker update redux plugin to speed up state hashing

tracker/tracker-redux/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openreplay/tracker-redux",
   "description": "Tracker plugin for Redux state recording",
-  "version": "3.5.1",
+  "version": "3.6.1-beta.2",
   "keywords": [
     "redux",
     "logging",
@@ -21,9 +21,8 @@
     "build-cjs": "rm -Rf cjs && tsc --project tsconfig-cjs.json && echo '{ \"type\": \"commonjs\" }' > cjs/package.json && replace-in-files cjs/* --string='@openreplay/tracker' --replacement='@openreplay/tracker/cjs'",
     "prepublishOnly": "npm run build"
   },
-  "dependencies": {},
   "peerDependencies": {
-    "@openreplay/tracker": ">=3.5.0",
+    "@openreplay/tracker": ">=12.0.6",
     "redux": "^4.0.0"
   },
   "devDependencies": {

tracker/tracker-redux/src/index.ts

@@ -1,11 +1,15 @@
 import { App, Messages } from '@openreplay/tracker';
-import { Encoder, sha1 } from './syncod/index.js';
+import { Encoder, murmur } from './syncod-v2/index.js';
 
 export interface Options {
   actionFilter: (action: any) => boolean;
   actionTransformer: (action: any) => any; // null will be ignored
   actionType: (action: any) => any; // empty string and non-string will be ignored
   stateTransformer: (state: any) => any;
+  stateUpdateBatching: {
+    enabled: boolean;
+    throttle: number;
+  }
 }
 
 export default function(opts: Partial<Options> = {}) {
@@ -15,6 +19,10 @@ export default function(opts: Partial<Options> = {}) {
       actionTransformer: action => action,
       actionType: action => action.type,
       stateTransformer: state => state,
+      stateUpdateBatching: {
+        enabled: true,
+        throttle: 50,
+      }
     },
     opts,
   );
@@ -22,10 +30,24 @@ export default function(opts: Partial<Options> = {}) {
     if (app === null) {
       return () => next => action => next(action);
     }
-    const encoder = new Encoder(sha1, 50);
+    const encoder = new Encoder(murmur, 50);
     app.attachStopCallback(() => {
       encoder.clear()
     })
+
+    let lastCommit: number;
+    let lastState: string | null = null;
+
+    const batchEncoding = (state: Record<string, any>) => {
+      if (!lastState || !lastCommit || Date.now() - lastCommit > options.stateUpdateBatching.throttle) {
+        const _state = encoder.encode(options.stateTransformer(state));
+        lastCommit = Date.now();
+        lastState = _state;
+        return _state;
+      } else {
+        return lastState
+      }
+    }
     return ({ getState }) => next => action => {
       if (!app.active() || !options.actionFilter(action)) {
         return next(action);
@@ -39,10 +61,15 @@ export default function(opts: Partial<Options> = {}) {
           app.send(Messages.StateAction(type));
         }
         const _action = encoder.encode(options.actionTransformer(action));
-        const _state = encoder.encode(options.stateTransformer(getState()));
+        let _currState: string
+        if (options.stateUpdateBatching.enabled) {
+          _currState = batchEncoding(getState());
+        } else {
+          _currState = encoder.encode(options.stateTransformer(getState()));
+        }
         const _table = encoder.commit();
         for (let key in _table) app.send(Messages.OTable(key, _table[key]));
-        app.send(Messages.Redux(_action, _state, duration));
+        app.send(Messages.Redux(_action, _currState, duration));
       } catch {
         encoder.clear();
       }

tracker/tracker-redux/src/syncod-v2/types.d.ts

@@ -0,0 +1,20 @@
+declare type HashFunction = (str: string) => string;
+declare type Dict = { [key: string]: string };
+
+export class Encoder {
+  constructor(hash: HashFunction, slen?: number);
+  commit(): Dict;
+  encode(obj: any): string;
+  clear(): void;
+}
+
+export class Decoder {
+  constructor();
+  set(ref: string, enc: string): void;
+  assign(dict: Dict): void;
+  decode(enc: string): any;
+  clear(): void;
+}
+
+export const sha1: HashFunction;
+export const murmur: HashFunction;

tracker/tracker-redux/src/syncod/index.ts

@@ -1,5 +0,0 @@
-// TODO: SSR solution for all asayer libraries
-import Encoder from "./encoder.js";
-import sha1 from "./sha1.js";
-
-export { Encoder, sha1 };

tracker/tracker-redux/tsconfig.json

@@ -3,10 +3,11 @@
     "noImplicitThis": true,
     "strictNullChecks": true,
     "alwaysStrict": true,
-    "target": "es6",
+    "target": "es2020",
     "module": "es6",
-    "moduleResolution": "node",
+    "moduleResolution": "nodenext",
     "declaration": true,
-    "outDir": "./lib"
+    "outDir": "./lib",
+    "lib": ["es2020", "dom"]
   }
 }

tracker/tracker-zustand/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openreplay/tracker-zustand",
   "description": "Tracker plugin for Zustand state recording",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "keywords": [
     "zustand",
     "state",
@@ -22,7 +22,9 @@
     "build-cjs": "rm -Rf cjs && tsc --project tsconfig-cjs.json && echo '{ \"type\": \"commonjs\" }' > cjs/package.json && replace-in-files cjs/* --string='@openreplay/tracker' --replacement='@openreplay/tracker/cjs'",
     "prepublishOnly": "npm run build"
   },
-  "dependencies": { "zustand": "^4.5.2" },
+  "dependencies": {
+    "zustand": "^4.5.2"
+  },
   "peerDependencies": {
     "@openreplay/tracker": ">=12.0.0"
   },

tracker/tracker-zustand/src/index.ts

@@ -1,5 +1,5 @@
 import { App, Messages } from "@openreplay/tracker";
-import { Encoder, murmur } from "./syncod-v2/src/index.js";
+import { Encoder, murmur } from "./syncod-v2/index.js";
 import { StateCreator, StoreMutatorIdentifier } from "zustand";
 
 export type StateLogger = <

tracker/tracker-zustand/src/syncod-v2/chars.ts

@@ -1,4 +1,4 @@
-const chars = {};
+const chars: Record<string, string> = {};
 
 [
   "DEL",

tracker/tracker-zustand/src/syncod-v2/decoder.ts

@@ -0,0 +1,79 @@
+import _ from "./chars.js";
+
+export default class Decoder {
+  _dict: Map<any, any>;
+  constructor() {
+    this._dict = new Map();
+  }
+
+  set(ref, enc) {
+    this._dict.set(ref, enc);
+  }
+
+  assign(dict) {
+    for (let ref in dict) {
+      this._dict.set(ref, dict[ref]);
+    }
+  }
+
+  clear() {
+    this._dict.clear();
+  }
+
+  _unref_str(str) {
+    let s = this._dict.get(str);
+    if (s !== undefined) {
+      return s;
+    }
+    return str;
+  }
+
+  decode(enc) {
+    const p = enc[0],
+      b = enc.slice(1);
+    switch (p) {
+      case _.UNDEF:
+        return undefined;
+      case _.TRUE:
+        return true;
+      case _.FALSE:
+        return false;
+      case _.FUNCTION:
+        return Function.prototype;
+      case _.NUMBER:
+        return parseFloat(b);
+      case _.BIGINT:
+        return BigInt(b);
+      case _.STRING:
+        return this._unref_str(b);
+      case _.SYMBOL:
+        return Symbol(this._unref_str(b));
+      case _.NULL:
+        return null;
+    }
+    const unref = this._dict.get(b);
+    if (unref === undefined) {
+      throw "index missing code";
+    }
+    if (typeof unref === "object") {
+      return unref;
+    }
+    const args = unref.length === 0 ? [] : unref.split(_.DEL);
+    switch (p) {
+      case _.ARRAY:
+        this._dict.set(b, args);
+        for (let i = 0; i < args.length; i++) {
+          args[i] = this.decode(args[i]);
+        }
+        return args;
+      case _.OBJECT:
+        const obj = {};
+        this._dict.set(b, obj);
+        for (let i = 0; i < args.length; i += 2) {
+          obj[this._unref_str(args[i])] = this.decode(args[i + 1]);
+        }
+        return obj;
+    }
+    throw "unrecognized prefix";
+  }
+}

tracker/tracker-zustand/src/syncod-v2/encoder.ts

@@ -1,220 +1,120 @@
 import _ from "./chars.js";
-// @ts-ignore
 
-// @ts-ignore
+type HashFunction = (str: string) => string;
+
 export default class Encoder {
-// @ts-ignore
-  constructor(hash, slen = Infinity) {
-// @ts-ignore
+  _hash: HashFunction;
+  _slen: number;
+  _refmap: Map<any, any>;
+  _refset: Set<any>;
+
+  constructor(hash: HashFunction, slen = Infinity) {
     this._hash = hash;
-// @ts-ignore
     this._slen = slen;
-// @ts-ignore
     this._refmap = new Map();
-// @ts-ignore
     this._refset = new Set();
-// @ts-ignore
   }
-// @ts-ignore
 
-// @ts-ignore
   _ref_str(str) {
-// @ts-ignore
-    if (str.length < this._slen && str.indexOf(_.DEL) === -1) {
-// @ts-ignore
+    if (str.length < this._slen && !str.includes(_.DEL)) {
       return str;
-// @ts-ignore
     }
-// @ts-ignore
     let ref = this._refmap.get(str);
-// @ts-ignore
     if (ref === undefined) {
-// @ts-ignore
       ref = this._hash(str);
-// @ts-ignore
       this._refmap.set(str, ref);
-// @ts-ignore
     }
-// @ts-ignore
     return ref;
-// @ts-ignore
   }
-// @ts-ignore
 
-// @ts-ignore
   _encode_prim(obj) {
-// @ts-ignore
-    switch (typeof obj) {
-// @ts-ignore
+    const type = typeof obj;
+    switch (type) {
       case "undefined":
-// @ts-ignore
         return _.UNDEF;
-// @ts-ignore
       case "boolean":
-// @ts-ignore
         return obj ? _.TRUE : _.FALSE;
-// @ts-ignore
       case "number":
-// @ts-ignore
         return _.NUMBER + obj.toString();
-// @ts-ignore
       case "bigint":
-// @ts-ignore
         return _.BIGINT + obj.toString();
-// @ts-ignore
       case "function":
-// @ts-ignore
         return _.FUNCTION;
-// @ts-ignore
       case "string":
-// @ts-ignore
         return _.STRING + this._ref_str(obj);
-// @ts-ignore
       case "symbol":
-// @ts-ignore
         return _.SYMBOL + this._ref_str(obj.toString().slice(7, -1));
-// @ts-ignore
     }
-// @ts-ignore
     if (obj === null) {
-// @ts-ignore
       return _.NULL;
-// @ts-ignore
     }
-// @ts-ignore
   }
-// @ts-ignore
 
-// @ts-ignore
   _encode_obj(obj, ref = this._refmap.get(obj)) {
-// @ts-ignore
     return (Array.isArray(obj) ? _.ARRAY : _.OBJECT) + ref;
-// @ts-ignore
   }
-// @ts-ignore
 
-// @ts-ignore
   _encode_term(obj) {
-// @ts-ignore
     return this._encode_prim(obj) || this._encode_obj(obj);
-// @ts-ignore
   }
-// @ts-ignore
 
-// @ts-ignore
   _encode_deep(obj, depth) {
-// @ts-ignore
     const enc = this._encode_prim(obj);
-// @ts-ignore
     if (enc !== undefined) {
-// @ts-ignore
       return enc;
-// @ts-ignore
     }
-// @ts-ignore
     const ref = this._refmap.get(obj);
-// @ts-ignore
     switch (typeof ref) {
-// @ts-ignore
       case "number":
-// @ts-ignore
         return (depth - ref).toString();
-// @ts-ignore
       case "string":
-// @ts-ignore
         return this._encode_obj(obj, ref);
-// @ts-ignore
     }
-// @ts-ignore
     this._refmap.set(obj, depth);
-// @ts-ignore
+
     const hash = this._hash(
-// @ts-ignore
       (Array.isArray(obj)
-// @ts-ignore
         ? obj.map(v => this._encode_deep(v, depth + 1))
-// @ts-ignore
         : Object.keys(obj)
-// @ts-ignore
             .sort()
-// @ts-ignore
             .map(
-// @ts-ignore
               k =>
-// @ts-ignore
                 this._ref_str(k) + _.DEL + this._encode_deep(obj[k], depth + 1)
-// @ts-ignore
             )
-// @ts-ignore
       ).join(_.DEL)
-// @ts-ignore
     );
-// @ts-ignore
+
     this._refmap.set(obj, hash);
-// @ts-ignore
     return this._encode_obj(obj, hash);
-// @ts-ignore
   }
-// @ts-ignore
 
-// @ts-ignore
   encode(obj) {
-// @ts-ignore
     return this._encode_deep(obj, 0);
-// @ts-ignore
   }
-// @ts-ignore
 
-// @ts-ignore
   commit() {
-// @ts-ignore
     const dict = {};
-// @ts-ignore
     this._refmap.forEach((ref, obj) => {
-// @ts-ignore
       if (this._refset.has(ref)) {
-// @ts-ignore
         return;
-// @ts-ignore
       }
-// @ts-ignore
       this._refset.add(ref);
-// @ts-ignore
       if (typeof obj !== "string") {
-// @ts-ignore
         obj = (Array.isArray(obj)
-// @ts-ignore
           ? obj.map(v => this._encode_term(v))
-// @ts-ignore
           : Object.keys(obj).map(
-// @ts-ignore
               k => this._ref_str(k) + _.DEL + this._encode_term(obj[k])
-// @ts-ignore
             )
-// @ts-ignore
         ).join(_.DEL);
-// @ts-ignore
       }
-// @ts-ignore
       dict[ref] = obj;
-// @ts-ignore
     });
-// @ts-ignore
     this._refmap.clear();
-// @ts-ignore
     return dict;
-// @ts-ignore
   }
-// @ts-ignore
 
-// @ts-ignore
   clear() {
-// @ts-ignore
     this._refmap.clear();
-// @ts-ignore
     this._refset.clear();
-// @ts-ignore
   }
-// @ts-ignore
 }
-// @ts-ignore

tracker/tracker-zustand/src/syncod-v2/index.ts

@@ -0,0 +1,6 @@
+import Encoder from "./encoder.js";
+import Decoder from "./decoder.js";
+import sha1 from "./sha1.js";
+import murmur from "./mur.js";
+
+export { Encoder, Decoder, sha1, murmur };

tracker/tracker-zustand/src/syncod-v2/mur.ts

@@ -0,0 +1,162 @@
+function murmurhash3_32_rp(key, seed) {
+  var keyLength, tailLength, tailLength4, bodyLength, bodyLength8, h1, k1, i, c1_low, c1_high, c2_low, c2_high, k1B, c3;
+  keyLength = key.length;
+  tailLength = keyLength & 3;
+  bodyLength = keyLength - tailLength;
+  tailLength4 = bodyLength & 7;
+  bodyLength8 = bodyLength - tailLength4;
+  h1 = seed;
+
+  //c1 = 0xcc9e2d51;
+  c1_low = 0x2d51;
+  c1_high = 0xcc9e0000;
+
+  //c2 = 0x1b873593;
+  c2_low = 0x3593;
+  c2_high = 0x1b870000;
+
+  c3 = 0xe6546b64;
+
+
+  //----------
+  // body
+
+  i = 0;
+
+  while (i < bodyLength8) {
+
+    k1 =
+      ((key.charCodeAt(i) & 0xff)) |
+      ((key.charCodeAt(++i) & 0xff) << 8) |
+      ((key.charCodeAt(++i) & 0xff) << 16) |
+      ((key.charCodeAt(++i) & 0xff) << 24);
+
+    k1B =
+      ((key.charCodeAt(++i) & 0xff)) |
+      ((key.charCodeAt(++i) & 0xff) << 8) |
+      ((key.charCodeAt(++i) & 0xff) << 16) |
+      ((key.charCodeAt(++i) & 0xff) << 24);
+
+    ++i;
+
+
+    //k1 *= c1;
+    k1 = (c1_high * k1 | 0) + (c1_low * k1);
+    //k1 = ROTL32(k1,15);
+    k1 = (k1 << 15) | (k1 >>> 17);
+    //k1 *= c2;
+    k1 = (c2_high * k1 | 0) + (c2_low * k1);
+
+    //h1 ^= k1;
+    h1 ^= k1;
+    //h1 = ROTL32(h1,13);
+    h1 = (h1 << 13) | (h1 >>> 19);
+    //h1 = h1*5+0xe6546b64;
+    h1 = h1 * 5 + c3;
+
+
+    //k1 *= c1;
+    k1B = (c1_high * k1B | 0) + (c1_low * k1B);
+    //k1 = ROTL32(k1,15);
+    k1B = (k1B << 15) | (k1B >>> 17);
+    //k1 *= c2;
+    k1B = (c2_high * k1B | 0) + (c2_low * k1B);
+
+    //h1 ^= k1;
+    h1 ^= k1B;
+    //h1 = ROTL32(h1,13);
+    h1 = (h1 << 13) | (h1 >>> 19);
+    //h1 = h1*5+0xe6546b64;
+    h1 = h1 * 5 + c3;
+
+  }   //while (i < bodyLength8) {
+
+
+  if (tailLength4) {
+
+    k1 =
+      ((key.charCodeAt(i) & 0xff)) |
+      ((key.charCodeAt(++i) & 0xff) << 8) |
+      ((key.charCodeAt(++i) & 0xff) << 16) |
+      ((key.charCodeAt(++i) & 0xff) << 24);
+
+    ++i;
+
+    //k1 *= c1;
+    k1 = (c1_high * k1 | 0) + (c1_low * k1);
+    //k1 = ROTL32(k1,15);
+    k1 = (k1 << 15) | (k1 >>> 17);
+    //k1 *= c2;
+    k1 = (c2_high * k1 | 0) + (c2_low * k1);
+
+    //h1 ^= k1;
+    h1 ^= k1;
+    //h1 = ROTL32(h1,13);
+    h1 = (h1 << 13) | (h1 >>> 19);
+    //h1 = h1*5+0xe6546b64;
+    h1 = h1 * 5 + c3;
+
+  }   //if (tailLength4) {
+
+
+  //----------
+  // tail
+
+  k1 = 0;
+
+  switch (tailLength) {
+
+    case 3: k1 ^= (key.charCodeAt(i + 2) & 0xff) << 16;
+    case 2: k1 ^= (key.charCodeAt(i + 1) & 0xff) << 8;
+    case 1: k1 ^= (key.charCodeAt(i) & 0xff);
+
+      //k1 *= c1;
+      k1 = (c1_high * k1 | 0) + (c1_low * k1);
+      //k1 = ROTL32(k1,15);
+      k1 = (k1 << 15) | (k1 >>> 17);
+      //k1 *= c2;
+      k1 = (c2_high * k1 | 0) + (c2_low * k1);
+      //h1 ^= k1;
+      h1 ^= k1;
+
+  }   //switch (tailLength) {
+
+
+  //----------
+  // finalization
+
+  h1 ^= keyLength;
+
+  //h1 = fmix32(h1);
+  {
+    //h ^= h >> 16;
+    h1 ^= h1 >>> 16;
+    //h1 *= 0x85ebca6b;
+    h1 = (0x85eb0000 * h1 | 0) + (0xca6b * h1);
+    //h ^= h >> 13;
+    h1 ^= h1 >>> 13;
+    //h1 *= 0xc2b2ae35;
+    h1 = (0xc2b20000 * h1 | 0) + (0xae35 * h1);
+    //h ^= h >> 16;
+    h1 ^= h1 >>> 16;
+  }
+
+
+  return h1 >>> 0;    //convert to unsigned int
+
+}
+
+
+function murmurHash3ToBase64(key, seed = 0) {
+  let hashValue = murmurhash3_32_rp(key, seed);
+  return hashToBase64(hashValue);
+}
+
+function hashToBase64(hash) {
+  let buffer = new ArrayBuffer(4); // 32 bits for hash
+  let view = new DataView(buffer);
+  view.setUint32(0, hash, false); // Use big-endian
+  return btoa(String.fromCharCode.apply(null, new Uint8Array(buffer)));
+}
+
+export default murmurHash3ToBase64;

tracker/tracker/src/main/tsconfig.json

@@ -2,7 +2,7 @@
   "extends": "../../tsconfig-base.json",
   "compilerOptions": {
     "allowSyntheticDefaultImports": true,
-    "lib": ["es2017", "dom"],
+    "lib": ["es2020", "dom"],
     "declaration": true
   },
   "references": [{ "path": "../common" }]