fix(tracker): fix blob generation

tracker/tracker-assist/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 7.0.3
+
+- small fix for canvas context tracking
+
 ## 7.0.1
 
 - mark live sessions with ux test active

tracker/tracker-assist/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openreplay/tracker-assist",
   "description": "Tracker plugin for screen assistance through the WebRTC",
-  "version": "7.0.1",
+  "version": "7.0.3",
   "keywords": [
     "WebRTC",
     "assistance",
@@ -34,7 +34,7 @@
     "socket.io-client": "^4.7.2"
   },
   "peerDependencies": {
-    "@openreplay/tracker": "^11.0.1"
+    "@openreplay/tracker": "^11.0.5"
   },
   "devDependencies": {
     "@openreplay/tracker": "file:../tracker",

tracker/tracker-assist/src/Assist.ts

@@ -535,7 +535,7 @@ export default class Assist {
         }
 
         if (!callUI) {
-          callUI = new CallWindow(console.log, this.options.callUITemplate)
+          callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
           callUI.setVideoToggleCallback(updateVideoFeed)
         }
         callUI.showControls(initiateCallEnd)
@@ -583,7 +583,6 @@ export default class Assist {
         sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIds))
         this.emit('UPDATE_SESSION', { agentIds: callingPeerIds, isCallActive: true, })
       }).catch(reason => { // in case of Confirm.remove() without user answer (not a error)
-        console.log(reason)
         app.debug.log(reason)
       })
     })
@@ -615,6 +614,7 @@ export default class Assist {
               }
             })
           },
+          app.debug.error,
         )
         this.canvasMap.set(id, canvasHandler)
       }

tracker/tracker-assist/src/CallWindow.ts

@@ -42,7 +42,7 @@ export default class CallWindow {
 
 		const doc = iframe.contentDocument
 		if (!doc) {
-			console.error('OpenReplay: CallWindow iframe document is not reachable.')
+			logError('OpenReplay: CallWindow iframe document is not reachable.')
 			return
 		}
 

tracker/tracker-assist/src/Canvas.ts

@@ -6,9 +6,9 @@ export default class CanvasRecorder {
     private readonly canvas: HTMLCanvasElement,
     private readonly canvasId: number,
     private readonly fps: number,
-    private readonly onStream: (stream: MediaStream) => void
+    private readonly onStream: (stream: MediaStream) => void,
+    private readonly logError: (...args: any[]) => void,
   ) {
-    this.canvas.getContext('2d', { alpha: true, })
     const stream = this.canvas.captureStream(this.fps)
     this.emitStream(stream)
   }
@@ -39,7 +39,7 @@ export default class CanvasRecorder {
 
     void video.play()
     video.addEventListener('error', (e) => {
-      console.error('Video error:', e)
+      this.logError('Video error:', e)
     })
   }
 
@@ -50,7 +50,7 @@ export default class CanvasRecorder {
     if (this.stream) {
       this.onStream(this.stream)
     } else {
-      console.error('no stream for canvas', this.canvasId)
+      this.logError('no stream for canvas', this.canvasId)
     }
   }
 

tracker/tracker-assist/src/index.ts

@@ -8,17 +8,16 @@ import Assist from './Assist.js'
 export default function(opts?: Partial<Options>) {
   return function(app: App | null, appOptions: { __DISABLE_SECURE_MODE?: boolean } = {}) {
     // @ts-ignore
-    if (app === null || !navigator?.mediaDevices?.getUserMedia) { // 93.04% browsers
+    if (app === null || !navigator?.mediaDevices?.getUserMedia) {
       return
     }
     if (!app.checkRequiredVersion || !app.checkRequiredVersion('REQUIRED_TRACKER_VERSION')) {
       console.warn('OpenReplay Assist: couldn\'t load. The minimum required version of @openreplay/tracker@REQUIRED_TRACKER_VERSION is not met')
       return
     }
-    app.notify.log('OpenReplay Assist initializing.')
+
     const assist = new Assist(app, opts, appOptions.__DISABLE_SECURE_MODE)
     app.debug.log(assist)
     return assist
-
   }
 }

tracker/tracker-assist/src/version.ts

@@ -1 +1 @@
-export const pkgVersion = '7.0.1'
+export const pkgVersion = '7.0.3'

tracker/tracker/CHANGELOG.md

@@ -1,4 +1,22 @@
-# 11.0.1
+# 11.0.6
+
+- fix blob generation for canvas capture (Cannot read properties of null (reading '1'))
+
+# 11.0.5
+
+- add method to restart canvas tracking (in case of context recreation)
+- scan dom tree for canvas els on tracker start
+
+# 11.0.4
+
+- some additional security for canvas capture (check if canvas el itself is obscured/ignored)
+
+# 11.0.3
+
+- move all logs under internal debugger
+- fix for XHR proxy ORSC 'abort' state
+
+# 11.0.1 & 11.0.2
 
 - minor fixes and refactoring
 

tracker/tracker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openreplay/tracker",
   "description": "The OpenReplay tracker main package",
-  "version": "11.0.2",
+  "version": "11.0.6",
   "keywords": [
     "logging",
     "replay"

tracker/tracker/src/main/app/canvas.ts

@@ -10,6 +10,7 @@ interface CanvasSnapshot {
 interface Options {
   fps: number
   quality: 'low' | 'medium' | 'high'
+  isDebug?: boolean
 }
 
 class CanvasRecorder {
@@ -25,47 +26,66 @@ class CanvasRecorder {
   }
 
   startTracking() {
-    this.app.nodes.attachNodeCallback((node: Node): void => {
+    setTimeout(() => {
-      const id = this.app.nodes.getID(node)
+      this.app.nodes.scanTree(this.handleCanvasEl)
-      if (!id || !hasTag(node, 'canvas') || this.snapshots[id]) {
+      this.app.nodes.attachNodeCallback((node: Node): void => {
-        return
+        this.handleCanvasEl(node)
-      }
+      })
-      const ts = this.app.timestamp()
+    }, 500)
-      this.snapshots[id] = {
+  }
-        images: [],
+
-        createdAt: ts,
+  restartTracking = () => {
-      }
+    this.clear()
-      const canvasMsg = CanvasNode(id.toString(), ts)
+    this.app.nodes.scanTree(this.handleCanvasEl)
-      this.app.send(canvasMsg as Message)
+  }
-      const int = setInterval(() => {
+
-        const cid = this.app.nodes.getID(node)
+  handleCanvasEl = (node: Node) => {
-        const canvas = cid ? this.app.nodes.getNode(cid) : undefined
+    const id = this.app.nodes.getID(node)
-        if (!canvas || !hasTag(canvas, 'canvas') || canvas !== node) {
+    if (!id || !hasTag(node, 'canvas')) {
-          console.log('Canvas element not in sync')
+      return
-          clearInterval(int)
+    }
-        } else {
+
-          const snapshot = captureSnapshot(canvas, this.options.quality)
+    const isIgnored = this.app.sanitizer.isObscured(id) || this.app.sanitizer.isHidden(id)
-          this.snapshots[id].images.push({ id: this.app.timestamp(), data: snapshot })
+    if (isIgnored || !hasTag(node, 'canvas') || this.snapshots[id]) {
-          if (this.snapshots[id].images.length > 9) {
+      return
-            this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt)
+    }
-            this.snapshots[id].images = []
+    const ts = this.app.timestamp()
-          }
+    this.snapshots[id] = {
+      images: [],
+      createdAt: ts,
+    }
+    const canvasMsg = CanvasNode(id.toString(), ts)
+    this.app.send(canvasMsg as Message)
+    const int = setInterval(() => {
+      const cid = this.app.nodes.getID(node)
+      const canvas = cid ? this.app.nodes.getNode(cid) : undefined
+      if (!canvas || !hasTag(canvas, 'canvas') || canvas !== node) {
+        this.app.debug.log('Canvas element not in sync')
+        clearInterval(int)
+      } else {
+        const snapshot = captureSnapshot(canvas, this.options.quality)
+        this.snapshots[id].images.push({ id: this.app.timestamp(), data: snapshot })
+        if (this.snapshots[id].images.length > 9) {
+          this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt)
+          this.snapshots[id].images = []
         }
-      }, this.interval)
+      }
-      this.intervals.push(int)
+    }, this.interval)
-    })
+    this.intervals.push(int)
   }
 
   sendSnaps(images: { data: string; id: number }[], canvasId: number, createdAt: number) {
     if (Object.keys(this.snapshots).length === 0) {
-      console.log(this.snapshots)
       return
     }
     const formData = new FormData()
     images.forEach((snapshot) => {
-      const blob = dataUrlToBlob(snapshot.data)[0]
+      const blob = dataUrlToBlob(snapshot.data)
-      formData.append('snapshot', blob, `${createdAt}_${canvasId}_${snapshot.id}.jpeg`)
+      if (!blob) return
-      // saveImageData(snapshot.data, `${createdAt}_${canvasId}_${snapshot.id}.jpeg`)
+      formData.append('snapshot', blob[0], `${createdAt}_${canvasId}_${snapshot.id}.jpeg`)
+      if (this.options.isDebug) {
+        saveImageData(snapshot.data, `${createdAt}_${canvasId}_${snapshot.id}.jpeg`)
+      }
     })
 
     fetch(this.app.options.ingestPoint + '/v1/web/images', {
@@ -75,16 +95,15 @@ class CanvasRecorder {
       },
       body: formData,
     })
-      .then((r) => {
+      .then(() => {
-        console.log('done', r)
+        return true
       })
       .catch((e) => {
-        console.error('error saving canvas', e)
+        this.app.debug.error('error saving canvas', e)
       })
   }
 
   clear() {
-    console.log('cleaning up')
     this.intervals.forEach((int) => clearInterval(int))
     this.snapshots = {}
   }
@@ -101,10 +120,11 @@ function captureSnapshot(canvas: HTMLCanvasElement, quality: 'low' | 'medium' |
   return canvas.toDataURL(imageFormat, qualityInt[quality])
 }
 
-function dataUrlToBlob(dataUrl: string): [Blob, Uint8Array] {
+function dataUrlToBlob(dataUrl: string): [Blob, Uint8Array] | null {
   const [header, base64] = dataUrl.split(',')
-  // @ts-ignore
+  const encParts = header.match(/:(.*?);/)
-  const mime = header.match(/:(.*?);/)[1]
+  if (!encParts) return null
+  const mime = encParts[1]
   const blobStr = atob(base64)
   let n = blobStr.length
   const u8arr = new Uint8Array(n)

tracker/tracker/src/main/app/index.ts

@@ -106,6 +106,7 @@ type AppOptions = {
   __is_snippet: boolean
   __debug_report_edp: string | null
   __debug__?: LoggerOptions
+  __save_canvas_locally?: boolean
   localStorage: Storage | null
   sessionStorage: Storage | null
   forceSingleTab?: boolean
@@ -183,6 +184,7 @@ export default class App {
         verbose: false,
         __is_snippet: false,
         __debug_report_edp: null,
+        __save_canvas_locally: false,
         localStorage: null,
         sessionStorage: null,
         disableStringDict: false,
@@ -236,9 +238,10 @@ export default class App {
           this.stop(false)
           void this.start({}, true)
         } else if (data === 'not_init') {
-          console.warn('WebWorker: writer not initialised. Restarting tracker')
+          this.debug.warn('OR WebWorker: writer not initialised. Restarting tracker')
         } else if (data.type === 'failure') {
           this.stop(false)
+          this.debug.error('worker_failed', data.reason)
           this._debug('worker_failed', data.reason)
         } else if (data.type === 'compress') {
           const batch = data.batch
@@ -246,7 +249,7 @@ export default class App {
           if (batchSize > this.compressionThreshold) {
             gzip(data.batch, { mtime: 0 }, (err, result) => {
               if (err) {
-                console.error('Openreplay compression error:', err)
+                this.debug.error('Openreplay compression error:', err)
                 this.stop(false)
                 if (this.restartAttempts < 3) {
                   this.restartAttempts += 1
@@ -595,7 +598,7 @@ export default class App {
     const sessionToken = this.session.getSessionToken()
     const isNewSession = needNewSessionID || !sessionToken
 
-    console.log(
+    this.debug.log(
       'OpenReplay: starting session; need new session id?',
       needNewSessionID,
       'session token: ',
@@ -682,7 +685,7 @@ export default class App {
           projectID,
         })
         if (!isNewSession && token === sessionToken) {
-          console.log('continuing session on new tab', this.session.getTabId())
+          this.debug.log('continuing session on new tab', this.session.getTabId())
           // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
           this.send(TabChange(this.session.getTabId()))
         }
@@ -701,17 +704,21 @@ export default class App {
         this.compressionThreshold = compressionThreshold
         const onStartInfo = { sessionToken: token, userUUID, sessionID }
 
+        if (canvasEnabled) {
+          this.canvasRecorder =
+            this.canvasRecorder ??
+            new CanvasRecorder(this, {
+              fps: canvasFPS,
+              quality: canvasQuality,
+              isDebug: this.options.__save_canvas_locally,
+            })
+          this.canvasRecorder.startTracking()
+        }
         // TODO: start as early as possible (before receiving the token)
         this.startCallbacks.forEach((cb) => cb(onStartInfo)) // MBTODO: callbacks after DOM "mounted" (observed)
         this.observer.observe()
         this.ticker.start()
 
-        if (canvasEnabled) {
-          this.canvasRecorder =
-            this.canvasRecorder ??
-            new CanvasRecorder(this, { fps: canvasFPS, quality: canvasQuality })
-          this.canvasRecorder.startTracking()
-        }
         this.activityState = ActivityState.Active
 
         this.notify.log('OpenReplay tracking started.')
@@ -766,6 +773,10 @@ export default class App {
       })
   }
 
+  restartCanvasTracking = () => {
+    this.canvasRecorder?.restartTracking()
+  }
+
   onUxtCb = []
 
   addOnUxtCb(cb: (id: number) => void) {

tracker/tracker/src/main/app/nodes.ts

@@ -16,6 +16,10 @@ export default class Nodes {
     this.nodeCallbacks.push(nodeCallback)
   }
 
+  scanTree = (cb: (node: Node | void) => void) => {
+    this.nodes.forEach((node) => cb(node))
+  }
+
   attachNodeListener(node: Node, type: string, listener: EventListener, useCapture = true): void {
     const id = this.getID(node)
     if (id === undefined) {

tracker/tracker/src/main/app/observer/iframe_observer.ts

@@ -12,7 +12,7 @@ export default class IFrameObserver extends Observer {
     this.observeRoot(doc, (docID) => {
       //MBTODO: do not send if empty (send on load? it might be in-place iframe, like our replayer, which does not get loaded)
       if (docID === undefined) {
-        console.log('OpenReplay: Iframe document not bound')
+        this.app.debug.log('OpenReplay: Iframe document not bound')
         return
       }
       this.app.send(CreateIFrameDocument(hostID, docID))

tracker/tracker/src/main/app/observer/shadow_root_observer.ts

@@ -10,7 +10,7 @@ export default class ShadowRootObserver extends Observer {
     } // log
     this.observeRoot(shRoot, (rootID) => {
       if (rootID === undefined) {
-        console.log('OpenReplay: Shadow Root was not bound')
+        this.app.debug.error('OpenReplay: Shadow Root was not bound')
         return
       }
       this.app.send(CreateIFrameDocument(hostID, rootID))

tracker/tracker/src/main/index.ts

@@ -223,6 +223,13 @@ export default class API {
     return this.featureFlags.flags
   }
 
+  public restartCanvasTracking = () => {
+    if (this.app === null) {
+      return
+    }
+    this.app.restartCanvasTracking()
+  }
+
   use<T>(fn: (app: App | null, options?: Options) => T): T {
     return fn(this.app, this.options)
   }

tracker/tracker/src/main/modules/Network/xhrProxy.ts

@@ -146,7 +146,7 @@ export class XHRProxyHandler<T extends XMLHttpRequest> implements ProxyHandler<T
   protected setOnReadyStateChange(target: T, key: string, orscFunction: (args: any[]) => any) {
     return Reflect.set(target, key, (...args: any[]) => {
       this.onReadyStateChange()
-      orscFunction.apply(target, args)
+      orscFunction?.apply(target, args)
     })
   }
 

tracker/tracker/src/webworker/index.ts

@@ -115,7 +115,7 @@ self.onmessage = ({ data }: { data: ToWorkerData }): any => {
 
   if (data.type === 'compressed') {
     if (!sender) {
-      console.debug('WebWorker: sender not initialised. Compressed batch.')
+      console.debug('OR WebWorker: sender not initialised. Compressed batch.')
       initiateRestart()
       return
     }
@@ -123,7 +123,7 @@ self.onmessage = ({ data }: { data: ToWorkerData }): any => {
   }
   if (data.type === 'uncompressed') {
     if (!sender) {
-      console.debug('WebWorker: sender not initialised. Uncompressed batch.')
+      console.debug('OR WebWorker: sender not initialised. Uncompressed batch.')
       initiateRestart()
       return
     }
@@ -163,13 +163,13 @@ self.onmessage = ({ data }: { data: ToWorkerData }): any => {
 
   if (data.type === 'auth') {
     if (!sender) {
-      console.debug('WebWorker: sender not initialised. Received auth.')
+      console.debug('OR WebWorker: sender not initialised. Received auth.')
       initiateRestart()
       return
     }
 
     if (!writer) {
-      console.debug('WebWorker: writer not initialised. Received auth.')
+      console.debug('OR WebWorker: writer not initialised. Received auth.')
       initiateRestart()
       return
     }