fix(ui): canvas replay back/forth bug (#1896)

* fix(tracker): change canvas scaling

* fix(tracker): 12.0.3

* fix(tracker): 12.0.3

frontend/app/player/web/managers/CanvasManager.ts

@@ -2,8 +2,6 @@ import { VElement } from "Player/web/managers/DOM/VirtualDOM";
 
 export default class CanvasManager {
   private fileData: string | undefined;
-  private canvasEl: HTMLVideoElement
-  private canvasCtx: CanvasRenderingContext2D | null = null;
   private videoTag = document.createElement('video')
   private lastTs = 0;
 
@@ -38,10 +36,6 @@ export default class CanvasManager {
     this.videoTag.setAttribute('crossorigin', 'anonymous');
     this.videoTag.src = this.fileData;
     this.videoTag.currentTime = 0;
-
-    const node = this.getNode(parseInt(this.nodeId, 10)) as unknown as VElement
-    this.canvasCtx = (node.node as HTMLCanvasElement).getContext('2d');
-    this.canvasEl = node.node as HTMLVideoElement;
   }
 
   move(t: number) {
@@ -49,11 +43,14 @@ export default class CanvasManager {
     this.lastTs = t;
     const playTime = t - this.delta
     if (playTime > 0) {
+      const node = this.getNode(parseInt(this.nodeId, 10)) as unknown as VElement
+      const canvasCtx = (node.node as HTMLCanvasElement).getContext('2d');
+      const canvasEl = node.node as HTMLVideoElement;
       if (!this.videoTag.paused) {
           void this.videoTag.pause()
       }
       this.videoTag.currentTime = playTime/1000;
-      this.canvasCtx?.drawImage(this.videoTag, 0, 0, this.canvasEl.width, this.canvasEl.height);
+      canvasCtx?.drawImage(this.videoTag, 0, 0, canvasEl.width, canvasEl.height);
     }
   }
 }

tracker/tracker/CHANGELOG.md

@@ -1,3 +1,15 @@
+# 12.0.3
+
+- fixed scaling option for canvas (to ignore window.devicePixelRatio and always render the canvas as 1)
+
+# 12.0.2
+
+- fix for canvas snapshot check
+
+# 12.0.1
+
+- pause canvas snapshotting when its offscreen
+
 # 12.0.0
 
 - offline session recording and manual sending

tracker/tracker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openreplay/tracker",
   "description": "The OpenReplay tracker main package",
-  "version": "12.0.0-beta.10",
+  "version": "12.0.3",
   "keywords": [
     "logging",
     "replay"

tracker/tracker/src/main/app/canvas.ts

@@ -5,12 +5,15 @@ import Message, { CanvasNode } from './messages.gen.js'
 interface CanvasSnapshot {
   images: { data: string; id: number }[]
   createdAt: number
+  paused: boolean
+  dummy: HTMLCanvasElement
 }
 
 interface Options {
   fps: number
   quality: 'low' | 'medium' | 'high'
   isDebug?: boolean
+  fixedScaling?: boolean
 }
 
 class CanvasRecorder {
@@ -27,19 +30,19 @@ class CanvasRecorder {
 
   startTracking() {
     setTimeout(() => {
-      this.app.nodes.scanTree(this.handleCanvasEl)
+      this.app.nodes.scanTree(this.captureCanvas)
       this.app.nodes.attachNodeCallback((node: Node): void => {
-        this.handleCanvasEl(node)
+        this.captureCanvas(node)
       })
     }, 500)
   }
 
   restartTracking = () => {
     this.clear()
-    this.app.nodes.scanTree(this.handleCanvasEl)
+    this.app.nodes.scanTree(this.captureCanvas)
   }
 
-  handleCanvasEl = (node: Node) => {
+  captureCanvas = (node: Node) => {
     const id = this.app.nodes.getID(node)
     if (!id || !hasTag(node, 'canvas')) {
       return
@@ -49,10 +52,41 @@ class CanvasRecorder {
     if (isIgnored || !hasTag(node, 'canvas') || this.snapshots[id]) {
       return
     }
+
+    const observer = new IntersectionObserver((entries) => {
+      entries.forEach((entry) => {
+        if (entry.isIntersecting) {
+          if (entry.target) {
+            if (this.snapshots[id] && this.snapshots[id].createdAt) {
+              this.snapshots[id].paused = false
+            } else {
+              this.recordCanvas(entry.target, id)
+            }
+            /**
+             * We can switch this to start observing when element is in the view
+             * but otherwise right now we're just pausing when it's not
+             * just to save some bandwidth and space on backend
+             * */
+            // observer.unobserve(entry.target)
+          } else {
+            if (this.snapshots[id]) {
+              this.snapshots[id].paused = true
+            }
+          }
+        }
+      })
+    })
+
+    observer.observe(node)
+  }
+
+  recordCanvas = (node: Node, id: number) => {
     const ts = this.app.timestamp()
     this.snapshots[id] = {
       images: [],
       createdAt: ts,
+      paused: false,
+      dummy: document.createElement('canvas'),
     }
     const canvasMsg = CanvasNode(id.toString(), ts)
     this.app.send(canvasMsg as Message)
@@ -63,11 +97,18 @@ class CanvasRecorder {
         this.app.debug.log('Canvas element not in sync')
         clearInterval(int)
       } else {
-        const snapshot = captureSnapshot(canvas, this.options.quality)
-        this.snapshots[id].images.push({ id: this.app.timestamp(), data: snapshot })
-        if (this.snapshots[id].images.length > 9) {
-          this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt)
-          this.snapshots[id].images = []
+        if (!this.snapshots[id].paused) {
+          const snapshot = captureSnapshot(
+            canvas,
+            this.options.quality,
+            this.snapshots[id].dummy,
+            this.options.fixedScaling,
+          )
+          this.snapshots[id].images.push({ id: this.app.timestamp(), data: snapshot })
+          if (this.snapshots[id].images.length > 9) {
+            this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt)
+            this.snapshots[id].images = []
+          }
         }
       }
     }, this.interval)
@@ -110,18 +151,36 @@ class CanvasRecorder {
 }
 
 const qualityInt = {
-  low: 0.33,
+  low: 0.35,
   medium: 0.55,
   high: 0.8,
 }
 
-function captureSnapshot(canvas: HTMLCanvasElement, quality: 'low' | 'medium' | 'high' = 'medium') {
+function captureSnapshot(
+  canvas: HTMLCanvasElement,
+  quality: 'low' | 'medium' | 'high' = 'medium',
+  dummy: HTMLCanvasElement,
+  fixedScaling = false,
+) {
   const imageFormat = 'image/jpeg' // or /png'
-  return canvas.toDataURL(imageFormat, qualityInt[quality])
+  if (fixedScaling) {
+    const canvasScaleRatio = window.devicePixelRatio || 1
+    dummy.width = canvas.width / canvasScaleRatio
+    dummy.height = canvas.height / canvasScaleRatio
+    const ctx = dummy.getContext('2d')
+    if (!ctx) {
+      return ''
+    }
+    ctx.drawImage(canvas, 0, 0, dummy.width, dummy.height)
+    return dummy.toDataURL(imageFormat, qualityInt[quality])
+  } else {
+    return canvas.toDataURL(imageFormat, qualityInt[quality])
+  }
 }
 
 function dataUrlToBlob(dataUrl: string): [Blob, Uint8Array] | null {
   const [header, base64] = dataUrl.split(',')
+  if (!header || !base64) return null
   const encParts = header.match(/:(.*?);/)
   if (!encParts) return null
   const mime = encParts[1]

tracker/tracker/src/main/app/index.ts

@@ -117,6 +117,7 @@ type AppOptions = {
   __debug_report_edp: string | null
   __debug__?: ILogLevel
   __save_canvas_locally?: boolean
+  fixedCanvasScaling?: boolean
   localStorage: Storage | null
   sessionStorage: Storage | null
   forceSingleTab?: boolean
@@ -211,6 +212,7 @@ export default class App {
         disableStringDict: false,
         forceSingleTab: false,
         assistSocketHost: '',
+        fixedCanvasScaling: false,
       },
       options,
     )
@@ -1063,6 +1065,7 @@ export default class App {
               fps: canvasFPS,
               quality: canvasQuality,
               isDebug: this.options.__save_canvas_locally,
+              fixedScaling: this.options.fixedCanvasScaling,
             })
           this.canvasRecorder.startTracking()
         }