From 2192681149d7bf8facba1d29fc40178f9a6beeee Mon Sep 17 00:00:00 2001 From: Delirium Date: Mon, 19 Feb 2024 16:53:15 +0100 Subject: [PATCH] fix(ui): canvas replay back/forth bug (#1896) * fix(tracker): change canvas scaling * fix(tracker): 12.0.3 * fix(tracker): 12.0.3 --- .../app/player/web/managers/CanvasManager.ts | 11 +-- tracker/tracker/CHANGELOG.md | 12 +++ tracker/tracker/package.json | 2 +- tracker/tracker/src/main/app/canvas.ts | 83 ++++++++++++++++--- tracker/tracker/src/main/app/index.ts | 3 + 5 files changed, 91 insertions(+), 20 deletions(-) diff --git a/frontend/app/player/web/managers/CanvasManager.ts b/frontend/app/player/web/managers/CanvasManager.ts index a4e38a550..03d417985 100644 --- a/frontend/app/player/web/managers/CanvasManager.ts +++ b/frontend/app/player/web/managers/CanvasManager.ts @@ -2,8 +2,6 @@ import { VElement } from "Player/web/managers/DOM/VirtualDOM"; export default class CanvasManager { private fileData: string | undefined; - private canvasEl: HTMLVideoElement - private canvasCtx: CanvasRenderingContext2D | null = null; private videoTag = document.createElement('video') private lastTs = 0; @@ -38,10 +36,6 @@ export default class CanvasManager { this.videoTag.setAttribute('crossorigin', 'anonymous'); this.videoTag.src = this.fileData; this.videoTag.currentTime = 0; - - const node = this.getNode(parseInt(this.nodeId, 10)) as unknown as VElement - this.canvasCtx = (node.node as HTMLCanvasElement).getContext('2d'); - this.canvasEl = node.node as HTMLVideoElement; } move(t: number) { @@ -49,11 +43,14 @@ export default class CanvasManager { this.lastTs = t; const playTime = t - this.delta if (playTime > 0) { + const node = this.getNode(parseInt(this.nodeId, 10)) as unknown as VElement + const canvasCtx = (node.node as HTMLCanvasElement).getContext('2d'); + const canvasEl = node.node as HTMLVideoElement; if (!this.videoTag.paused) { void this.videoTag.pause() } this.videoTag.currentTime = playTime/1000; - this.canvasCtx?.drawImage(this.videoTag, 0, 0, this.canvasEl.width, this.canvasEl.height); + canvasCtx?.drawImage(this.videoTag, 0, 0, canvasEl.width, canvasEl.height); } } } \ No newline at end of file diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index d56ae2ea3..84f9ca16e 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -1,3 +1,15 @@ +# 12.0.3 + +- fixed scaling option for canvas (to ignore window.devicePixelRatio and always render the canvas as 1) + +# 12.0.2 + +- fix for canvas snapshot check + +# 12.0.1 + +- pause canvas snapshotting when its offscreen + # 12.0.0 - offline session recording and manual sending diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index bf32f160f..04c7d54ac 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "12.0.0-beta.10", + "version": "12.0.3", "keywords": [ "logging", "replay" diff --git a/tracker/tracker/src/main/app/canvas.ts b/tracker/tracker/src/main/app/canvas.ts index 3633e4852..f2c7934e9 100644 --- a/tracker/tracker/src/main/app/canvas.ts +++ b/tracker/tracker/src/main/app/canvas.ts @@ -5,12 +5,15 @@ import Message, { CanvasNode } from './messages.gen.js' interface CanvasSnapshot { images: { data: string; id: number }[] createdAt: number + paused: boolean + dummy: HTMLCanvasElement } interface Options { fps: number quality: 'low' | 'medium' | 'high' isDebug?: boolean + fixedScaling?: boolean } class CanvasRecorder { @@ -27,19 +30,19 @@ class CanvasRecorder { startTracking() { setTimeout(() => { - this.app.nodes.scanTree(this.handleCanvasEl) + this.app.nodes.scanTree(this.captureCanvas) this.app.nodes.attachNodeCallback((node: Node): void => { - this.handleCanvasEl(node) + this.captureCanvas(node) }) }, 500) } restartTracking = () => { this.clear() - this.app.nodes.scanTree(this.handleCanvasEl) + this.app.nodes.scanTree(this.captureCanvas) } - handleCanvasEl = (node: Node) => { + captureCanvas = (node: Node) => { const id = this.app.nodes.getID(node) if (!id || !hasTag(node, 'canvas')) { return @@ -49,10 +52,41 @@ class CanvasRecorder { if (isIgnored || !hasTag(node, 'canvas') || this.snapshots[id]) { return } + + const observer = new IntersectionObserver((entries) => { + entries.forEach((entry) => { + if (entry.isIntersecting) { + if (entry.target) { + if (this.snapshots[id] && this.snapshots[id].createdAt) { + this.snapshots[id].paused = false + } else { + this.recordCanvas(entry.target, id) + } + /** + * We can switch this to start observing when element is in the view + * but otherwise right now we're just pausing when it's not + * just to save some bandwidth and space on backend + * */ + // observer.unobserve(entry.target) + } else { + if (this.snapshots[id]) { + this.snapshots[id].paused = true + } + } + } + }) + }) + + observer.observe(node) + } + + recordCanvas = (node: Node, id: number) => { const ts = this.app.timestamp() this.snapshots[id] = { images: [], createdAt: ts, + paused: false, + dummy: document.createElement('canvas'), } const canvasMsg = CanvasNode(id.toString(), ts) this.app.send(canvasMsg as Message) @@ -63,11 +97,18 @@ class CanvasRecorder { this.app.debug.log('Canvas element not in sync') clearInterval(int) } else { - const snapshot = captureSnapshot(canvas, this.options.quality) - this.snapshots[id].images.push({ id: this.app.timestamp(), data: snapshot }) - if (this.snapshots[id].images.length > 9) { - this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt) - this.snapshots[id].images = [] + if (!this.snapshots[id].paused) { + const snapshot = captureSnapshot( + canvas, + this.options.quality, + this.snapshots[id].dummy, + this.options.fixedScaling, + ) + this.snapshots[id].images.push({ id: this.app.timestamp(), data: snapshot }) + if (this.snapshots[id].images.length > 9) { + this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt) + this.snapshots[id].images = [] + } } } }, this.interval) @@ -110,18 +151,36 @@ class CanvasRecorder { } const qualityInt = { - low: 0.33, + low: 0.35, medium: 0.55, high: 0.8, } -function captureSnapshot(canvas: HTMLCanvasElement, quality: 'low' | 'medium' | 'high' = 'medium') { +function captureSnapshot( + canvas: HTMLCanvasElement, + quality: 'low' | 'medium' | 'high' = 'medium', + dummy: HTMLCanvasElement, + fixedScaling = false, +) { const imageFormat = 'image/jpeg' // or /png' - return canvas.toDataURL(imageFormat, qualityInt[quality]) + if (fixedScaling) { + const canvasScaleRatio = window.devicePixelRatio || 1 + dummy.width = canvas.width / canvasScaleRatio + dummy.height = canvas.height / canvasScaleRatio + const ctx = dummy.getContext('2d') + if (!ctx) { + return '' + } + ctx.drawImage(canvas, 0, 0, dummy.width, dummy.height) + return dummy.toDataURL(imageFormat, qualityInt[quality]) + } else { + return canvas.toDataURL(imageFormat, qualityInt[quality]) + } } function dataUrlToBlob(dataUrl: string): [Blob, Uint8Array] | null { const [header, base64] = dataUrl.split(',') + if (!header || !base64) return null const encParts = header.match(/:(.*?);/) if (!encParts) return null const mime = encParts[1] diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index ef34dc67a..a2872a877 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -117,6 +117,7 @@ type AppOptions = { __debug_report_edp: string | null __debug__?: ILogLevel __save_canvas_locally?: boolean + fixedCanvasScaling?: boolean localStorage: Storage | null sessionStorage: Storage | null forceSingleTab?: boolean @@ -211,6 +212,7 @@ export default class App { disableStringDict: false, forceSingleTab: false, assistSocketHost: '', + fixedCanvasScaling: false, }, options, ) @@ -1063,6 +1065,7 @@ export default class App { fps: canvasFPS, quality: canvasQuality, isDebug: this.options.__save_canvas_locally, + fixedScaling: this.options.fixedCanvasScaling, }) this.canvasRecorder.startTracking() }