From 5db94fc172ac3db3c6727ff5cd1e48b6b68984ba Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 7 Jun 2024 13:33:17 +0200 Subject: [PATCH] feat ui: optimize canvas recording --- .../app/player/web/managers/CanvasManager.ts | 2 +- tracker/tracker/CHANGELOG.md | 5 ++ tracker/tracker/package.json | 2 +- tracker/tracker/src/main/app/canvas.ts | 54 ++++++++++++------- tracker/tracker/src/main/app/index.ts | 36 +++++++++++-- 5 files changed, 75 insertions(+), 24 deletions(-) diff --git a/frontend/app/player/web/managers/CanvasManager.ts b/frontend/app/player/web/managers/CanvasManager.ts index e65449b93..a8e61cc37 100644 --- a/frontend/app/player/web/managers/CanvasManager.ts +++ b/frontend/app/player/web/managers/CanvasManager.ts @@ -86,7 +86,7 @@ export default class CanvasManager extends ListWalker { public mapToSnapshots(files: TarFile[]) { const tempArr: Timestamp[] = []; - const filenameRegexp = /(\d+)_(\d+)_(\d+)\.jpeg$/; + const filenameRegexp = /(\d+)_(\d+)_(\d+)\.webp$/; const firstPair = files[0].name.match(filenameRegexp); if (!firstPair) { console.error('Invalid file name format', files[0].name); diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index 719066509..769ef43ea 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -1,3 +1,8 @@ +# 13.0.1 + +- moved canvas snapshots to webp, additional option to utilize useAnimationFrame method (for webgl) +- simpler, faster canvas recording manager + # 13.0.0 - `assistOnly` flag for tracker options (EE only feature) diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 936e48062..79797e080 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "13.0.0", + "version": "13.0.1-4", "keywords": [ "logging", "replay" diff --git a/tracker/tracker/src/main/app/canvas.ts b/tracker/tracker/src/main/app/canvas.ts index dca48da56..0f6c12460 100644 --- a/tracker/tracker/src/main/app/canvas.ts +++ b/tracker/tracker/src/main/app/canvas.ts @@ -3,7 +3,7 @@ import { hasTag } from './guards.js' import Message, { CanvasNode } from './messages.gen.js' interface CanvasSnapshot { - images: { data: string; id: number }[] + images: { data: Blob; id: number }[] createdAt: number paused: boolean dummy: HTMLCanvasElement @@ -14,6 +14,7 @@ interface Options { quality: 'low' | 'medium' | 'high' isDebug?: boolean fixedScaling?: boolean + useAnimationFrame?: boolean } class CanvasRecorder { @@ -90,6 +91,23 @@ class CanvasRecorder { } const canvasMsg = CanvasNode(id.toString(), ts) this.app.send(canvasMsg as Message) + + const captureFn = (canvas: HTMLCanvasElement) => { + captureSnapshot( + canvas, + this.options.quality, + this.snapshots[id].dummy, + this.options.fixedScaling, + (blob) => { + if (!blob) return + this.snapshots[id].images.push({ id: this.app.timestamp(), data: blob }) + if (this.snapshots[id].images.length > 9) { + this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt) + this.snapshots[id].images = [] + } + }, + ) + } const int = setInterval(() => { const cid = this.app.nodes.getID(node) const canvas = cid ? this.app.nodes.getNode(cid) : undefined @@ -98,16 +116,12 @@ class CanvasRecorder { clearInterval(int) } else { if (!this.snapshots[id].paused) { - const snapshot = captureSnapshot( - canvas, - this.options.quality, - this.snapshots[id].dummy, - this.options.fixedScaling, - ) - this.snapshots[id].images.push({ id: this.app.timestamp(), data: snapshot }) - if (this.snapshots[id].images.length > 9) { - this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt) - this.snapshots[id].images = [] + if (this.options.useAnimationFrame) { + requestAnimationFrame(() => { + captureFn(canvas) + }) + } else { + captureFn(canvas) } } } @@ -115,17 +129,17 @@ class CanvasRecorder { this.intervals.push(int) } - sendSnaps(images: { data: string; id: number }[], canvasId: number, createdAt: number) { + sendSnaps(images: { data: Blob; id: number }[], canvasId: number, createdAt: number) { if (Object.keys(this.snapshots).length === 0) { return } const formData = new FormData() images.forEach((snapshot) => { - const blob = dataUrlToBlob(snapshot.data) + const blob = snapshot.data if (!blob) return - formData.append('snapshot', blob[0], `${createdAt}_${canvasId}_${snapshot.id}.jpeg`) + formData.append('snapshot', blob, `${createdAt}_${canvasId}_${snapshot.id}.webp`) if (this.options.isDebug) { - saveImageData(snapshot.data, `${createdAt}_${canvasId}_${snapshot.id}.jpeg`) + saveImageData(blob, `${createdAt}_${canvasId}_${snapshot.id}.webp`) } }) @@ -161,8 +175,9 @@ function captureSnapshot( quality: 'low' | 'medium' | 'high' = 'medium', dummy: HTMLCanvasElement, fixedScaling = false, + onBlob: (blob: Blob | null) => void, ) { - const imageFormat = 'image/jpeg' // or /png' + const imageFormat = 'image/webp' if (fixedScaling) { const canvasScaleRatio = window.devicePixelRatio || 1 dummy.width = canvas.width / canvasScaleRatio @@ -172,9 +187,9 @@ function captureSnapshot( return '' } ctx.drawImage(canvas, 0, 0, dummy.width, dummy.height) - return dummy.toDataURL(imageFormat, qualityInt[quality]) + dummy.toBlob(onBlob, imageFormat, qualityInt[quality]) } else { - return canvas.toDataURL(imageFormat, qualityInt[quality]) + canvas.toBlob(onBlob, imageFormat, qualityInt[quality]) } } @@ -195,7 +210,8 @@ function dataUrlToBlob(dataUrl: string): [Blob, Uint8Array] | null { return [new Blob([u8arr], { type: mime }), u8arr] } -function saveImageData(imageDataUrl: string, name: string) { +function saveImageData(imageDataBlob: Blob, name: string) { + const imageDataUrl = URL.createObjectURL(imageDataBlob) const link = document.createElement('a') link.href = imageDataUrl link.download = name diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 993d3d9fd..388876de8 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -125,6 +125,12 @@ type AppOptions = { disableStringDict?: boolean assistSocketHost?: string disableCanvas?: boolean + canvas: { + disableCanvas?: boolean + fixedCanvasScaling?: boolean + __save_canvas_locally?: boolean + useAnimationFrame?: boolean + } /** @deprecated */ onStart?: StartCallback @@ -192,6 +198,23 @@ export default class App { options: Partial, private readonly signalError: (error: string, apis: string[]) => void, ) { + if ( + Object.keys(options).findIndex((k) => ['fixedCanvasScaling', 'disableCanvas'].includes(k)) !== + -1 + ) { + console.warn( + 'Openreplay: canvas options are moving to separate key "canvas" in next update. Please update your configuration.', + ) + options = { + ...options, + canvas: { + __save_canvas_locally: options.__save_canvas_locally, + fixedCanvasScaling: options.fixedCanvasScaling, + disableCanvas: options.disableCanvas, + }, + } + } + this.contextId = Math.random().toString(36).slice(2) this.projectKey = projectKey this.networkOptions = options.network @@ -218,6 +241,12 @@ export default class App { fixedCanvasScaling: false, disableCanvas: false, assistOnly: false, + canvas: { + disableCanvas: false, + fixedCanvasScaling: false, + __save_canvas_locally: false, + useAnimationFrame: false, + }, }, options, ) @@ -1085,14 +1114,15 @@ export default class App { await this.tagWatcher.fetchTags(this.options.ingestPoint, token) this.activityState = ActivityState.Active - if (canvasEnabled && !this.options.disableCanvas) { + if (canvasEnabled && !this.options.canvas.disableCanvas) { this.canvasRecorder = this.canvasRecorder ?? new CanvasRecorder(this, { fps: canvasFPS, quality: canvasQuality, - isDebug: this.options.__save_canvas_locally, - fixedScaling: this.options.fixedCanvasScaling, + isDebug: this.options.canvas.__save_canvas_locally, + fixedScaling: this.options.canvas.fixedCanvasScaling, + useAnimationFrame: this.options.canvas.useAnimationFrame, }) this.canvasRecorder.startTracking() }