From b8ed5c5d9efa5758274a03b0acf2c1b7283243b9 Mon Sep 17 00:00:00 2001 From: Delirium Date: Wed, 10 Apr 2024 16:28:16 +0200 Subject: [PATCH] fix ui: fix canvas replay manager, fix mob sorting (#2071) * fix ui: fix canvas replay manager, fix mob sorting * fix whitespaces --- frontend/app/player/common/ListWalker.ts | 42 +++++++ frontend/app/player/web/MessageLoader.ts | 115 ++++++++++++++---- frontend/app/player/web/TabManager.ts | 47 ++++--- .../app/player/web/managers/CanvasManager.ts | 96 ++++++++++++--- .../app/player/web/managers/DOM/DOMManager.ts | 19 +-- .../player/web/managers/DOM/safeCSSRules.ts | 17 ++- tracker/tracker/src/main/app/canvas.ts | 2 +- 7 files changed, 265 insertions(+), 73 deletions(-) diff --git a/frontend/app/player/common/ListWalker.ts b/frontend/app/player/common/ListWalker.ts index 90a87bcca..d29fa3c1c 100644 --- a/frontend/app/player/common/ListWalker.ts +++ b/frontend/app/player/common/ListWalker.ts @@ -137,6 +137,48 @@ export default class ListWalker { return changed ? this.list[ this.p - 1 ] : null; } + prevTs = 0; + getNew(t: number, index?: number): T | null { + let key: string = "time"; //TODO + let val = t; + + let changed = this.prevTs > t; + this.prevTs = t; + // @ts-ignore + while (this.p < this.length && this.list[this.p][key] <= val) { + this.moveNext() + changed = true; + } + // @ts-ignore + while (this.p > 0 && this.list[ this.p - 1 ][key] > val) { + this.movePrev() + changed = true; + } + return changed ? this.list[ this.p - 1 ] : null; + } + + + findLast(t: number): T | null { + let left = 0; + let right = this.list.length - 1; + let result: T | null = null; + + while (left <= right) { + const mid = Math.floor((left + right) / 2); + const currentItem = this.list[mid]; + + if (currentItem.time <= t) { + result = currentItem; + left = mid + 1; + } else { + right = mid - 1; + } + } + + return result; + } + + /** * Moves over the messages starting from the current+1 to the last one with the time <= t * applying callback on each of them diff --git a/frontend/app/player/web/MessageLoader.ts b/frontend/app/player/web/MessageLoader.ts index 059c86080..6e0b92358 100644 --- a/frontend/app/player/web/MessageLoader.ts +++ b/frontend/app/player/web/MessageLoader.ts @@ -105,33 +105,8 @@ export default class MessageLoader { } }); - const DOMMessages = [ - MType.CreateElementNode, - MType.CreateTextNode, - MType.MoveNode, - MType.RemoveNode, - ]; - const sortedMsgs = msgs.sort((m1, m2) => { - if (m1.time !== m2.time) return m1.time - m2.time; - - if (m1.tp === MType.CreateDocument && m2.tp !== MType.CreateDocument) - return -1; - if (m1.tp !== MType.CreateDocument && m2.tp === MType.CreateDocument) - return 1; - - const m1IsDOM = DOMMessages.includes(m1.tp); - const m2IsDOM = DOMMessages.includes(m2.tp); - if (m1IsDOM && m2IsDOM) { - // @ts-ignore DOM msg has id but checking for 'id' in m is expensive - if (m1.id !== m2.id) return m1.id - m2.id; - return m1.tp - m2.tp; - } - - if (m1IsDOM && !m2IsDOM) return -1; - if (!m1IsDOM && m2IsDOM) return 1; - - return 0; - }); + const sortedMsgs = msgs.sort((m1, m2) => m1.time - m2.time); + // .sort(brokenDomSorter); if (brokenMessages > 0) { console.warn('Broken timestamp messages', brokenMessages); @@ -295,3 +270,89 @@ export default class MessageLoader { this.store.update(MessageLoader.INITIAL_STATE); } } + +const DOMMessages = [ + MType.CreateElementNode, + MType.CreateTextNode, + MType.MoveNode, + MType.RemoveNode, +]; + +function brokenDomSorter(m1: PlayerMsg, m2: PlayerMsg) { + if (m1.time !== m2.time) return m1.time - m2.time; + + if (m1.tp === MType.CreateDocument && m2.tp !== MType.CreateDocument) + return -1; + if (m1.tp !== MType.CreateDocument && m2.tp === MType.CreateDocument) + return 1; + + // if (m1.tp === MType.CreateIFrameDocument && m2.tp === MType.CreateElementNode) { + // if (m2.id === m1.frameID) return 1; + // if (m2.parentID === m1.id) return -1 + // } + // if (m1.tp === MType.CreateElementNode && m2.tp === MType.CreateIFrameDocument) { + // if (m1.id === m2.frameID) return -1; + // if (m1.parentID === m2.id) return 1 + // } + const m1IsDOM = DOMMessages.includes(m1.tp); + const m2IsDOM = DOMMessages.includes(m2.tp); + if (m1IsDOM && m2IsDOM) { + // @ts-ignore DOM msg has id but checking for 'id' in m is expensive + if (m1.id !== m2.id) return m1.id - m2.id; + return m1.tp - m2.tp; + } + + if (m1IsDOM && !m2IsDOM) return -1; + if (!m1IsDOM && m2IsDOM) return 1; + + return 0; +} + +/** + * Search for orphan nodes in session + */ +function findBrokenNodes(nodes: any[]) { + const idToNode = {}; + const orphans: any[] = []; + const result = {}; + + // Map all nodes by id for quick access and identify potential orphans + nodes.forEach((node) => { + // @ts-ignore + idToNode[node.id] = { ...node, children: [] }; + }); + + // Identify true orphans (nodes whose parentID does not exist) + nodes.forEach((node) => { + if (node.parentID) { + // @ts-ignore + const parentNode = idToNode[node.parentID]; + if (parentNode) { + // @ts-ignore + parentNode.children.push(idToNode[node.id]); + } else { + orphans.push(node.id); // parentID does not exist + } + } + }); + + // Recursively collect all descendants of a node + function collectDescendants(nodeId) { + // @ts-ignore + const node = idToNode[nodeId]; + node.children.forEach((child) => { + collectDescendants(child.id); + }); + return node; + } + + // Build trees for each orphan + orphans.forEach((orId: number) => { + // @ts-ignore + result[orId] = collectDescendants(orId); + }); + + return result; +} + +window.searchOrphans = findBrokenNodes; diff --git a/frontend/app/player/web/TabManager.ts b/frontend/app/player/web/TabManager.ts index 66361f559..6fec1e3be 100644 --- a/frontend/app/player/web/TabManager.ts +++ b/frontend/app/player/web/TabManager.ts @@ -1,15 +1,15 @@ import type { Store } from 'Player'; import { - getResourceFromNetworkRequest, - getResourceFromResourceTiming, Log, ResourceType, + getResourceFromNetworkRequest, + getResourceFromResourceTiming } from 'Player'; import ListWalker from 'Player/common/ListWalker'; import Lists, { - INITIAL_STATE as LISTS_INITIAL_STATE, InitialLists, - State as ListsState, + INITIAL_STATE as LISTS_INITIAL_STATE, + State as ListsState } from 'Player/web/Lists'; import CanvasManager from 'Player/web/managers/CanvasManager'; import { VElement } from 'Player/web/managers/DOM/VirtualDOM'; @@ -19,20 +19,22 @@ import WindowNodeCounter from 'Player/web/managers/WindowNodeCounter'; import { CanvasNode, ConnectionInformation, - Message, MType, + Message, ResourceTiming, SetPageLocation, SetViewportScroll, - SetViewportSize, + SetViewportSize } from 'Player/web/messages'; import { isDOMType } from 'Player/web/messages/filters.gen'; +import { TYPES as EVENT_TYPES } from 'Types/session/event'; import Screen from 'Player/web/Screen/Screen'; // @ts-ignore import { Decoder } from 'syncod'; -import { TYPES as EVENT_TYPES } from 'Types/session/event'; + import type { PerformanceChartPoint } from './managers/PerformanceTrackManager'; + export interface TabState extends ListsState { performanceAvailability?: PerformanceTrackManager['availability']; performanceChartData: PerformanceChartPoint[]; @@ -182,7 +184,8 @@ export default class TabSessionManager { msg.nodeId, delta, [tarball, mp4file], - this.getNode as (id: number) => VElement | undefined + this.getNode as (id: number) => VElement | undefined, + this.sessionStart, ); this.canvasManagers[managerId] = { manager, start: msg.timestamp, running: false }; this.canvasReplayWalker.append(msg); @@ -278,6 +281,17 @@ export default class TabSessionManager { this.windowNodeCounter.removeNode(msg.id); this.performanceTrackManager.setCurrentNodesCount(this.windowNodeCounter.count); break; + case MType.LoadFontFace: + if (msg.source.startsWith('url(/')) { + const relativeUrl = msg.source.substring(4); + const lastUrl = this.locationManager.findLast(msg.time)?.url + if (lastUrl) { + const u = new URL(lastUrl); + const base = u.protocol + '//' + u.hostname + '/'; + msg.source = `url(${base}${relativeUrl}`; + } + } + break; } this.performanceTrackManager.addNodeCountPointIfNeed(msg.time); isDOMType(msg.tp) && this.pagesManager.appendMessage(msg); @@ -340,15 +354,18 @@ export default class TabSessionManager { } this.canvasReplayWalker.moveApply(t, (canvasMsg) => { if (canvasMsg) { - this.canvasManagers[`${canvasMsg.timestamp}_${canvasMsg.nodeId}`].manager.startVideo(); - this.canvasManagers[`${canvasMsg.timestamp}_${canvasMsg.nodeId}`].running = true; + const managerId = `${canvasMsg.timestamp}_${canvasMsg.nodeId}` + const possibleManager = this.canvasManagers[managerId] + if (possibleManager && !possibleManager.running) { + this.canvasManagers[managerId].manager.startVideo(); + this.canvasManagers[managerId].running = true; + } } }) - const runningManagers = Object.keys(this.canvasManagers).filter( - (key) => this.canvasManagers[key].running - ); - runningManagers.forEach((key) => { - const manager = this.canvasManagers[key].manager; + const runningManagers = Object.values(this.canvasManagers).filter( + (manager) => manager.running + ) + runningManagers.forEach(({ manager }) => { manager.move(t); }); }); diff --git a/frontend/app/player/web/managers/CanvasManager.ts b/frontend/app/player/web/managers/CanvasManager.ts index f7cc4b958..e65449b93 100644 --- a/frontend/app/player/web/managers/CanvasManager.ts +++ b/frontend/app/player/web/managers/CanvasManager.ts @@ -1,8 +1,8 @@ -import { TarFile } from 'js-untar'; import ListWalker from 'Player/common/ListWalker'; -import { VElement } from 'Player/web/managers/DOM/VirtualDOM'; -import unpack from 'Player/common/unpack'; import unpackTar from 'Player/common/tarball'; +import unpack from 'Player/common/unpack'; +import { VElement } from 'Player/web/managers/DOM/VirtualDOM'; +import { TarFile } from 'js-untar'; const playMode = { video: 'video', @@ -21,6 +21,7 @@ export default class CanvasManager extends ListWalker { private lastTs = 0; private playMode: string = playMode.snaps; private snapshots: Record = {}; + private debugCanvas: HTMLCanvasElement | undefined; constructor( /** @@ -32,7 +33,8 @@ export default class CanvasManager extends ListWalker { */ private readonly delta: number, private readonly links: [tar?: string, mp4?: string], - private readonly getNode: (id: number) => VElement | undefined + private readonly getNode: (id: number) => VElement | undefined, + private readonly sessionStart: number, ) { super(); // first we try to grab tar, then fallback to mp4 @@ -52,9 +54,34 @@ export default class CanvasManager extends ListWalker { } }); } else { - return console.error('Failed to load canvas recording for node', this.nodeId); + return console.error( + 'Failed to load canvas recording for node', + this.nodeId + ); } }); + + // @ts-ignore + if (window.__or_debug === true) { + let debugContainer = document.querySelector('.imgDebug'); + if (!debugContainer) { + debugContainer = document.createElement('div'); + debugContainer.className = 'imgDebug'; + Object.assign(debugContainer.style, { + position: 'fixed', + top: '0', + left: 0, + display: 'flex', + flexDirection: 'column', + }); + document.body.appendChild(debugContainer); + } + const debugCanvas = document.createElement('canvas'); + debugCanvas.width = 300; + debugCanvas.height = 200; + this.debugCanvas = debugCanvas; + debugContainer.appendChild(debugCanvas); + } } public mapToSnapshots(files: TarFile[]) { @@ -65,13 +92,14 @@ export default class CanvasManager extends ListWalker { console.error('Invalid file name format', files[0].name); return; } - const sessionStart = firstPair ? parseInt(firstPair[1], 10) : 0; + files.forEach((file) => { - const [_, _1, _2, imageTimestampStr] = file.name.match(filenameRegexp) ?? [0, 0, 0, '0']; + const [_, _1, _2, imageTimestampStr] = file.name.match( + filenameRegexp + ) ?? [0, 0, 0, '0']; const imageTimestamp = parseInt(imageTimestampStr, 10); - - const messageTime = imageTimestamp - sessionStart; + const messageTime = imageTimestamp - this.sessionStart; this.snapshots[messageTime] = file; tempArr.push({ time: messageTime }); }); @@ -127,19 +155,30 @@ export default class CanvasManager extends ListWalker { if (node && node.node) { const canvasCtx = (node.node as HTMLCanvasElement).getContext('2d'); const canvasEl = node.node as HTMLVideoElement; - canvasCtx?.drawImage(this.snapImage, 0, 0, canvasEl.width, canvasEl.height); + canvasCtx?.drawImage( + this.snapImage, + 0, + 0, + canvasEl.width, + canvasEl.height + ); + this.debugCanvas + ?.getContext('2d') + ?.drawImage(this.snapImage, 0, 0, 300, 200); } else { console.error(`CanvasManager: Node ${this.nodeId} not found`); + } }; + } else { + if (!this.fileData) return; + this.videoTag.setAttribute('autoplay', 'true'); + this.videoTag.setAttribute('muted', 'true'); + this.videoTag.setAttribute('playsinline', 'true'); + this.videoTag.setAttribute('crossorigin', 'anonymous'); + this.videoTag.src = this.fileData; + this.videoTag.currentTime = 0; } - if (!this.fileData) return; - this.videoTag.setAttribute('autoplay', 'true'); - this.videoTag.setAttribute('muted', 'true'); - this.videoTag.setAttribute('playsinline', 'true'); - this.videoTag.setAttribute('crossorigin', 'anonymous'); - this.videoTag.src = this.fileData; - this.videoTag.currentTime = 0; }; move(t: number) { @@ -163,15 +202,21 @@ export default class CanvasManager extends ListWalker { void this.videoTag.pause(); } this.videoTag.currentTime = playTime / 1000; - canvasCtx?.drawImage(this.videoTag, 0, 0, canvasEl.width, canvasEl.height); + canvasCtx?.drawImage( + this.videoTag, + 0, + 0, + canvasEl.width, + canvasEl.height + ); } else { - console.error(`CanvasManager: Node ${this.nodeId} not found`); + console.error(`VideoMode CanvasManager: Node ${this.nodeId} not found`); } } }; moveReadySnap = (t: number) => { - const msg = this.moveGetLast(t); + const msg = this.getNew(t); if (msg) { const file = this.snapshots[msg.time]; if (file) { @@ -180,3 +225,14 @@ export default class CanvasManager extends ListWalker { } }; } + +function saveImageData(imageDataUrl: string, name: string) { + const link = document.createElement('a'); + link.href = imageDataUrl; + link.download = name; + link.style.display = 'none'; + + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); +} diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index baa30d300..ff61f6f3f 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -1,4 +1,5 @@ import logger from 'App/logger'; +import { resolveURL } from "../../messages/rewriter/urlResolve"; import type Screen from '../../Screen/Screen'; import type { Message, SetNodeScroll } from '../../messages'; @@ -393,15 +394,17 @@ export default class DOMManager extends ListWalker { vRoot.whenReady(vNode => { if (vNode instanceof VShadowRoot) { logger.error(`Node ${vNode} expected to be a Document`, msg); return } let descr: Object | undefined - try { - descr = JSON.parse(msg.descriptors) - descr = typeof descr === 'object' ? descr : undefined - } catch { - logger.warn("Can't parse font-face descriptors: ", msg) + if (msg.descriptors) { + try { + descr = JSON.parse(msg.descriptors) + descr = typeof descr === 'object' ? descr : undefined + } catch { + logger.warn("Can't parse font-face descriptors: ", msg) + } } const ff = new FontFace(msg.family, msg.source, descr) vNode.node.fonts.add(ff) - ff.load() // TODOTODO: wait for this one in StylesManager in a common way with styles + void ff.load() }) return } @@ -410,11 +413,11 @@ export default class DOMManager extends ListWalker { /** * Moves and applies all the messages from the current (or from the beginning, if t < current.time) - * to the one with msg.time >= `t` + * to the one with msg[time] >= `t` * * This function autoresets pointer if necessary (better name?) * - * @returns Promise that fulfulls when necessary changes get applied + * @returns Promise that fulfills when necessary changes get applied * (the async part exists mostly due to styles loading) */ async moveReady(t: number): Promise { diff --git a/frontend/app/player/web/managers/DOM/safeCSSRules.ts b/frontend/app/player/web/managers/DOM/safeCSSRules.ts index e97caa82f..22dec1c77 100644 --- a/frontend/app/player/web/managers/DOM/safeCSSRules.ts +++ b/frontend/app/player/web/managers/DOM/safeCSSRules.ts @@ -1,17 +1,30 @@ import logger from 'App/logger'; +function isChromium(item) { + return ['Chromium', 'Google Chrome', 'NewBrowser'].includes(item.brand); +} +// @ts-ignore +const isChromeLike = navigator.userAgentData.brands.some(isChromium) + export function insertRule( sheet: { insertRule: (rule: string, index?: number) => void }, msg: { rule: string, index: number } ) { - + /** + * inserting -moz- styles in chrome-like browsers causes issues and warnings + * changing them to -webkit- is usually fine because they're covered by native styles + * and not inserting them will break sheet indexing + * */ + if (msg.rule.includes('-moz-') && isChromeLike) { + msg.rule = msg.rule.replace(/-moz-/g, '-webkit-') + } try { sheet.insertRule(msg.rule, msg.index) } catch (e) { try { sheet.insertRule(msg.rule) } catch (e) { - logger.warn("Cannot insert rule.", e, msg) + logger.warn("Cannot insert rule.", e, '\nmessage: ', msg) } } } diff --git a/tracker/tracker/src/main/app/canvas.ts b/tracker/tracker/src/main/app/canvas.ts index f2c7934e9..dca48da56 100644 --- a/tracker/tracker/src/main/app/canvas.ts +++ b/tracker/tracker/src/main/app/canvas.ts @@ -107,7 +107,7 @@ class CanvasRecorder { this.snapshots[id].images.push({ id: this.app.timestamp(), data: snapshot }) if (this.snapshots[id].images.length > 9) { this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt) - this.snapshots[id].images = [] + this.snapshots[id].images = [] } } }