jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix ui: fix canvas replay manager, fix mob sorting (#2071)

Browse source 
* fix ui: fix canvas replay manager, fix mob sorting

* fix whitespaces
This commit is contained in:
Delirium 2024-04-10 16:28:16 +02:00 • committed by GitHub
parent 67fc534414
commit b8ed5c5d9e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 265 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
frontend/app/player/common/ListWalker.ts
View file

@ -137,6 +137,48 @@ export default class ListWalker<T extends Timed> {
return changed ? this.list[ this.p - 1 ] : null;
}
prevTs = 0;
getNew(t: number, index?: number): T | null {
let key: string = "time"; //TODO
let val = t;
let changed = this.prevTs > t;
this.prevTs = t;
// @ts-ignore
while (this.p < this.length && this.list[this.p][key] <= val) {
this.moveNext()
changed = true;
}
// @ts-ignore
while (this.p > 0 && this.list[ this.p - 1 ][key] > val) {
this.movePrev()
changed = true;
}
return changed ? this.list[ this.p - 1 ] : null;
}
findLast(t: number): T | null {
let left = 0;
let right = this.list.length - 1;
let result: T | null = null;
while (left <= right) {
const mid = Math.floor((left + right) / 2);
const currentItem = this.list[mid];
if (currentItem.time <= t) {
result = currentItem;
left = mid + 1;
} else {
right = mid - 1;
}
}
return result;
}
/**
* Moves over the messages starting from the current+1 to the last one with the time <= t
* applying callback on each of them

115
frontend/app/player/web/MessageLoader.ts
View file

@ -105,33 +105,8 @@ export default class MessageLoader {
}
});
const DOMMessages = [
MType.CreateElementNode,
MType.CreateTextNode,
MType.MoveNode,
MType.RemoveNode,
];
const sortedMsgs = msgs.sort((m1, m2) => {
if (m1.time !== m2.time) return m1.time - m2.time;
if (m1.tp === MType.CreateDocument && m2.tp !== MType.CreateDocument)
return -1;
if (m1.tp !== MType.CreateDocument && m2.tp === MType.CreateDocument)
return 1;
const m1IsDOM = DOMMessages.includes(m1.tp);
const m2IsDOM = DOMMessages.includes(m2.tp);
if (m1IsDOM && m2IsDOM) {
// @ts-ignore DOM msg has id but checking for 'id' in m is expensive
if (m1.id !== m2.id) return m1.id - m2.id;
return m1.tp - m2.tp;
}
if (m1IsDOM && !m2IsDOM) return -1;
if (!m1IsDOM && m2IsDOM) return 1;
return 0;
});
const sortedMsgs = msgs.sort((m1, m2) => m1.time - m2.time);
// .sort(brokenDomSorter);
if (brokenMessages > 0) {
console.warn('Broken timestamp messages', brokenMessages);
@ -295,3 +270,89 @@ export default class MessageLoader {
this.store.update(MessageLoader.INITIAL_STATE);
}
}
const DOMMessages = [
MType.CreateElementNode,
MType.CreateTextNode,
MType.MoveNode,
MType.RemoveNode,
];
function brokenDomSorter(m1: PlayerMsg, m2: PlayerMsg) {
if (m1.time !== m2.time) return m1.time - m2.time;
if (m1.tp === MType.CreateDocument && m2.tp !== MType.CreateDocument)
return -1;
if (m1.tp !== MType.CreateDocument && m2.tp === MType.CreateDocument)
return 1;
// if (m1.tp === MType.CreateIFrameDocument && m2.tp === MType.CreateElementNode) {
// if (m2.id === m1.frameID) return 1;
// if (m2.parentID === m1.id) return -1
// }
// if (m1.tp === MType.CreateElementNode && m2.tp === MType.CreateIFrameDocument) {
// if (m1.id === m2.frameID) return -1;
// if (m1.parentID === m2.id) return 1
// }
const m1IsDOM = DOMMessages.includes(m1.tp);
const m2IsDOM = DOMMessages.includes(m2.tp);
if (m1IsDOM && m2IsDOM) {
// @ts-ignore DOM msg has id but checking for 'id' in m is expensive
if (m1.id !== m2.id) return m1.id - m2.id;
return m1.tp - m2.tp;
}
if (m1IsDOM && !m2IsDOM) return -1;
if (!m1IsDOM && m2IsDOM) return 1;
return 0;
}
/**
* Search for orphan nodes in session
*/
function findBrokenNodes(nodes: any[]) {
const idToNode = {};
const orphans: any[] = [];
const result = {};
// Map all nodes by id for quick access and identify potential orphans
nodes.forEach((node) => {
// @ts-ignore
idToNode[node.id] = { ...node, children: [] };
});
// Identify true orphans (nodes whose parentID does not exist)
nodes.forEach((node) => {
if (node.parentID) {
// @ts-ignore
const parentNode = idToNode[node.parentID];
if (parentNode) {
// @ts-ignore
parentNode.children.push(idToNode[node.id]);
} else {
orphans.push(node.id); // parentID does not exist
}
}
});
// Recursively collect all descendants of a node
function collectDescendants(nodeId) {
// @ts-ignore
const node = idToNode[nodeId];
node.children.forEach((child) => {
collectDescendants(child.id);
});
return node;
}
// Build trees for each orphan
orphans.forEach((orId: number) => {
// @ts-ignore
result[orId] = collectDescendants(orId);
});
return result;
}
window.searchOrphans = findBrokenNodes;

47
frontend/app/player/web/TabManager.ts
View file

@ -1,15 +1,15 @@
import type { Store } from 'Player';
import {
getResourceFromNetworkRequest,
getResourceFromResourceTiming,
Log,
ResourceType,
getResourceFromNetworkRequest,
getResourceFromResourceTiming
} from 'Player';
import ListWalker from 'Player/common/ListWalker';
import Lists, {
INITIAL_STATE as LISTS_INITIAL_STATE,
InitialLists,
State as ListsState,
INITIAL_STATE as LISTS_INITIAL_STATE,
State as ListsState
} from 'Player/web/Lists';
import CanvasManager from 'Player/web/managers/CanvasManager';
import { VElement } from 'Player/web/managers/DOM/VirtualDOM';
@ -19,20 +19,22 @@ import WindowNodeCounter from 'Player/web/managers/WindowNodeCounter';
import {
CanvasNode,
ConnectionInformation,
Message,
MType,
Message,
ResourceTiming,
SetPageLocation,
SetViewportScroll,
SetViewportSize,
SetViewportSize
} from 'Player/web/messages';
import { isDOMType } from 'Player/web/messages/filters.gen';
import { TYPES as EVENT_TYPES } from 'Types/session/event';
import Screen from 'Player/web/Screen/Screen';
// @ts-ignore
import { Decoder } from 'syncod';
import { TYPES as EVENT_TYPES } from 'Types/session/event';
import type { PerformanceChartPoint } from './managers/PerformanceTrackManager';
export interface TabState extends ListsState {
performanceAvailability?: PerformanceTrackManager['availability'];
performanceChartData: PerformanceChartPoint[];
@ -182,7 +184,8 @@ export default class TabSessionManager {
msg.nodeId,
delta,
[tarball, mp4file],
this.getNode as (id: number) => VElement | undefined
this.getNode as (id: number) => VElement | undefined,
this.sessionStart,
);
this.canvasManagers[managerId] = { manager, start: msg.timestamp, running: false };
this.canvasReplayWalker.append(msg);
@ -278,6 +281,17 @@ export default class TabSessionManager {
this.windowNodeCounter.removeNode(msg.id);
this.performanceTrackManager.setCurrentNodesCount(this.windowNodeCounter.count);
break;
case MType.LoadFontFace:
if (msg.source.startsWith('url(/')) {
const relativeUrl = msg.source.substring(4);
const lastUrl = this.locationManager.findLast(msg.time)?.url
if (lastUrl) {
const u = new URL(lastUrl);
const base = u.protocol + '//' + u.hostname + '/';
msg.source = `url(${base}${relativeUrl}`;
}
}
break;
}
this.performanceTrackManager.addNodeCountPointIfNeed(msg.time);
isDOMType(msg.tp) && this.pagesManager.appendMessage(msg);
@ -340,15 +354,18 @@ export default class TabSessionManager {
}
this.canvasReplayWalker.moveApply(t, (canvasMsg) => {
if (canvasMsg) {
this.canvasManagers[`${canvasMsg.timestamp}_${canvasMsg.nodeId}`].manager.startVideo();
this.canvasManagers[`${canvasMsg.timestamp}_${canvasMsg.nodeId}`].running = true;
const managerId = `${canvasMsg.timestamp}_${canvasMsg.nodeId}`
const possibleManager = this.canvasManagers[managerId]
if (possibleManager && !possibleManager.running) {
this.canvasManagers[managerId].manager.startVideo();
this.canvasManagers[managerId].running = true;
}
}
})
const runningManagers = Object.keys(this.canvasManagers).filter(
(key) => this.canvasManagers[key].running
);
runningManagers.forEach((key) => {
const manager = this.canvasManagers[key].manager;
const runningManagers = Object.values(this.canvasManagers).filter(
(manager) => manager.running
)
runningManagers.forEach(({ manager }) => {
manager.move(t);
});
});

96
frontend/app/player/web/managers/CanvasManager.ts
View file

@ -1,8 +1,8 @@
import { TarFile } from 'js-untar';
import ListWalker from 'Player/common/ListWalker';
import { VElement } from 'Player/web/managers/DOM/VirtualDOM';
import unpack from 'Player/common/unpack';
import unpackTar from 'Player/common/tarball';
import unpack from 'Player/common/unpack';
import { VElement } from 'Player/web/managers/DOM/VirtualDOM';
import { TarFile } from 'js-untar';
const playMode = {
video: 'video',
@ -21,6 +21,7 @@ export default class CanvasManager extends ListWalker<Timestamp> {
private lastTs = 0;
private playMode: string = playMode.snaps;
private snapshots: Record<number, TarFile> = {};
private debugCanvas: HTMLCanvasElement | undefined;
constructor(
/**
@ -32,7 +33,8 @@ export default class CanvasManager extends ListWalker<Timestamp> {
*/
private readonly delta: number,
private readonly links: [tar?: string, mp4?: string],
private readonly getNode: (id: number) => VElement | undefined
private readonly getNode: (id: number) => VElement | undefined,
private readonly sessionStart: number,
) {
super();
// first we try to grab tar, then fallback to mp4
@ -52,9 +54,34 @@ export default class CanvasManager extends ListWalker<Timestamp> {
}
});
} else {
return console.error('Failed to load canvas recording for node', this.nodeId);
return console.error(
'Failed to load canvas recording for node',
this.nodeId
);
}
});
// @ts-ignore
if (window.__or_debug === true) {
let debugContainer = document.querySelector<HTMLDivElement>('.imgDebug');
if (!debugContainer) {
debugContainer = document.createElement('div');
debugContainer.className = 'imgDebug';
Object.assign(debugContainer.style, {
position: 'fixed',
top: '0',
left: 0,
display: 'flex',
flexDirection: 'column',
});
document.body.appendChild(debugContainer);
}
const debugCanvas = document.createElement('canvas');
debugCanvas.width = 300;
debugCanvas.height = 200;
this.debugCanvas = debugCanvas;
debugContainer.appendChild(debugCanvas);
}
}
public mapToSnapshots(files: TarFile[]) {
@ -65,13 +92,14 @@ export default class CanvasManager extends ListWalker<Timestamp> {
console.error('Invalid file name format', files[0].name);
return;
}
const sessionStart = firstPair ? parseInt(firstPair[1], 10) : 0;
files.forEach((file) => {
const [_, _1, _2, imageTimestampStr] = file.name.match(filenameRegexp) ?? [0, 0, 0, '0'];
const [_, _1, _2, imageTimestampStr] = file.name.match(
filenameRegexp
) ?? [0, 0, 0, '0'];
const imageTimestamp = parseInt(imageTimestampStr, 10);
const messageTime = imageTimestamp - sessionStart;
const messageTime = imageTimestamp - this.sessionStart;
this.snapshots[messageTime] = file;
tempArr.push({ time: messageTime });
});
@ -127,19 +155,30 @@ export default class CanvasManager extends ListWalker<Timestamp> {
if (node && node.node) {
const canvasCtx = (node.node as HTMLCanvasElement).getContext('2d');
const canvasEl = node.node as HTMLVideoElement;
canvasCtx?.drawImage(this.snapImage, 0, 0, canvasEl.width, canvasEl.height);
canvasCtx?.drawImage(
this.snapImage,
0,
0,
canvasEl.width,
canvasEl.height
);
this.debugCanvas
?.getContext('2d')
?.drawImage(this.snapImage, 0, 0, 300, 200);
} else {
console.error(`CanvasManager: Node ${this.nodeId} not found`);
}
};
} else {
if (!this.fileData) return;
this.videoTag.setAttribute('autoplay', 'true');
this.videoTag.setAttribute('muted', 'true');
this.videoTag.setAttribute('playsinline', 'true');
this.videoTag.setAttribute('crossorigin', 'anonymous');
this.videoTag.src = this.fileData;
this.videoTag.currentTime = 0;
}
if (!this.fileData) return;
this.videoTag.setAttribute('autoplay', 'true');
this.videoTag.setAttribute('muted', 'true');
this.videoTag.setAttribute('playsinline', 'true');
this.videoTag.setAttribute('crossorigin', 'anonymous');
this.videoTag.src = this.fileData;
this.videoTag.currentTime = 0;
};
move(t: number) {
@ -163,15 +202,21 @@ export default class CanvasManager extends ListWalker<Timestamp> {
void this.videoTag.pause();
}
this.videoTag.currentTime = playTime / 1000;
canvasCtx?.drawImage(this.videoTag, 0, 0, canvasEl.width, canvasEl.height);
canvasCtx?.drawImage(
this.videoTag,
0,
0,
canvasEl.width,
canvasEl.height
);
} else {
console.error(`CanvasManager: Node ${this.nodeId} not found`);
console.error(`VideoMode CanvasManager: Node ${this.nodeId} not found`);
}
}
};
moveReadySnap = (t: number) => {
const msg = this.moveGetLast(t);
const msg = this.getNew(t);
if (msg) {
const file = this.snapshots[msg.time];
if (file) {
@ -180,3 +225,14 @@ export default class CanvasManager extends ListWalker<Timestamp> {
}
};
}
function saveImageData(imageDataUrl: string, name: string) {
const link = document.createElement('a');
link.href = imageDataUrl;
link.download = name;
link.style.display = 'none';
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
}

19
frontend/app/player/web/managers/DOM/DOMManager.ts
View file

@ -1,4 +1,5 @@
import logger from 'App/logger';
import { resolveURL } from "../../messages/rewriter/urlResolve";
import type Screen from '../../Screen/Screen';
import type { Message, SetNodeScroll } from '../../messages';
@ -393,15 +394,17 @@ export default class DOMManager extends ListWalker<Message> {
vRoot.whenReady(vNode => {
if (vNode instanceof VShadowRoot) { logger.error(`Node ${vNode} expected to be a Document`, msg); return }
let descr: Object | undefined
try {
descr = JSON.parse(msg.descriptors)
descr = typeof descr === 'object' ? descr : undefined
} catch {
logger.warn("Can't parse font-face descriptors: ", msg)
if (msg.descriptors) {
try {
descr = JSON.parse(msg.descriptors)
descr = typeof descr === 'object' ? descr : undefined
} catch {
logger.warn("Can't parse font-face descriptors: ", msg)
}
}
const ff = new FontFace(msg.family, msg.source, descr)
vNode.node.fonts.add(ff)
ff.load() // TODOTODO: wait for this one in StylesManager in a common way with styles
void ff.load()
})
return
}
@ -410,11 +413,11 @@ export default class DOMManager extends ListWalker<Message> {
/**
* Moves and applies all the messages from the current (or from the beginning, if t < current.time)
* to the one with msg.time >= `t`
* to the one with msg[time] >= `t`
*
* This function autoresets pointer if necessary (better name?)
*
* @returns Promise that fulfulls when necessary changes get applied
* @returns Promise that fulfills when necessary changes get applied
* (the async part exists mostly due to styles loading)
*/
async moveReady(t: number): Promise<void> {

17
frontend/app/player/web/managers/DOM/safeCSSRules.ts
View file

@ -1,17 +1,30 @@
import logger from 'App/logger';
function isChromium(item) {
return ['Chromium', 'Google Chrome', 'NewBrowser'].includes(item.brand);
}
// @ts-ignore
const isChromeLike = navigator.userAgentData.brands.some(isChromium)
export function insertRule(
sheet: { insertRule: (rule: string, index?: number) => void },
msg: { rule: string, index: number }
) {
/**
* inserting -moz- styles in chrome-like browsers causes issues and warnings
* changing them to -webkit- is usually fine because they're covered by native styles
* and not inserting them will break sheet indexing
* */
if (msg.rule.includes('-moz-') && isChromeLike) {
msg.rule = msg.rule.replace(/-moz-/g, '-webkit-')
}
try {
sheet.insertRule(msg.rule, msg.index)
} catch (e) {
try {
sheet.insertRule(msg.rule)
} catch (e) {
logger.warn("Cannot insert rule.", e, msg)
logger.warn("Cannot insert rule.", e, '\nmessage: ', msg)
}
}
}

2
tracker/tracker/src/main/app/canvas.ts
View file

@ -107,7 +107,7 @@ class CanvasRecorder {
this.snapshots[id].images.push({ id: this.app.timestamp(), data: snapshot })
if (this.snapshots[id].images.length > 9) {
this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt)
this.snapshots[id].images = []
this.snapshots[id].images = []
}
}
}