From e207d37e69a64205e9b164e23f2cdf47ea42ee04 Mon Sep 17 00:00:00 2001 From: Delirium Date: Thu, 22 Feb 2024 12:50:19 +0100 Subject: [PATCH] Ios player v2 (#1901) * feat(ui): new ios player start * fix(ui): image player * fix(ui): fix autoplay * fix(ui): fix copy paste code; error handler; default mode null * fix(ui): fix loader animation * fix(ui): memory cleanup * fix(ui): memory cleanup --- .../Player/MobilePlayer/ReplayWindow.tsx | 218 +++++++++++------- frontend/app/player/common/common.d.ts | 11 + frontend/app/player/common/tarball.ts | 25 ++ frontend/app/player/common/types.ts | 1 + frontend/app/player/common/unpack.ts | 37 +-- .../app/player/mobile/IOSMessageManager.ts | 122 ++++++---- frontend/app/player/mobile/IOSPlayer.ts | 148 +++++++----- .../player/mobile/managers/SnapshotManager.ts | 40 ++++ frontend/app/player/web/MessageLoader.ts | 75 +++--- frontend/app/player/web/Screen/Screen.ts | 2 + frontend/app/player/web/network/loadFiles.ts | 16 +- frontend/package.json | 1 + frontend/yarn.lock | 8 + 13 files changed, 459 insertions(+), 245 deletions(-) create mode 100644 frontend/app/player/common/common.d.ts create mode 100644 frontend/app/player/common/tarball.ts create mode 100644 frontend/app/player/mobile/managers/SnapshotManager.ts diff --git a/frontend/app/components/Session/Player/MobilePlayer/ReplayWindow.tsx b/frontend/app/components/Session/Player/MobilePlayer/ReplayWindow.tsx index a3c1f85d2..0394a3d34 100644 --- a/frontend/app/components/Session/Player/MobilePlayer/ReplayWindow.tsx +++ b/frontend/app/components/Session/Player/MobilePlayer/ReplayWindow.tsx @@ -1,123 +1,177 @@ -import React from 'react' +import { PlayerMode } from 'Player'; +import React from 'react'; import { MobilePlayerContext, IOSPlayerContext } from 'App/components/Session/playerContext'; import { observer } from 'mobx-react-lite'; -import { mapIphoneModel } from "Player/mobile/utils"; +import { mapIphoneModel } from 'Player/mobile/utils'; interface Props { - videoURL: string; + videoURL: string[]; userDevice: string; } const appleIcon = ` -` +`; function ReplayWindow({ videoURL, userDevice }: Props) { const playerContext = React.useContext(MobilePlayerContext); const videoRef = React.useRef(); + const imageRef = React.useRef(); + const containerRef = React.useRef(); - const time = playerContext.store.get().time + const { time, currentSnapshot, mode } = playerContext.store.get(); React.useEffect(() => { - if (videoRef.current) { - const timeSecs = time / 1000 - const delta = videoRef.current.currentTime - timeSecs + if (videoRef.current && mode === PlayerMode.VIDEO) { + const timeSecs = time / 1000; + const delta = videoRef.current.currentTime - timeSecs; if (videoRef.current.duration >= timeSecs && Math.abs(delta) > 0.1) { - videoRef.current.currentTime = timeSecs + videoRef.current.currentTime = timeSecs; } } - }, [time]) + }, [time, mode]); + React.useEffect(() => { + if (currentSnapshot && mode === PlayerMode.SNAPS) { + const blob = currentSnapshot.getBlobUrl(); + if (imageRef.current) { + imageRef.current.src = blob; + } + } + return () => { + if (imageRef.current) { + URL.revokeObjectURL(imageRef.current.src) + } + } + }, [currentSnapshot, mode]); React.useEffect(() => { - if (playerContext.player.screen.document && videoURL) { - playerContext.player.pause() - const { svg, styles } = mapIphoneModel(userDevice) + playerContext.player.pause() + const { svg, styles } = mapIphoneModel(userDevice); + if (!containerRef.current && playerContext.player.screen.document) { + const host = document.createElement('div'); + const shell = document.createElement('div'); + const icon = document.createElement('div'); + const videoContainer = document.createElement('div'); - const host = document.createElement('div') - const videoEl = document.createElement('video') - const sourceEl = document.createElement('source') - const shell = document.createElement('div') - const icon = document.createElement('div') - const videoContainer = document.createElement('div') + videoContainer.style.borderRadius = '10px'; + videoContainer.style.overflow = 'hidden'; + videoContainer.style.margin = styles.margin; + videoContainer.style.display = 'none'; + videoContainer.style.width = styles.screen.width + 'px'; + videoContainer.style.height = styles.screen.height + 'px'; - videoContainer.style.borderRadius = '10px' - videoContainer.style.overflow = 'hidden' - videoContainer.style.margin = styles.margin - videoContainer.style.display = 'none' - videoContainer.style.width = styles.screen.width + 'px' - videoContainer.style.height = styles.screen.height + 'px' + shell.innerHTML = svg; + Object.assign(icon.style, mobileIconStyle(styles)); + const spacer = document.createElement('div'); + spacer.style.width = '60px'; + spacer.style.height = '60px'; - videoContainer.appendChild(videoEl) + const loadingBar = document.createElement('div'); - shell.innerHTML = svg + Object.assign(loadingBar.style, mobileLoadingBarStyle(styles)); + icon.innerHTML = appleIcon; - videoEl.width = styles.screen.width - videoEl.height = styles.screen.height - videoEl.style.backgroundColor = '#333' + shell.style.position = 'absolute'; + shell.style.top = '0'; - Object.assign(icon.style, { - backgroundColor: '#333', - borderRadius: '10px', - width: styles.screen.width + 'px', - height: styles.screen.height + 'px', - margin: styles.margin, - display: 'flex', - flexDirection: 'column', - justifyContent: 'center', - alignItems: 'center', - }) - const spacer = document.createElement('div') - spacer.style.width = '60px' - spacer.style.height = '60px' + host.appendChild(videoContainer); + host.appendChild(shell); - const loadingBar = document.createElement('div') - Object.assign(loadingBar.style, { - width: styles.screen.width/2 + 'px', - height: '6px', - borderRadius: '3px', - backgroundColor: 'white', - }) - icon.innerHTML = appleIcon - icon.appendChild(spacer) - icon.appendChild(loadingBar) + icon.appendChild(spacer); + icon.appendChild(loadingBar); + host.appendChild(icon); - shell.style.position = 'absolute' - shell.style.top = '0' + containerRef.current = host; + videoContainer.id = '___or_replay-video'; + icon.id = '___or_ios-icon'; + host.id = '___or_ios-player'; - sourceEl.setAttribute('src', videoURL) - sourceEl.setAttribute('type', 'video/mp4') - - host.appendChild(videoContainer) - host.appendChild(shell) - host.appendChild(icon) - videoEl.appendChild(sourceEl) - - videoEl.addEventListener("loadeddata", () => { - videoContainer.style.display = 'block' - icon.style.display = 'none' - host.removeChild(icon) - console.log('loaded') - playerContext.player.play() - }) - - videoRef.current = videoEl - playerContext.player.injectPlayer(host) - playerContext.player.customScale(styles.shell.width, styles.shell.height) + playerContext.player.injectPlayer(host); + playerContext.player.customScale(styles.shell.width, styles.shell.height); playerContext.player.updateDimensions({ width: styles.screen.width, height: styles.screen.height, - }) + }); playerContext.player.updateOverlayStyle({ margin: styles.margin, width: styles.screen.width + 'px', height: styles.screen.height + 'px', - }) + }); } - }, [videoURL, playerContext.player.screen.document]) - return ( -
- ) + }, [playerContext.player.screen.document]); + + React.useEffect(() => { + const { styles } = mapIphoneModel(userDevice); + if (mode) { + const host = containerRef.current; + const videoContainer = + playerContext.player.screen.document?.getElementById('___or_replay-video'); + const icon = playerContext.player.screen.document?.getElementById('___or_ios-icon'); + if (host && videoContainer && icon) { + if (mode === PlayerMode.SNAPS) { + const imagePlayer = document.createElement('img'); + imagePlayer.style.width = styles.screen.width + 'px'; + imagePlayer.style.height = styles.screen.height + 'px'; + imagePlayer.style.backgroundColor = '#333'; + + videoContainer.appendChild(imagePlayer); + const removeLoader = () => { + host.removeChild(icon); + videoContainer.style.display = 'block'; + imagePlayer.removeEventListener('load', removeLoader); + }; + imagePlayer.addEventListener('load', removeLoader); + imageRef.current = imagePlayer; + playerContext.player.play(); + } + if (mode === PlayerMode.VIDEO) { + const mp4URL = videoURL.find((url) => url.includes('.mp4')); + if (mp4URL) { + const videoEl = document.createElement('video'); + const sourceEl = document.createElement('source'); + + videoContainer.appendChild(videoEl); + + videoEl.width = styles.screen.width; + videoEl.height = styles.screen.height; + videoEl.style.backgroundColor = '#333'; + + sourceEl.setAttribute('src', mp4URL); + sourceEl.setAttribute('type', 'video/mp4'); + videoEl.appendChild(sourceEl); + + videoEl.addEventListener('loadeddata', () => { + host.removeChild(icon); + videoContainer.style.display = 'block'; + playerContext.player.play(); + }); + + videoRef.current = videoEl; + } + } + } + } + }, [videoURL, playerContext.player.screen.document, mode]); + return
; } -export default observer(ReplayWindow); \ No newline at end of file +const mobileLoadingBarStyle = (styles: any) => ({ + width: styles.screen.width / 2 + 'px', + height: '6px', + borderRadius: '3px', + backgroundColor: 'white', +}); +const mobileIconStyle = (styles: any) => ({ + backgroundColor: '#333', + borderRadius: '10px', + width: styles.screen.width + 'px', + height: styles.screen.height + 'px', + margin: styles.margin, + display: 'flex', + flexDirection: 'column', + justifyContent: 'center', + alignItems: 'center', +}); + +export default observer(ReplayWindow); diff --git a/frontend/app/player/common/common.d.ts b/frontend/app/player/common/common.d.ts new file mode 100644 index 000000000..c1f9eda43 --- /dev/null +++ b/frontend/app/player/common/common.d.ts @@ -0,0 +1,11 @@ + +declare module 'js-untar' { + export interface TarFile { + name: string + blob: Blob + buffer: ArrayBuffer + getBlobUrl: () => string + } + + export default function untar(tarFile: ArrayBuffer): Promise +} \ No newline at end of file diff --git a/frontend/app/player/common/tarball.ts b/frontend/app/player/common/tarball.ts new file mode 100644 index 000000000..ed226400c --- /dev/null +++ b/frontend/app/player/common/tarball.ts @@ -0,0 +1,25 @@ +import untar, { TarFile } from 'js-untar'; + +const unpackTar = (data: Uint8Array): Promise => { + const isTar = true + // tarball ustar starts from 257, 75 73 74 61 72, but this is getting lost here for some reason + // so we rely on try catch + // data[257] === 0x75 && + // data[258] === 0x73 && + // data[259] === 0x74 && + // data[260] === 0x61 && + // data[261] === 0x72 && + // data[262] === 0x00; + + if (isTar) { + const now = performance.now(); + return untar(data.buffer).then((files) => { + console.debug('Tar unpack time', Math.floor(performance.now() - now) + 'ms'); + return files; + }); + } else { + return Promise.reject('Not a tarball file'); + } +}; + +export default unpackTar; \ No newline at end of file diff --git a/frontend/app/player/common/types.ts b/frontend/app/player/common/types.ts index 060bb94ae..9d49d1b2a 100644 --- a/frontend/app/player/common/types.ts +++ b/frontend/app/player/common/types.ts @@ -39,6 +39,7 @@ export interface SessionFilesInfo { milliseconds: number valueOf: () => number } + videoURL: string[] domURL: string[] devtoolsURL: string[] /** deprecated */ diff --git a/frontend/app/player/common/unpack.ts b/frontend/app/player/common/unpack.ts index 52960daff..e1f2891c5 100644 --- a/frontend/app/player/common/unpack.ts +++ b/frontend/app/player/common/unpack.ts @@ -1,39 +1,40 @@ import * as fzstd from 'fzstd'; -import { gunzipSync } from 'fflate' +import { gunzipSync } from 'fflate'; const unpack = (b: Uint8Array): Uint8Array => { // zstd magical numbers 40 181 47 253 - const isZstd = b[0] === 0x28 && b[1] === 0xb5 && b[2] === 0x2f && b[3] === 0xfd - const isGzip = b[0] === 0x1F && b[1] === 0x8B && b[2] === 0x08; + const isZstd = b[0] === 0x28 && b[1] === 0xb5 && b[2] === 0x2f && b[3] === 0xfd; + const isGzip = b[0] === 0x1f && b[1] === 0x8b && b[2] === 0x08; + let data = b; if (isGzip) { - const now = performance.now() - const data = gunzipSync(b) + const now = performance.now(); + const uData = gunzipSync(b); console.debug( - "Gunzip time", + 'Gunzip time', Math.floor(performance.now() - now) + 'ms', 'size', Math.floor(b.byteLength / 1024), '->', - Math.floor(data.byteLength / 1024), + Math.floor(uData.byteLength / 1024), 'kb' - ) - return data + ); + data = uData; } if (isZstd) { - const now = performance.now() - const data = fzstd.decompress(b) + const now = performance.now(); + const uData = fzstd.decompress(b); console.debug( - "Zstd unpack time", + 'Zstd unpack time', Math.floor(performance.now() - now) + 'ms', 'size', Math.floor(b.byteLength / 1024), '->', - Math.floor(data.byteLength / 1024), + Math.floor(uData.byteLength / 1024), 'kb' - ) - return data + ); + data = uData; } - return b -} + return data; +}; -export default unpack \ No newline at end of file +export default unpack; diff --git a/frontend/app/player/mobile/IOSMessageManager.ts b/frontend/app/player/mobile/IOSMessageManager.ts index 38b896bf4..3276bc9d3 100644 --- a/frontend/app/player/mobile/IOSMessageManager.ts +++ b/frontend/app/player/mobile/IOSMessageManager.ts @@ -1,5 +1,6 @@ import logger from 'App/logger'; -import { getResourceFromNetworkRequest } from "Player"; +import { TarFile } from "js-untar"; +import { getResourceFromNetworkRequest } from 'Player'; import type { Store } from 'Player'; import { IMessageManager } from 'Player/player/Animator'; @@ -11,43 +12,52 @@ import Lists, { INITIAL_STATE as LISTS_INITIAL_STATE, State as ListsState, } from './IOSLists'; -import IOSPerformanceTrackManager, { PerformanceChartPoint } from "Player/mobile/managers/IOSPerformanceTrackManager"; +import IOSPerformanceTrackManager, { + PerformanceChartPoint, +} from 'Player/mobile/managers/IOSPerformanceTrackManager'; import { MType } from '../web/messages'; import type { Message } from '../web/messages'; +import SnapshotManager from 'Player/mobile/managers/SnapshotManager'; import Screen, { INITIAL_STATE as SCREEN_INITIAL_STATE, State as ScreenState, } from '../web/Screen/Screen'; -import { Log } from './types/log' +import { Log } from './types/log'; import type { SkipInterval } from '../web/managers/ActivityManager'; -export const performanceWarnings = ['thermalState', 'memoryWarning', 'lowDiskSpace', 'isLowPowerModeEnabled', 'batteryLevel'] +export const performanceWarnings = [ + 'thermalState', + 'memoryWarning', + 'lowDiskSpace', + 'isLowPowerModeEnabled', + 'batteryLevel', +]; const perfWarningFrustrations = { thermalState: { - title: "Overheating", - icon: "thermometer-sun", + title: 'Overheating', + icon: 'thermometer-sun', }, memoryWarning: { - title: "High Memory Usage", - icon: "memory-ios" + title: 'High Memory Usage', + icon: 'memory-ios', }, lowDiskSpace: { - title: "Low Disk Space", - icon: "low-disc-space" + title: 'Low Disk Space', + icon: 'low-disc-space', }, isLowPowerModeEnabled: { - title: "Low Power Mode", - icon: "battery-charging" + title: 'Low Power Mode', + icon: 'battery-charging', }, batteryLevel: { - title: "Low Battery", - icon: "battery" - } -} + title: 'Low Battery', + icon: 'battery', + }, +}; export interface State extends ScreenState, ListsState { skipIntervals: SkipInterval[]; @@ -64,9 +74,15 @@ export interface State extends ScreenState, ListsState { messagesProcessed: boolean; eventCount: number; updateWarnings: number; + currentSnapshot: TarFile | null; } -const userEvents = [MType.IosSwipeEvent, MType.IosClickEvent, MType.IosInputEvent, MType.IosScreenChanges]; +const userEvents = [ + MType.IosSwipeEvent, + MType.IosClickEvent, + MType.IosInputEvent, + MType.IosScreenChanges, +]; export default class IOSMessageManager implements IMessageManager { static INITIAL_STATE: State = { @@ -83,6 +99,7 @@ export default class IOSMessageManager implements IMessageManager { lastMessageTime: 0, messagesProcessed: false, messagesLoading: false, + currentSnapshot: null, }; private activityManager: ActivityManager | null = null; @@ -92,6 +109,7 @@ export default class IOSMessageManager implements IMessageManager { private lastMessageTime: number = 0; private touchManager: TouchManager; private lists: Lists; + public snapshotManager: SnapshotManager; constructor( private readonly session: Record, @@ -104,6 +122,7 @@ export default class IOSMessageManager implements IMessageManager { this.lists = new Lists(initialLists); this.touchManager = new TouchManager(screen); this.activityManager = new ActivityManager(this.session.duration.milliseconds); // only if not-live + this.snapshotManager = new SnapshotManager(); } public updateDimensions(dimensions: { width: number; height: number }) { @@ -111,16 +130,16 @@ export default class IOSMessageManager implements IMessageManager { } public updateLists(lists: Partial) { - const exceptions = lists.exceptions - exceptions?.forEach(e => { + const exceptions = lists.exceptions; + exceptions?.forEach((e) => { this.lists.lists.exceptions.insert(e); - this.lists.lists.log.insert(e) - }) - lists.frustrations?.forEach(f => { + this.lists.lists.log.insert(e); + }); + lists.frustrations?.forEach((f) => { this.lists.lists.frustrations.insert(f); - }) + }); - const eventCount = this.lists.lists.event.count //lists?.event?.length || 0; + const eventCount = this.lists.lists.event.count; //lists?.event?.length || 0; const currentState = this.state.get(); this.state.update({ eventCount: currentState.eventCount + eventCount, @@ -134,8 +153,8 @@ export default class IOSMessageManager implements IMessageManager { } public getListsFullState = () => { - return this.lists.getFullListsState(); - } + return this.lists.getFullListsState(); + }; private waitingForFiles: boolean = false; public onFileReadSuccess = () => { @@ -144,29 +163,29 @@ export default class IOSMessageManager implements IMessageManager { eventCount: this.lists?.lists.event?.length || 0, performanceChartData: this.performanceManager.chartData, ...this.lists.getFullListsState(), - } + }; if (this.activityManager) { this.activityManager.end(); - newState['skipIntervals'] = this.activityManager.list + newState['skipIntervals'] = this.activityManager.list; } this.state.update(newState); }; public onFileReadFailed = (...e: any[]) => { logger.error(e); - this.state.update({error: true}); + this.state.update({ error: true }); this.uiErrorHandler?.error('Error requesting a session file'); }; public onFileReadFinally = () => { this.waitingForFiles = false; - this.state.update({messagesProcessed: true}); + this.state.update({ messagesProcessed: true }); }; public startLoading = () => { this.waitingForFiles = true; - this.state.update({messagesProcessed: false}); + this.state.update({ messagesProcessed: false }); this.setMessagesLoading(true); }; @@ -182,7 +201,7 @@ export default class IOSMessageManager implements IMessageManager { if (lastPerformanceTrackMessage) { Object.assign(stateToUpdate, { performanceChartTime: lastPerformanceTrackMessage.time, - }) + }); } this.touchManager.move(t); @@ -194,61 +213,67 @@ export default class IOSMessageManager implements IMessageManager { this.setMessagesLoading(true); } - Object.assign(stateToUpdate, this.lists.moveGetState(t)) - Object.assign(stateToUpdate, { performanceListNow: this.lists.lists.performance.listNow }) + const snapshot = this.snapshotManager.moveReady(t); + if (snapshot) { + Object.assign(stateToUpdate, { + currentSnapshot: snapshot, + }); + } + Object.assign(stateToUpdate, this.lists.moveGetState(t)); + Object.assign(stateToUpdate, { performanceListNow: this.lists.lists.performance.listNow }); Object.keys(stateToUpdate).length > 0 && this.state.update(stateToUpdate); } distributeMessage = (msg: Message & { tabId: string }): void => { const lastMessageTime = Math.max(msg.time, this.lastMessageTime); this.lastMessageTime = lastMessageTime; - this.state.update({lastMessageTime}); + this.state.update({ lastMessageTime }); if (userEvents.includes(msg.tp)) { this.activityManager?.updateAcctivity(msg.time); } switch (msg.tp) { case MType.IosPerformanceEvent: - const performanceStats = ['background', 'memoryUsage', 'mainThreadCPU'] + const performanceStats = ['background', 'memoryUsage', 'mainThreadCPU']; if (performanceStats.includes(msg.name)) { this.performanceManager.append(msg); } if (performanceWarnings.includes(msg.name)) { // @ts-ignore - const item = perfWarningFrustrations[msg.name] + const item = perfWarningFrustrations[msg.name]; this.lists.lists.performance.append({ ...msg, name: item.title, techName: msg.name, icon: item.icon, - type: 'ios_perf_event' - } as any) + type: 'ios_perf_event', + } as any); } break; // case MType.IosInputEvent: // console.log('input', msg) // break; case MType.IosNetworkCall: - this.lists.lists.fetch.insert(getResourceFromNetworkRequest(msg, this.sessionStart)) + this.lists.lists.fetch.insert(getResourceFromNetworkRequest(msg, this.sessionStart)); break; case MType.WsChannel: - this.lists.lists.websocket.insert(msg) + this.lists.lists.websocket.insert(msg); break; case MType.IosEvent: // @ts-ignore - this.lists.lists.event.insert({...msg, source: 'openreplay'}); + this.lists.lists.event.insert({ ...msg, source: 'openreplay' }); break; case MType.IosSwipeEvent: case MType.IosClickEvent: this.touchManager.append(msg); break; case MType.IosLog: - const log = {...msg, level: msg.severity} + const log = { ...msg, level: msg.severity }; // @ts-ignore this.lists.lists.log.append(Log(log)); break; default: - console.log(msg) + console.log(msg); // stuff break; } @@ -257,16 +282,17 @@ export default class IOSMessageManager implements IMessageManager { setMessagesLoading = (messagesLoading: boolean) => { this.screen.display(!messagesLoading); // @ts-ignore idk - this.state.update({messagesLoading, ready: !messagesLoading && !this.state.get().cssLoading}); + this.state.update({ messagesLoading, ready: !messagesLoading && !this.state.get().cssLoading }); }; - private setSize({height, width}: { height: number; width: number }) { - this.screen.scale({height, width}); - this.state.update({width, height}); + private setSize({ height, width }: { height: number; width: number }) { + this.screen.scale({ height, width }); + this.state.update({ width, height }); } // TODO: clean managers? clean() { + this.snapshotManager?.clean(); this.state.update(IOSMessageManager.INITIAL_STATE); } } diff --git a/frontend/app/player/mobile/IOSPlayer.ts b/frontend/app/player/mobile/IOSPlayer.ts index bb975302e..839230f3f 100644 --- a/frontend/app/player/mobile/IOSPlayer.ts +++ b/frontend/app/player/mobile/IOSPlayer.ts @@ -1,10 +1,15 @@ -import { Log, LogLevel, SessionFilesInfo } from 'Player' +import { Log, LogLevel, SessionFilesInfo } from 'Player'; -import type { Store } from 'Player' -import MessageLoader from "Player/web/MessageLoader"; -import Player from '../player/Player' -import Screen, { ScaleMode } from '../web/Screen/Screen' -import IOSMessageManager from "Player/mobile/IOSMessageManager"; +import type { Store } from 'Player'; +import MessageLoader from 'Player/web/MessageLoader'; +import Player from '../player/Player'; +import Screen, { ScaleMode } from '../web/Screen/Screen'; +import IOSMessageManager from 'Player/mobile/IOSMessageManager'; + +export const PlayerMode = { + VIDEO: 'video', + SNAPS: 'snaps', +}; export default class IOSPlayer extends Player { static readonly INITIAL_STATE = { @@ -12,120 +17,137 @@ export default class IOSPlayer extends Player { ...MessageLoader.INITIAL_STATE, ...IOSMessageManager.INITIAL_STATE, scale: 1, - } - public screen: Screen - protected messageManager: IOSMessageManager - protected readonly messageLoader: MessageLoader + mode: null, + autoplay: false, + }; + public screen: Screen; + protected messageManager: IOSMessageManager; + protected readonly messageLoader: MessageLoader; + constructor( protected wpState: Store, session: SessionFilesInfo, public readonly uiErrorHandler?: { error: (msg: string) => void } ) { - const screen = new Screen(true, ScaleMode.Embed) - const messageManager = new IOSMessageManager(session, wpState, screen, uiErrorHandler) + const hasTar = session.videoURL.some((url) => url.includes('.tar.')); + const screen = new Screen(true, ScaleMode.Embed); + const messageManager = new IOSMessageManager(session, wpState, screen, uiErrorHandler); const messageLoader = new MessageLoader( session, wpState, messageManager, false, uiErrorHandler - ) + ); super(wpState, messageManager); - this.screen = screen - this.messageManager = messageManager - this.messageLoader = messageLoader + this.pause() + this.screen = screen; + this.messageManager = messageManager; + this.messageLoader = messageLoader; - void messageLoader.loadFiles() - const endTime = session.duration?.valueOf() || 0 + if (hasTar) { + messageLoader + .loadTarball(session.videoURL.find((url) => url.includes('.tar.'))!) + .then((files) => { + if (files) { + this.wpState.update({ mode: PlayerMode.SNAPS }); + this.messageManager.snapshotManager.mapToSnapshots(files); + } + }) + .catch((e) => { + this.wpState.update({ mode: PlayerMode.VIDEO }); + }); + } + void messageLoader.loadFiles(); + const endTime = session.duration?.valueOf() || 0; wpState.update({ session, endTime, - }) + }); } attach = (parent: HTMLElement) => { - this.screen.attach(parent) - } + this.screen.attach(parent); + }; public updateDimensions(dimensions: { width: number; height: number }) { - return this.messageManager.updateDimensions(dimensions) + return this.messageManager.updateDimensions(dimensions); } public updateLists(session: any) { - const exceptions = session.crashes.concat(session.errors || []) + const exceptions = session.crashes.concat(session.errors || []); const lists = { - event: session.events.map((e: Record) => { - if (e.name === 'Click') e.name = 'Touch' - return e - }) || [], + event: + session.events.map((e: Record) => { + if (e.name === 'Click') e.name = 'Touch'; + return e; + }) || [], frustrations: session.frustrations || [], stack: session.stackEvents || [], - exceptions: exceptions.map(({ name, ...rest }: any) => - Log({ - level: LogLevel.ERROR, - value: name, - name, - message: rest.reason, - errorId: rest.crashId || rest.errorId, - ...rest, - }) - ) || [], - } + exceptions: + exceptions.map(({ name, ...rest }: any) => + Log({ + level: LogLevel.ERROR, + value: name, + name, + message: rest.reason, + errorId: rest.crashId || rest.errorId, + ...rest, + }) + ) || [], + }; - return this.messageManager.updateLists(lists) + return this.messageManager.updateLists(lists); } public updateOverlayStyle(style: Partial) { - this.screen.updateOverlayStyle(style) + this.screen.updateOverlayStyle(style); } injectPlayer = (player: HTMLElement) => { - this.screen.addToBody(player) - this.screen.addMobileStyles() + this.screen.addToBody(player); + this.screen.addMobileStyles(); window.addEventListener('resize', () => this.customScale(this.customConstrains.width, this.customConstrains.height) - ) - } + ); + }; scale = () => { // const { width, height } = this.wpState.get() if (!this.screen) return; - console.debug("using customConstrains to scale player") + console.debug('using customConstrains to scale player'); // sometimes happens in live assist sessions for some reason - this.screen?.scale?.(this.customConstrains) - } + this.screen?.scale?.(this.customConstrains); + }; customConstrains = { width: 0, height: 0, - } + }; customScale = (width: number, height: number) => { if (!this.screen) return; - this.screen?.scale?.({ width, height }) - this.customConstrains = { width, height } - this.wpState.update({ scale: this.screen.getScale() }) - } + this.screen?.scale?.({ width, height }); + this.customConstrains = { width, height }; + this.wpState.update({ scale: this.screen.getScale() }); + }; addFullscreenBoundary = (isFullscreen?: boolean) => { if (isFullscreen) { - this.screen?.addFullscreenBoundary() + this.screen?.addFullscreenBoundary(); } else { - this.screen?.addMobileStyles() + this.screen?.addMobileStyles(); } - } - + }; clean = () => { - super.clean() - this.screen.clean() + super.clean(); + this.screen.clean(); // @ts-ignore this.screen = undefined; - this.messageLoader.clean() + this.messageLoader.clean(); // @ts-ignore this.messageManager = undefined; - window.removeEventListener('resize', this.scale) - } - - + window.removeEventListener('resize', this.scale); + }; } diff --git a/frontend/app/player/mobile/managers/SnapshotManager.ts b/frontend/app/player/mobile/managers/SnapshotManager.ts new file mode 100644 index 000000000..7c8f63a09 --- /dev/null +++ b/frontend/app/player/mobile/managers/SnapshotManager.ts @@ -0,0 +1,40 @@ +import { TarFile } from "js-untar"; +import ListWalker from 'Player/common/ListWalker'; + +interface Snapshots { + [timestamp: number]: TarFile +} + +type Timestamp = { time: number } + + +export default class SnapshotManager extends ListWalker { + private snapshots: Snapshots = {} + + public mapToSnapshots(files: TarFile[]) { + const filenameRegexp = /(\d+)_1_(\d+)\.jpeg$/; + const firstPair = files[0].name.match(filenameRegexp) + const sessionStart = firstPair ? parseInt(firstPair[1], 10) : 0 + files.forEach(file => { + const [_, _2, imageTimestamp] = file + .name + .match(filenameRegexp) + ?.map(n => parseInt(n, 10)) ?? [0, 0, 0] + const messageTime = imageTimestamp - sessionStart + this.snapshots[messageTime] = file + this.append({ time: messageTime }) + }) + } + + public moveReady(t: number) { + const msg = this.moveGetLast(t) + if (msg) { + return this.snapshots[msg.time] + } + } + + public clean() { + this.snapshots = {} + this.reset() + } +} \ No newline at end of file diff --git a/frontend/app/player/web/MessageLoader.ts b/frontend/app/player/web/MessageLoader.ts index a89a20091..01585b6e4 100644 --- a/frontend/app/player/web/MessageLoader.ts +++ b/frontend/app/player/web/MessageLoader.ts @@ -1,11 +1,12 @@ -import type { Store, SessionFilesInfo, PlayerMsg } from "Player"; -import { decryptSessionBytes } from './network/crypto'; +import type { Store, SessionFilesInfo, PlayerMsg } from 'Player'; +import { decryptSessionBytes } from './network/crypto'; import MFileReader from './messages/MFileReader'; -import { loadFiles, requestEFSDom, requestEFSDevtools } from './network/loadFiles'; +import { loadFiles, requestEFSDom, requestEFSDevtools, requestTarball } from './network/loadFiles'; import logger from 'App/logger'; import unpack from 'Player/common/unpack'; import MessageManager from 'Player/web/MessageManager'; import IOSMessageManager from 'Player/mobile/IOSMessageManager'; +import unpackTar from 'Player/common/tarball'; interface State { firstFileLoading: boolean; @@ -79,6 +80,18 @@ export default class MessageLoader { this.messageManager.setMessagesLoading(false); }; + async loadTarball(url: string) { + try { + const tarBufferZstd = await requestTarball(url); + if (tarBufferZstd) { + const tar = unpack(tarBufferZstd); + return await unpackTar(tar); + } + } catch (e) { + throw e + } + } + async loadDomFiles(urls: string[], parser: (b: Uint8Array) => Promise) { if (urls.length > 0) { this.store.update({ domLoading: true }); @@ -116,10 +129,10 @@ export default class MessageLoader { this.messageManager.startLoading(); try { - await this.loadMobs() + await this.loadMobs(); } catch (sessionLoadError) { try { - await this.loadEFSMobs() + await this.loadEFSMobs(); } catch (unprocessedLoadError) { this.messageManager.onFileReadFailed(sessionLoadError, unprocessedLoadError); } @@ -129,35 +142,35 @@ export default class MessageLoader { } } - loadMobs = async () => { - const loadMethod = - this.session.domURL && this.session.domURL.length > 0 - ? { - mobUrls: this.session.domURL, - parser: () => this.createNewParser(true, this.processMessages, 'dom'), - } - : { - mobUrls: this.session.mobsUrl, - parser: () => this.createNewParser(false, this.processMessages, 'dom'), - }; + loadMobs = async () => { + const loadMethod = + this.session.domURL && this.session.domURL.length > 0 + ? { + mobUrls: this.session.domURL, + parser: () => this.createNewParser(true, this.processMessages, 'dom'), + } + : { + mobUrls: this.session.mobsUrl, + parser: () => this.createNewParser(false, this.processMessages, 'dom'), + }; - const parser = loadMethod.parser(); - const devtoolsParser = this.createNewParser(true, this.processMessages, 'devtools'); + const parser = loadMethod.parser(); + const devtoolsParser = this.createNewParser(true, this.processMessages, 'devtools'); - /** - * to speed up time to replay - * we load first dom mob file before the rest - * (because parser can read them in parallel) - * as a tradeoff we have some copy-paste code - * for the devtools file - * */ + /** + * to speed up time to replay + * we load first dom mob file before the rest + * (because parser can read them in parallel) + * as a tradeoff we have some copy-paste code + * for the devtools file + * */ await loadFiles([loadMethod.mobUrls[0]], parser); const restDomFilesPromise = this.loadDomFiles([...loadMethod.mobUrls.slice(1)], parser); const restDevtoolsFilesPromise = this.loadDevtools(devtoolsParser); await Promise.allSettled([restDomFilesPromise, restDevtoolsFilesPromise]); this.messageManager.onFileReadSuccess(); - } + }; loadEFSMobs = async () => { this.store.update({ domLoading: true, devtoolsLoading: true }); @@ -172,16 +185,16 @@ export default class MessageLoader { const devtoolsParser = this.createNewParser(false, this.processMessages, 'devtoolsEFS'); const parseDomPromise: Promise = domData.status === 'fulfilled' - ? domParser(domData.value) - : Promise.reject('No dom file in EFS'); + ? domParser(domData.value) + : Promise.reject('No dom file in EFS'); const parseDevtoolsPromise: Promise = devtoolsData.status === 'fulfilled' - ? devtoolsParser(devtoolsData.value) - : Promise.reject('No devtools file in EFS'); + ? devtoolsParser(devtoolsData.value) + : Promise.reject('No devtools file in EFS'); await Promise.all([parseDomPromise, parseDevtoolsPromise]); this.messageManager.onFileReadSuccess(); - } + }; clean() { this.store.update(MessageLoader.INITIAL_STATE); diff --git a/frontend/app/player/web/Screen/Screen.ts b/frontend/app/player/web/Screen/Screen.ts index 1c7299ffa..0ca1f1479 100644 --- a/frontend/app/player/web/Screen/Screen.ts +++ b/frontend/app/player/web/Screen/Screen.ts @@ -114,6 +114,8 @@ export default class Screen { if (this.document) { this.document.body.style.margin = '0'; this.document.body.appendChild(el); + } else { + console.error('Attempt to add to player screen without document'); } } diff --git a/frontend/app/player/web/network/loadFiles.ts b/frontend/app/player/web/network/loadFiles.ts index 084f54780..29dcc7f8d 100644 --- a/frontend/app/player/web/network/loadFiles.ts +++ b/frontend/app/player/web/network/loadFiles.ts @@ -48,6 +48,16 @@ export async function requestEFSDevtools(sessionId: string) { return await requestEFSMobFile(sessionId + "/devtools.mob") } +export async function requestTarball(url: string) { + const res = await window.fetch(url) + if (res.ok) { + const buf = await res.arrayBuffer() + return new Uint8Array(buf) + } else { + throw new Error(res.status.toString()) + } +} + async function requestEFSMobFile(filename: string) { const api = new APIClient() const res = await api.fetch('/unprocessed/' + filename) @@ -58,13 +68,13 @@ async function requestEFSMobFile(filename: string) { } const processAPIStreamResponse = (response: Response, skippable: boolean) => { - return new Promise((res, rej) => { + return new Promise((res, rej) => { if (response.status === 404 && skippable) { return rej(ALLOWED_404) } if (response.status >= 400) { return rej(`Bad file status code ${response.status}. Url: ${response.url}`) } - res(response.blob()) - }).then(async blob => new Uint8Array(await blob.arrayBuffer())) + res(response.arrayBuffer()) + }).then(async buf => new Uint8Array(buf)) } diff --git a/frontend/package.json b/frontend/package.json index 9ca4e1423..0e9bba33e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -43,6 +43,7 @@ "html2canvas": "^1.4.1", "immutable": "^4.0.0-rc.12", "jest-environment-jsdom": "^29.5.0", + "js-untar": "^2.0.0", "jsbi": "^4.1.0", "jshint": "^2.11.1", "jspdf": "^2.5.1", diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 6923417f2..4dddbae0b 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -15261,6 +15261,13 @@ __metadata: languageName: node linkType: hard +"js-untar@npm:^2.0.0": + version: 2.0.0 + resolution: "js-untar@npm:2.0.0" + checksum: d84138561b2ed12870ba07ff62c55dfb0f9884cd1eb32044ba6272eacbed4ec476e08cc62226f5824d8bda3de3597a506582ff9bdf0845f221d3e092f9d8b025 + languageName: node + linkType: hard + "js-yaml@npm:^3.13.1": version: 3.14.1 resolution: "js-yaml@npm:3.14.1" @@ -18152,6 +18159,7 @@ __metadata: immutable: ^4.0.0-rc.12 jest: ^29.5.0 jest-environment-jsdom: ^29.5.0 + js-untar: ^2.0.0 jsbi: ^4.1.0 jshint: ^2.11.1 jspdf: ^2.5.1