From a99f684b83b704967e1d34b1ab1f5a09b736264d Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Fri, 6 May 2022 00:10:08 +0200 Subject: [PATCH] feat(frontend-player): sequential (pre)load for multifile sessions --- .../MessageDistributor/MessageDistributor.ts | 251 +++++++++--------- .../messages/MFileReader.ts | 65 ++--- .../MessageDistributor/messages/urlResolve.ts | 1 - .../MessageDistributor/network/loadFiles.ts | 49 ++++ 4 files changed, 203 insertions(+), 163 deletions(-) create mode 100644 frontend/app/player/MessageDistributor/network/loadFiles.ts diff --git a/frontend/app/player/MessageDistributor/MessageDistributor.ts b/frontend/app/player/MessageDistributor/MessageDistributor.ts index 68ae14c72..653b05f54 100644 --- a/frontend/app/player/MessageDistributor/MessageDistributor.ts +++ b/frontend/app/player/MessageDistributor/MessageDistributor.ts @@ -8,11 +8,11 @@ import Profile from 'Types/session/profile'; import ReduxAction from 'Types/session/reduxAction'; import { update } from '../store'; -import { +import { init as initListsDepr, append as listAppend, - setStartTime as setListsStartTime - } from '../lists'; + setStartTime as setListsStartTime +} from '../lists'; import StatedScreen from './StatedScreen/StatedScreen'; @@ -26,6 +26,7 @@ import ActivityManager from './managers/ActivityManager'; import AssistManager from './managers/AssistManager'; import MFileReader from './messages/MFileReader'; +import loadFiles from './network/loadFiles'; import { INITIAL_STATE as SUPER_INITIAL_STATE, State as SuperState } from './StatedScreen/StatedScreen'; import { INITIAL_STATE as ASSIST_INITIAL_STATE, State as AssistState } from './managers/AssistManager'; @@ -33,7 +34,7 @@ import { INITIAL_STATE as ASSIST_INITIAL_STATE, State as AssistState } from './m import type { PerformanceChartPoint } from './managers/PerformanceTrackManager'; import type { SkipInterval } from './managers/ActivityManager'; -const LIST_NAMES = [ "redux", "mobx", "vuex", "ngrx", "graphql", "exceptions", "profiles", "longtasks" ] as const; +const LIST_NAMES = ["redux", "mobx", "vuex", "ngrx", "graphql", "exceptions", "profiles", "longtasks"] as const; const LISTS_INITIAL_STATE = {}; LIST_NAMES.forEach(name => { LISTS_INITIAL_STATE[`${name}ListNow`] = []; @@ -65,15 +66,15 @@ type ListsObject = { } function initLists(): ListsObject { - const lists: Partial = {} ; + const lists: Partial = {}; for (var i = 0; i < LIST_NAMES.length; i++) { - lists[ LIST_NAMES[i] ] = new ListWalker(); + lists[LIST_NAMES[i]] = new ListWalker(); } return lists as ListsObject; } -import type { +import type { Message, SetPageLocation, ConnectionInformation, @@ -110,7 +111,7 @@ export default class MessageDistributor extends StatedScreen { private navigationStartOffset: number = 0; private lastMessageTime: number = 0; - constructor(private readonly session: any /*Session*/, jwt: string, config, live: boolean) { + constructor(private readonly session: any /*Session*/, jwt: string, config, live: boolean) { super(); this.pagesManager = new PagesManager(this, this.session.isMobile) this.mouseManager = new MouseManager(this); @@ -128,7 +129,7 @@ export default class MessageDistributor extends StatedScreen { /* == REFACTOR_ME == */ const eventList = this.session.events.toJSON(); initListsDepr({ - event: eventList, + event: eventList, stack: this.session.stackEvents.toJSON(), resource: this.session.resources.toJSON(), }); @@ -146,96 +147,83 @@ export default class MessageDistributor extends StatedScreen { } } - - // subscribeOnMessages(sockUrl) { - // this.setMessagesLoading(true); - // const socket = new WebSocket(sockUrl); - // socket.binaryType = 'arraybuffer'; - // socket.onerror = (e) => { - // // TODO: reconnect - // update({ error: true }); - // } - // socket.onmessage = (socketMessage) => { - // const data = new Uint8Array(socketMessage.data); - // const msgs = []; - // messageGenerator // parseBuffer(msgs, data); - // // TODO: count indexes. Now will not work due to wrong indexes - // //msgs.forEach(this.distributeMessage); - // this.setMessagesLoading(false); - // this.setDisconnected(false); - // } - // this._socket = socket; - // } - + private waitingForFiles: boolean = false private loadMessages(): void { - const fileUrl: string = this.session.mobsUrl; - this.setMessagesLoading(true); - window.fetch(fileUrl) - .then(r => r.arrayBuffer()) - .then(b => { - const r = new MFileReader(new Uint8Array(b), this.sessionStart); - const msgs: Array = []; + this.setMessagesLoading(true) + this.waitingForFiles = true - while (r.hasNext()) { - const next = r.next(); - if (next != null) { - this.distributeMessage(next[0], next[1]); - msgs.push(next[0]); + const r = new MFileReader(new Uint8Array(), this.sessionStart) + const msgs: Array = [] + loadFiles(this.session.mobsUrl, + b => { + r.append(b) + let next: ReturnType + while (next = r.next()) { + const [msg, index] = next + this.distributeMessage(msg, index) + this.lastMessageTime = Math.max(msg.time, this.lastMessageTime) + + msgs.push(msg) } - } - // @ts-ignore Hack for upet (TODO: fix ordering in one mutation (removes first)) - const headChildrenIds = msgs.filter(m => m.parentID === 1).map(m => m.id); - //const createNodeTypes = ["create_text_node", "create_element_node"]; - this.pagesManager.sort((m1, m2) =>{ - if (m1.time === m2.time) { - if (m1.tp === "remove_node" && m2.tp !== "remove_node") { - if (headChildrenIds.includes(m1.id)) { - return -1; - } - } else if (m2.tp === "remove_node" && m1.tp !== "remove_node") { - if (headChildrenIds.includes(m2.id)) { - return 1; - } - } else if (m2.tp === "remove_node" && m1.tp === "remove_node") { - const m1FromHead = headChildrenIds.includes(m1.id); - const m2FromHead = headChildrenIds.includes(m2.id); - if (m1FromHead && !m2FromHead) { - return -1; - } else if (m2FromHead && !m1FromHead) { - return 1; + logger.info("Messages count: ", msgs.length, msgs) + + // @ts-ignore Hack for upet (TODO: fix ordering in one mutation in tracker(removes first)) + const headChildrenIds = msgs.filter(m => m.parentID === 1).map(m => m.id); + this.pagesManager.sort((m1, m2) => { + if (m1.time === m2.time) { + if (m1.tp === "remove_node" && m2.tp !== "remove_node") { + if (headChildrenIds.includes(m1.id)) { + return -1; + } + } else if (m2.tp === "remove_node" && m1.tp !== "remove_node") { + if (headChildrenIds.includes(m2.id)) { + return 1; + } + } else if (m2.tp === "remove_node" && m1.tp === "remove_node") { + const m1FromHead = headChildrenIds.includes(m1.id); + const m2FromHead = headChildrenIds.includes(m2.id); + if (m1FromHead && !m2FromHead) { + return -1; + } else if (m2FromHead && !m1FromHead) { + return 1; + } } } - } - return 0; - }) - + return 0; + }) - logger.info("Messages count: ", msgs.length, msgs); - - const stateToUpdate: {[key:string]: any} = { - performanceChartData: this.performanceTrackManager.chartData, - performanceAvaliability: this.performanceTrackManager.avaliability, - }; - this.activirtManager?.end(); - stateToUpdate.skipIntervals = this.activirtManager?.list || []; - LIST_NAMES.forEach(key => { - stateToUpdate[ `${ key }List` ] = this.lists[ key ].list; - }); - update(stateToUpdate); - - this.windowNodeCounter.reset(); - - this.setMessagesLoading(false); + const stateToUpdate: {[key:string]: any} = { + performanceChartData: this.performanceTrackManager.chartData, + performanceAvaliability: this.performanceTrackManager.avaliability, + } + LIST_NAMES.forEach(key => { + stateToUpdate[ `${ key }List` ] = this.lists[ key ].list + }) + update(stateToUpdate) + this.setMessagesLoading(false) + } + ) + .then(() => { + this.windowNodeCounter.reset() + if (this.activirtManager) { + this.activirtManager.end() + update({ + skipIntervals: this.activirtManager.list + }) + } + this.waitingForFiles = false + this.setMessagesLoading(false) + }) + .catch(e => { + logger.error(e) + this.waitingForFiles = false + this.setMessagesLoading(false) + update({ error: true }) }) - .catch((e) => { - logger.error(e); - this.setMessagesLoading(false); - update({ error: true }); - }); } - move(t: number, index?: number):void { + move(t: number, index?: number): void { const stateToUpdate: Partial = {}; /* == REFACTOR_ME == */ const lastLoadedLocationMsg = this.loadedLocationManager.moveToLast(t, index); @@ -248,7 +236,7 @@ export default class MessageDistributor extends StatedScreen { if (llEvent.domContentLoadedTime != null) { stateToUpdate.domContentLoadedTime = { time: llEvent.domContentLoadedTime + this.navigationStartOffset, //TODO: predefined list of load event for the network tab (merge events & SetPageLocation: add navigationStart to db) - value: llEvent.domContentLoadedTime, + value: llEvent.domContentLoadedTime, } } if (llEvent.loadTime != null) { @@ -277,9 +265,9 @@ export default class MessageDistributor extends StatedScreen { } LIST_NAMES.forEach(key => { - const lastMsg = this.lists[ key ].moveToLast(t, key === 'exceptions' ? undefined : index); + const lastMsg = this.lists[key].moveToLast(t, key === 'exceptions' ? undefined : index); if (lastMsg != null) { - stateToUpdate[`${key}ListNow`] = this.lists[ key ].listNow; + stateToUpdate[`${key}ListNow`] = this.lists[key].listNow; } }); @@ -298,21 +286,25 @@ export default class MessageDistributor extends StatedScreen { this.window.scrollTo(lastScroll.x, lastScroll.y); } // Moving mouse and setting :hover classes on ready view - this.mouseManager.move(t); + this.mouseManager.move(t); const lastClick = this.clickManager.moveToLast(t); if (!!lastClick && t - lastClick.time < 600) { // happend during last 600ms this.cursor.click(); } // After all changes - redraw the marker //this.marker.redraw(); - }) + }) + + if (this.waitingForFiles && this.lastMessageTime <= t) { + this.setMessagesLoading(true) + } } _decodeMessage(msg, keys: Array) { const decoded = {}; try { keys.forEach(key => { - decoded[ key ] = this.decoder.decode(msg[ key ]); + decoded[key] = this.decoder.decode(msg[key]); }); } catch (e) { logger.error("Error on message decoding: ", e, msg); @@ -323,15 +315,13 @@ export default class MessageDistributor extends StatedScreen { /* Binded */ distributeMessage = (msg: Message, index: number): void => { - this.lastMessageTime = msg.time; - - if ([ + if ([ "mouse_move", "mouse_click", "create_element_node", // not a user activity, though visual change "set_input_value", "set_input_checked", - "set_viewport_size", + "set_viewport_size", "set_viewport_scroll", ].includes(msg.tp)) { this.activirtManager?.updateAcctivity(msg.time); @@ -343,13 +333,13 @@ export default class MessageDistributor extends StatedScreen { /* Lists: */ case "console_log": if (msg.level === 'debug') break; - listAppend("log", Log({ + listAppend("log", Log({ level: msg.level, value: msg.value, - time, + time, index, })); - break; + break; case "fetch": listAppend("fetch", Resource({ method: msg.method, @@ -362,118 +352,117 @@ export default class MessageDistributor extends StatedScreen { time: msg.timestamp - this.sessionStart, //~ index, })); - break; + break; /* */ case "set_page_location": this.locationManager.add(msg); if (msg.navigationStart > 0) { this.loadedLocationManager.add(msg); } - break; + break; case "set_viewport_size": this.resizeManager.add(msg); - break; + break; case "mouse_move": this.mouseManager.add(msg); - break; + break; case "mouse_click": this.clickManager.add(msg); - break; + break; case "set_viewport_scroll": this.scrollManager.add(msg); - break; + break; case "performance_track": this.performanceTrackManager.add(msg); - break; + break; case "set_page_visibility": this.performanceTrackManager.handleVisibility(msg) - break; + break; case "connection_information": this.connectionInfoManger.add(msg); - break; + break; case "o_table": this.decoder.set(msg.key, msg.value); - break; + break; case "redux": decoded = this._decodeMessage(msg, ["state", "action"]); logger.log(decoded) if (decoded != null) { this.lists.redux.add(decoded); } - break; + break; case "ng_rx": decoded = this._decodeMessage(msg, ["state", "action"]); logger.log(decoded) if (decoded != null) { this.lists.ngrx.add(decoded); - } - break; + } + break; case "vuex": decoded = this._decodeMessage(msg, ["state", "mutation"]); logger.log(decoded) if (decoded != null) { this.lists.vuex.add(decoded); - } - break; + } + break; case "mob_x": decoded = this._decodeMessage(msg, ["payload"]); logger.log(decoded) if (decoded != null) { this.lists.mobx.add(decoded); - } - break; + } + break; case "graph_ql": // @ts-ignore some hack? TODO: remove msg.duration = 0; this.lists.graphql.add(msg); - break; + break; case "profiler": this.lists.profiles.add(msg); - break; + break; case "long_task": this.lists.longtasks.add({ ...msg, time: msg.timestamp - this.sessionStart, }); - break; + break; default: - switch (msg.tp){ + switch (msg.tp) { case "create_document": this.windowNodeCounter.reset(); this.performanceTrackManager.setCurrentNodesCount(this.windowNodeCounter.count); - break; + break; case "create_text_node": case "create_element_node": this.windowNodeCounter.addNode(msg.id, msg.parentID); this.performanceTrackManager.setCurrentNodesCount(this.windowNodeCounter.count); - break; + break; case "move_node": this.windowNodeCounter.moveNode(msg.id, msg.parentID); this.performanceTrackManager.setCurrentNodesCount(this.windowNodeCounter.count); - break; + break; case "remove_node": this.windowNodeCounter.removeNode(msg.id); this.performanceTrackManager.setCurrentNodesCount(this.windowNodeCounter.count); - break; + break; } this.pagesManager.add(msg); - break; + break; } } - getLastMessageTime():number { + getLastMessageTime(): number { return this.lastMessageTime; } - getFirstMessageTime():number { + getFirstMessageTime(): number { return 0; //this.pagesManager.minTime; } // TODO: clean managers? clean() { super.clean(); - //if (this._socket) this._socket.close(); update(INITIAL_STATE); this.assistManager.clear(); } diff --git a/frontend/app/player/MessageDistributor/messages/MFileReader.ts b/frontend/app/player/MessageDistributor/messages/MFileReader.ts index 0204259e5..a4df3e2f8 100644 --- a/frontend/app/player/MessageDistributor/messages/MFileReader.ts +++ b/frontend/app/player/MessageDistributor/messages/MFileReader.ts @@ -7,62 +7,65 @@ import RawMessageReader from './RawMessageReader'; // needSkipMessage() and next() methods here use buf and p protected properties, // which should be probably somehow incapsulated export default class MFileReader extends RawMessageReader { - private pLastMessageID: number = 0; - private currentTime: number = 0; - public error: boolean = false; + private pLastMessageID: number = 0 + private currentTime: number = 0 + public error: boolean = false constructor(data: Uint8Array, private readonly startTime: number) { - super(data); + super(data) } private needSkipMessage(): boolean { - if (this.p === 0) return false; + if (this.p === 0) return false for (let i = 7; i >= 0; i--) { if (this.buf[ this.p + i ] !== this.buf[ this.pLastMessageID + i ]) { - return this.buf[ this.p + i ] - this.buf[ this.pLastMessageID + i ] < 0; + return this.buf[ this.p + i ] - this.buf[ this.pLastMessageID + i ] < 0 } } - return true; + return true } private readRawMessage(): RawMessage | null { - this.skip(8); + this.skip(8) try { - return super.readMessage(); + const msg = super.readMessage() + if (!msg) { + this.skip(-8) + } + return msg } catch (e) { - this.error = true; - logger.error("Read message error:", e); - return null; + this.error = true + logger.error("Read message error:", e) + return null } } - hasNext():boolean { - return !this.error && this.hasNextByte(); - } - next(): [ Message, number] | null { - if (!this.hasNext()) { - return null; + if (this.error || !this.hasNextByte()) { + return null } while (this.needSkipMessage()) { - this.readRawMessage(); + if (!this.readRawMessage()) { + return null + } } - this.pLastMessageID = this.p; - const rMsg = this.readRawMessage(); + this.pLastMessageID = this.p + + const rMsg = this.readRawMessage() if (!rMsg) { - return null; + return null } if (rMsg.tp === "timestamp") { - this.currentTime = rMsg.timestamp - this.startTime; - } else { - const msg = Object.assign(rMsg, { - time: this.currentTime, - _index: this.pLastMessageID, - }) - return [msg, this.pLastMessageID]; - } - return null; + this.currentTime = rMsg.timestamp - this.startTime + return this.next() + } + + const msg = Object.assign(rMsg, { + time: this.currentTime, + _index: this.pLastMessageID, + }) + return [msg, this.pLastMessageID] } } \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/messages/urlResolve.ts b/frontend/app/player/MessageDistributor/messages/urlResolve.ts index b80ff4f9a..44298ec08 100644 --- a/frontend/app/player/MessageDistributor/messages/urlResolve.ts +++ b/frontend/app/player/MessageDistributor/messages/urlResolve.ts @@ -6,7 +6,6 @@ export function resolveURL(baseURL: string, relURL: string): string { } -var match = /bar/.exec("foobar"); const re1 = /url\(("[^"]*"|'[^']*'|[^)]*)\)/g const re2 = /@import "(.*?)"/g function cssUrlsIndex(css: string): Array<[number, number]> { diff --git a/frontend/app/player/MessageDistributor/network/loadFiles.ts b/frontend/app/player/MessageDistributor/network/loadFiles.ts new file mode 100644 index 000000000..5deecf472 --- /dev/null +++ b/frontend/app/player/MessageDistributor/network/loadFiles.ts @@ -0,0 +1,49 @@ +const NO_NTH_FILE = "nnf" + +export default function load( + urls: string[], + onData: (Uint8Array) => void, +): Promise { + const firstFileURL = urls.shift() + if (!firstFileURL) { + return Promise.reject("No urls provided") + } + return window.fetch(firstFileURL) + .then(r => { + if (r.status >= 400) { + throw new Error(`no start file. status code ${ r.status }`) + } + return r.arrayBuffer() + }) + .then(b => new Uint8Array(b)) + .then(onData) + .then(() => + urls.reduce((p, url) => + p.then(() => + window.fetch(url) + .then(r => { + return new Promise((res, rej) => { + if (r.status == 404) { + rej(NO_NTH_FILE) + return + } + if (r.status >= 400) { + rej(`Bad endfile status code ${r.status}`) + return + } + res(r.arrayBuffer()) + }) + }) + .then(b => new Uint8Array(b)) + .then(onData) + ), + Promise.resolve(), + ) + ) + .catch(e => { + if (e === NO_NTH_FILE) { + return + } + throw e + }) +}