jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(frontend-player): sequential (pre)load for multifile sessions

Browse source
This commit is contained in:
Alex Kaminskii 2022-05-06 00:10:08 +02:00
parent 2d96705930
commit a99f684b83
4 changed files with 203 additions and 163 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

251
frontend/app/player/MessageDistributor/MessageDistributor.ts
View file

@ -8,11 +8,11 @@ import Profile from 'Types/session/profile';
import ReduxAction from 'Types/session/reduxAction';
import { update } from '../store';
import {
import {
init as initListsDepr,
append as listAppend,
setStartTime as setListsStartTime
} from '../lists';
setStartTime as setListsStartTime
} from '../lists';
import StatedScreen from './StatedScreen/StatedScreen';
@ -26,6 +26,7 @@ import ActivityManager from './managers/ActivityManager';
import AssistManager from './managers/AssistManager';
import MFileReader from './messages/MFileReader';
import loadFiles from './network/loadFiles';
import { INITIAL_STATE as SUPER_INITIAL_STATE, State as SuperState } from './StatedScreen/StatedScreen';
import { INITIAL_STATE as ASSIST_INITIAL_STATE, State as AssistState } from './managers/AssistManager';
@ -33,7 +34,7 @@ import { INITIAL_STATE as ASSIST_INITIAL_STATE, State as AssistState } from './m
import type { PerformanceChartPoint } from './managers/PerformanceTrackManager';
import type { SkipInterval } from './managers/ActivityManager';
const LIST_NAMES = [ "redux", "mobx", "vuex", "ngrx", "graphql", "exceptions", "profiles", "longtasks" ] as const;
const LIST_NAMES = ["redux", "mobx", "vuex", "ngrx", "graphql", "exceptions", "profiles", "longtasks"] as const;
const LISTS_INITIAL_STATE = {};
LIST_NAMES.forEach(name => {
LISTS_INITIAL_STATE[`${name}ListNow`] = [];
@ -65,15 +66,15 @@ type ListsObject = {
}
function initLists(): ListsObject {
const lists: Partial<ListsObject> = {} ;
const lists: Partial<ListsObject> = {};
for (var i = 0; i < LIST_NAMES.length; i++) {
lists[ LIST_NAMES[i] ] = new ListWalker();
lists[LIST_NAMES[i]] = new ListWalker();
}
return lists as ListsObject;
}
import type {
import type {
Message,
SetPageLocation,
ConnectionInformation,
@ -110,7 +111,7 @@ export default class MessageDistributor extends StatedScreen {
private navigationStartOffset: number = 0;
private lastMessageTime: number = 0;
constructor(private readonly session: any /*Session*/, jwt: string, config, live: boolean) {
constructor(private readonly session: any /*Session*/, jwt: string, config, live: boolean) {
super();
this.pagesManager = new PagesManager(this, this.session.isMobile)
this.mouseManager = new MouseManager(this);
@ -128,7 +129,7 @@ export default class MessageDistributor extends StatedScreen {
/* == REFACTOR_ME == */
const eventList = this.session.events.toJSON();
initListsDepr({
event: eventList,
event: eventList,
stack: this.session.stackEvents.toJSON(),
resource: this.session.resources.toJSON(),
});
@ -146,96 +147,83 @@ export default class MessageDistributor extends StatedScreen {
}
}
// subscribeOnMessages(sockUrl) {
// this.setMessagesLoading(true);
// const socket = new WebSocket(sockUrl);
// socket.binaryType = 'arraybuffer';
// socket.onerror = (e) => {
// // TODO: reconnect
// update({ error: true });
// }
// socket.onmessage = (socketMessage) => {
// const data = new Uint8Array(socketMessage.data);
// const msgs = [];
// messageGenerator // parseBuffer(msgs, data);
// // TODO: count indexes. Now will not work due to wrong indexes
// //msgs.forEach(this.distributeMessage);
// this.setMessagesLoading(false);
// this.setDisconnected(false);
// }
// this._socket = socket;
// }
private waitingForFiles: boolean = false
private loadMessages(): void {
const fileUrl: string = this.session.mobsUrl;
this.setMessagesLoading(true);
window.fetch(fileUrl)
.then(r => r.arrayBuffer())
.then(b => {
const r = new MFileReader(new Uint8Array(b), this.sessionStart);
const msgs: Array<Message> = [];
this.setMessagesLoading(true)
this.waitingForFiles = true
while (r.hasNext()) {
const next = r.next();
if (next != null) {
this.distributeMessage(next[0], next[1]);
msgs.push(next[0]);
const r = new MFileReader(new Uint8Array(), this.sessionStart)
const msgs: Array<Message> = []
loadFiles(this.session.mobsUrl,
b => {
r.append(b)
let next: ReturnType<MFileReader['next']>
while (next = r.next()) {
const [msg, index] = next
this.distributeMessage(msg, index)
this.lastMessageTime = Math.max(msg.time, this.lastMessageTime)
msgs.push(msg)
}
}
// @ts-ignore Hack for upet (TODO: fix ordering in one mutation (removes first))
const headChildrenIds = msgs.filter(m => m.parentID === 1).map(m => m.id);
//const createNodeTypes = ["create_text_node", "create_element_node"];
this.pagesManager.sort((m1, m2) =>{
if (m1.time === m2.time) {
if (m1.tp === "remove_node" && m2.tp !== "remove_node") {
if (headChildrenIds.includes(m1.id)) {
return -1;
}
} else if (m2.tp === "remove_node" && m1.tp !== "remove_node") {
if (headChildrenIds.includes(m2.id)) {
return 1;
}
} else if (m2.tp === "remove_node" && m1.tp === "remove_node") {
const m1FromHead = headChildrenIds.includes(m1.id);
const m2FromHead = headChildrenIds.includes(m2.id);
if (m1FromHead && !m2FromHead) {
return -1;
} else if (m2FromHead && !m1FromHead) {
return 1;
logger.info("Messages count: ", msgs.length, msgs)
// @ts-ignore Hack for upet (TODO: fix ordering in one mutation in tracker(removes first))
const headChildrenIds = msgs.filter(m => m.parentID === 1).map(m => m.id);
this.pagesManager.sort((m1, m2) => {
if (m1.time === m2.time) {
if (m1.tp === "remove_node" && m2.tp !== "remove_node") {
if (headChildrenIds.includes(m1.id)) {
return -1;
}
} else if (m2.tp === "remove_node" && m1.tp !== "remove_node") {
if (headChildrenIds.includes(m2.id)) {
return 1;
}
} else if (m2.tp === "remove_node" && m1.tp === "remove_node") {
const m1FromHead = headChildrenIds.includes(m1.id);
const m2FromHead = headChildrenIds.includes(m2.id);
if (m1FromHead && !m2FromHead) {
return -1;
} else if (m2FromHead && !m1FromHead) {
return 1;
}
}
}
}
return 0;
})
return 0;
})
logger.info("Messages count: ", msgs.length, msgs);
const stateToUpdate: {[key:string]: any} = {
performanceChartData: this.performanceTrackManager.chartData,
performanceAvaliability: this.performanceTrackManager.avaliability,
};
this.activirtManager?.end();
stateToUpdate.skipIntervals = this.activirtManager?.list || [];
LIST_NAMES.forEach(key => {
stateToUpdate[ `${ key }List` ] = this.lists[ key ].list;
});
update(stateToUpdate);
this.windowNodeCounter.reset();
this.setMessagesLoading(false);
const stateToUpdate: {[key:string]: any} = {
performanceChartData: this.performanceTrackManager.chartData,
performanceAvaliability: this.performanceTrackManager.avaliability,
}
LIST_NAMES.forEach(key => {
stateToUpdate[ `${ key }List` ] = this.lists[ key ].list
})
update(stateToUpdate)
this.setMessagesLoading(false)
}
)
.then(() => {
this.windowNodeCounter.reset()
if (this.activirtManager) {
this.activirtManager.end()
update({
skipIntervals: this.activirtManager.list
})
}
this.waitingForFiles = false
this.setMessagesLoading(false)
})
.catch(e => {
logger.error(e)
this.waitingForFiles = false
this.setMessagesLoading(false)
update({ error: true })
})
.catch((e) => {
logger.error(e);
this.setMessagesLoading(false);
update({ error: true });
});
}
move(t: number, index?: number):void {
move(t: number, index?: number): void {
const stateToUpdate: Partial<State> = {};
/* == REFACTOR_ME == */
const lastLoadedLocationMsg = this.loadedLocationManager.moveToLast(t, index);
@ -248,7 +236,7 @@ export default class MessageDistributor extends StatedScreen {
if (llEvent.domContentLoadedTime != null) {
stateToUpdate.domContentLoadedTime = {
time: llEvent.domContentLoadedTime + this.navigationStartOffset, //TODO: predefined list of load event for the network tab (merge events & SetPageLocation: add navigationStart to db)
value: llEvent.domContentLoadedTime,
value: llEvent.domContentLoadedTime,
}
}
if (llEvent.loadTime != null) {
@ -277,9 +265,9 @@ export default class MessageDistributor extends StatedScreen {
}
LIST_NAMES.forEach(key => {
const lastMsg = this.lists[ key ].moveToLast(t, key === 'exceptions' ? undefined : index);
const lastMsg = this.lists[key].moveToLast(t, key === 'exceptions' ? undefined : index);
if (lastMsg != null) {
stateToUpdate[`${key}ListNow`] = this.lists[ key ].listNow;
stateToUpdate[`${key}ListNow`] = this.lists[key].listNow;
}
});
@ -298,21 +286,25 @@ export default class MessageDistributor extends StatedScreen {
this.window.scrollTo(lastScroll.x, lastScroll.y);
}
// Moving mouse and setting :hover classes on ready view
this.mouseManager.move(t);
this.mouseManager.move(t);
const lastClick = this.clickManager.moveToLast(t);
if (!!lastClick && t - lastClick.time < 600) { // happend during last 600ms
this.cursor.click();
}
// After all changes - redraw the marker
//this.marker.redraw();
})
})
if (this.waitingForFiles && this.lastMessageTime <= t) {
this.setMessagesLoading(true)
}
}
_decodeMessage(msg, keys: Array<string>) {
const decoded = {};
try {
keys.forEach(key => {
decoded[ key ] = this.decoder.decode(msg[ key ]);
decoded[key] = this.decoder.decode(msg[key]);
});
} catch (e) {
logger.error("Error on message decoding: ", e, msg);
@ -323,15 +315,13 @@ export default class MessageDistributor extends StatedScreen {
/* Binded */
distributeMessage = (msg: Message, index: number): void => {
this.lastMessageTime = msg.time;
if ([
if ([
"mouse_move",
"mouse_click",
"create_element_node", // not a user activity, though visual change
"set_input_value",
"set_input_checked",
"set_viewport_size",
"set_viewport_size",
"set_viewport_scroll",
].includes(msg.tp)) {
this.activirtManager?.updateAcctivity(msg.time);
@ -343,13 +333,13 @@ export default class MessageDistributor extends StatedScreen {
/* Lists: */
case "console_log":
if (msg.level === 'debug') break;
listAppend("log", Log({
listAppend("log", Log({
level: msg.level,
value: msg.value,
time,
time,
index,
}));
break;
break;
case "fetch":
listAppend("fetch", Resource({
method: msg.method,
@ -362,118 +352,117 @@ export default class MessageDistributor extends StatedScreen {
time: msg.timestamp - this.sessionStart, //~
index,
}));
break;
break;
/* */
case "set_page_location":
this.locationManager.add(msg);
if (msg.navigationStart > 0) {
this.loadedLocationManager.add(msg);
}
break;
break;
case "set_viewport_size":
this.resizeManager.add(msg);
break;
break;
case "mouse_move":
this.mouseManager.add(msg);
break;
break;
case "mouse_click":
this.clickManager.add(msg);
break;
break;
case "set_viewport_scroll":
this.scrollManager.add(msg);
break;
break;
case "performance_track":
this.performanceTrackManager.add(msg);
break;
break;
case "set_page_visibility":
this.performanceTrackManager.handleVisibility(msg)
break;
break;
case "connection_information":
this.connectionInfoManger.add(msg);
break;
break;
case "o_table":
this.decoder.set(msg.key, msg.value);
break;
break;
case "redux":
decoded = this._decodeMessage(msg, ["state", "action"]);
logger.log(decoded)
if (decoded != null) {
this.lists.redux.add(decoded);
}
break;
break;
case "ng_rx":
decoded = this._decodeMessage(msg, ["state", "action"]);
logger.log(decoded)
if (decoded != null) {
this.lists.ngrx.add(decoded);
}
break;
}
break;
case "vuex":
decoded = this._decodeMessage(msg, ["state", "mutation"]);
logger.log(decoded)
if (decoded != null) {
this.lists.vuex.add(decoded);
}
break;
}
break;
case "mob_x":
decoded = this._decodeMessage(msg, ["payload"]);
logger.log(decoded)
if (decoded != null) {
this.lists.mobx.add(decoded);
}
break;
}
break;
case "graph_ql":
// @ts-ignore some hack? TODO: remove
msg.duration = 0;
this.lists.graphql.add(msg);
break;
break;
case "profiler":
this.lists.profiles.add(msg);
break;
break;
case "long_task":
this.lists.longtasks.add({
...msg,
time: msg.timestamp - this.sessionStart,
});
break;
break;
default:
switch (msg.tp){
switch (msg.tp) {
case "create_document":
this.windowNodeCounter.reset();
this.performanceTrackManager.setCurrentNodesCount(this.windowNodeCounter.count);
break;
break;
case "create_text_node":
case "create_element_node":
this.windowNodeCounter.addNode(msg.id, msg.parentID);
this.performanceTrackManager.setCurrentNodesCount(this.windowNodeCounter.count);
break;
break;
case "move_node":
this.windowNodeCounter.moveNode(msg.id, msg.parentID);
this.performanceTrackManager.setCurrentNodesCount(this.windowNodeCounter.count);
break;
break;
case "remove_node":
this.windowNodeCounter.removeNode(msg.id);
this.performanceTrackManager.setCurrentNodesCount(this.windowNodeCounter.count);
break;
break;
}
this.pagesManager.add(msg);
break;
break;
}
}
getLastMessageTime():number {
getLastMessageTime(): number {
return this.lastMessageTime;
}
getFirstMessageTime():number {
getFirstMessageTime(): number {
return 0; //this.pagesManager.minTime;
}
// TODO: clean managers?
clean() {
super.clean();
//if (this._socket) this._socket.close();
update(INITIAL_STATE);
this.assistManager.clear();
}

65
frontend/app/player/MessageDistributor/messages/MFileReader.ts
View file

@ -7,62 +7,65 @@ import RawMessageReader from './RawMessageReader';
// needSkipMessage() and next() methods here use buf and p protected properties,
// which should be probably somehow incapsulated
export default class MFileReader extends RawMessageReader {
private pLastMessageID: number = 0;
private currentTime: number = 0;
public error: boolean = false;
private pLastMessageID: number = 0
private currentTime: number = 0
public error: boolean = false
constructor(data: Uint8Array, private readonly startTime: number) {
super(data);
super(data)
}
private needSkipMessage(): boolean {
if (this.p === 0) return false;
if (this.p === 0) return false
for (let i = 7; i >= 0; i--) {
if (this.buf[ this.p + i ] !== this.buf[ this.pLastMessageID + i ]) {
return this.buf[ this.p + i ] - this.buf[ this.pLastMessageID + i ] < 0;
return this.buf[ this.p + i ] - this.buf[ this.pLastMessageID + i ] < 0
}
}
return true;
return true
}
private readRawMessage(): RawMessage | null {
this.skip(8);
this.skip(8)
try {
return super.readMessage();
const msg = super.readMessage()
if (!msg) {
this.skip(-8)
}
return msg
} catch (e) {
this.error = true;
logger.error("Read message error:", e);
return null;
this.error = true
logger.error("Read message error:", e)
return null
}
}
hasNext():boolean {
return !this.error && this.hasNextByte();
}
next(): [ Message, number] | null {
if (!this.hasNext()) {
return null;
if (this.error || !this.hasNextByte()) {
return null
}
while (this.needSkipMessage()) {
this.readRawMessage();
if (!this.readRawMessage()) {
return null
}
}
this.pLastMessageID = this.p;
const rMsg = this.readRawMessage();
this.pLastMessageID = this.p
const rMsg = this.readRawMessage()
if (!rMsg) {
return null;
return null
}
if (rMsg.tp === "timestamp") {
this.currentTime = rMsg.timestamp - this.startTime;
} else {
const msg = Object.assign(rMsg, {
time: this.currentTime,
_index: this.pLastMessageID,
})
return [msg, this.pLastMessageID];
}
return null;
this.currentTime = rMsg.timestamp - this.startTime
return this.next()
}
const msg = Object.assign(rMsg, {
time: this.currentTime,
_index: this.pLastMessageID,
})
return [msg, this.pLastMessageID]
}
}

1
frontend/app/player/MessageDistributor/messages/urlResolve.ts
View file

@ -6,7 +6,6 @@ export function resolveURL(baseURL: string, relURL: string): string {
}
var match = /bar/.exec("foobar");
const re1 = /url\(("[^"]*"|'[^']*'|[^)]*)\)/g
const re2 = /@import "(.*?)"/g
function cssUrlsIndex(css: string): Array<[number, number]> {

49
frontend/app/player/MessageDistributor/network/loadFiles.ts Normal file
View file

@ -0,0 +1,49 @@
const NO_NTH_FILE = "nnf"
export default function load(
urls: string[],
onData: (Uint8Array) => void,
): Promise<void> {
const firstFileURL = urls.shift()
if (!firstFileURL) {
return Promise.reject("No urls provided")
}
return window.fetch(firstFileURL)
.then(r => {
if (r.status >= 400) {
throw new Error(`no start file. status code ${ r.status }`)
}
return r.arrayBuffer()
})
.then(b => new Uint8Array(b))
.then(onData)
.then(() =>
urls.reduce((p, url) =>
p.then(() =>
window.fetch(url)
.then(r => {
return new Promise<ArrayBuffer>((res, rej) => {
if (r.status == 404) {
rej(NO_NTH_FILE)
return
}
if (r.status >= 400) {
rej(`Bad endfile status code ${r.status}`)
return
}
res(r.arrayBuffer())
})
})
.then(b => new Uint8Array(b))
.then(onData)
),
Promise.resolve(),
)
)
.catch(e => {
if (e === NO_NTH_FILE) {
return
}
throw e
})
}