mob-file reader-analyser (manual)

frontend/app/components/Session/Player/LivePlayer/Overlay/LiveOverlay.tsx

@@ -24,19 +24,17 @@ function Overlay({
   const { store } = React.useContext<ILivePlayerContext>(PlayerContext)
 
   const {
-    messagesLoading,
+    ready,
-    cssLoading,
     peerConnectionStatus,
     livePlay,
     calling,
     remoteControl,
     recordingState,
   } = store.get()
-  const loading = messagesLoading || cssLoading
   const liveStatusText = getStatusText(peerConnectionStatus)
   const connectionStatus = peerConnectionStatus
 
-  const showLiveStatusText = livePlay && liveStatusText && !loading;
+  const showLiveStatusText = livePlay && liveStatusText && ready;
 
   const showRequestWindow =
     (calling === CallingState.Connecting ||
@@ -66,7 +64,7 @@ function Overlay({
           connectionStatus={closedLive ? ConnectionStatus.Closed : connectionStatus}
         />
       )}
-      {loading ? <Loader /> : null}
+      {!ready ? <Loader /> : null}
     </>
   );
 }

frontend/app/components/Session_/Player/Controls/Controls.tsx

@@ -66,8 +66,7 @@ function Controls(props: any) {
     completed,
     skip,
     speed,
-    cssLoading,
+    ready,
-    messagesLoading,
     inspectorMode,
     markedTargets,
     exceptionsList,
@@ -88,7 +87,7 @@ function Controls(props: any) {
   } = props;
 
   const storageType = selectStorageType(store.get());
-  const disabled = disabledRedux || cssLoading || messagesLoading || inspectorMode || markedTargets;
+  const disabled = !ready || disabledRedux || inspectorMode || markedTargets;
   const profilesCount = profilesList.length;
   const graphqlCount = graphqlList.length;
   const showGraphql = graphqlCount > 0;
@@ -326,7 +325,6 @@ export default connect(
 //     nextProps.showFetch !== props.showFetch ||
 //     nextProps.fetchCount !== props.fetchCount ||
 //     nextProps.graphqlCount !== props.graphqlCount ||
-//     nextProps.liveTimeTravel !== props.liveTimeTravel ||
 //     nextProps.skipInterval !== props.skipInterval
 //   )
 //     return true;

frontend/app/components/Session_/Player/Overlay.tsx

@@ -21,23 +21,21 @@ function Overlay({
   const togglePlay = () => player.togglePlay()
   const {
     playing,
-    messagesLoading,
+    ready,
-    cssLoading,
     completed,
     autoplay,
     inspectorMode,
     markedTargets,
     activeTargetIndex,
   } = store.get()
-  const loading = messagesLoading || cssLoading
 
   const showAutoplayTimer = completed && autoplay && nextId
-  const showPlayIconLayer = !isClickmap && !markedTargets && !inspectorMode && !loading && !showAutoplayTimer;
+  const showPlayIconLayer = !isClickmap && !markedTargets && !inspectorMode && ready && !showAutoplayTimer;
 
   return (
     <>
       {showAutoplayTimer && <AutoplayTimer />}
-      {loading ? <Loader /> : null}
+      {!ready ? <Loader /> : null}
       {showPlayIconLayer && <PlayIconLayer playing={playing} togglePlay={togglePlay} />}
       {markedTargets && <ElementsMarker targets={markedTargets} activeIndex={activeTargetIndex} />}
     </>

frontend/app/player/common/StoreSubscriber.ts

@@ -0,0 +1,23 @@
+import { Store } from './types'
+
+
+export default class StoreSubscriber<G, S=G> implements Store<G, S> {
+  constructor(private store: Store<G, S>) {}
+  get() { return this.store.get() }
+  update(newState: Partial<S>) {
+    this.store.update(newState)
+    this.subscriptions.forEach(sb => sb())
+  }
+  private subscriptions: Function[] = []
+  subscribe<T>(selector: (g: G) => T, cb: (val: T) => void) {
+    let prevVal = selector(this.get())
+    const checkSubscription = () => {
+      const newVal = selector(this.get())
+      if (newVal !== prevVal) {
+        prevVal = newVal
+        cb(newVal)
+      }
+    }
+    this.subscriptions.push(checkSubscription)
+  }
+}

frontend/app/player/common/types.ts

@@ -10,10 +10,6 @@ export interface Moveable {
   move(time: number): void
 }
 
-export interface Cleanable {
-  clean(): void
-}
-
 export interface Interval {
   contains(t: number): boolean
   start: number

frontend/app/player/player/Animator.ts

@@ -80,7 +80,7 @@ export default class Animator {
         endTime,
         live,
         livePlay,
-        ready,  // = messagesLoading || cssLoading || disconnected
+        ready,
 
         lastMessageTime,
       } = this.store.get()

frontend/app/player/player/Player.ts

@@ -1,6 +1,6 @@
 import * as typedLocalStorage from './localStorage';
 
-import type { Moveable, Cleanable, Store } from '../common/types';
+import type { Moveable, Store } from '../common/types';
 import Animator from './Animator';
 import type { GetState as AnimatorGetState } from './Animator';
 
@@ -25,40 +25,38 @@ export type State = typeof Player.INITIAL_STATE
 export default class Player extends Animator {
   static INITIAL_STATE = {
     ...Animator.INITIAL_STATE,
-    skipToIssue: initialSkipToIssue,
-    showEvents: initialShowEvents,
 
+    showEvents: initialShowEvents,
     autoplay: initialAutoplay,
+
     skip: initialSkip,
     speed: initialSpeed,
   } as const
 
-  constructor(private pState: Store<State & AnimatorGetState>, private manager: Moveable & Cleanable) {
+  constructor(private pState: Store<State & AnimatorGetState>, private manager: Moveable) {
     super(pState, manager)
 
-    // Autoplay
+    // Autostart
-    if (pState.get().autoplay) {
+    let autostart = true // TODO: configurable
-      let autoPlay = true;
+    document.addEventListener("visibilitychange", () => {
-      document.addEventListener("visibilitychange", () => {
+      if (document.hidden) {
-        if (document.hidden) {
+        const { playing } = pState.get();
-          const { playing } = pState.get();
+        autostart = playing
-          autoPlay = playing
+        if (playing) {
-          if (playing) {
+          this.pause();
-            this.pause();
-          }
-        } else if (autoPlay) {
-          this.play();
         }
-      })
+      } else if (autostart) {
-
-      if (!document.hidden) {
         this.play();
       }
+    })
+    if (!document.hidden) {
+      this.play();
     }
   }
 
   /* === TODO: incapsulate in LSCache === */
 
+   //TODO: move to react part ("autoplay" responsible for auto-playing-next)
   toggleAutoplay() {
     const autoplay = !this.pState.get().autoplay
     localStorage.setItem(AUTOPLAY_STORAGE_KEY, `${autoplay}`);
@@ -72,13 +70,6 @@ export default class Player extends Animator {
     this.pState.update({ showEvents })
   }
 
-  // TODO: move to React part
-  toggleSkipToIssue() {
-    const skipToIssue = !this.pState.get().skipToIssue
-    localStorage.setItem(SKIP_TO_ISSUE_STORAGE_KEY, `${skipToIssue}`);
-    this.pState.update({ skipToIssue })
-  }
-
   toggleSkip() {
     const skip = !this.pState.get().skip
     localStorage.setItem(SKIP_STORAGE_KEY, `${skip}`);
@@ -108,7 +99,6 @@ export default class Player extends Animator {
 
   clean() {
     this.pause()
-    this.manager.clean()
   }
 
 }

frontend/app/player/web/MessageLoader.ts

@@ -0,0 +1,92 @@
+import logger from 'App/logger';
+
+import { decryptSessionBytes } from './network/crypto';
+import MFileReader from './messages/MFileReader';
+import { loadFiles, requestEFSDom, requestEFSDevtools } from './network/loadFiles';
+import type {
+  Message,
+} from './messages';
+
+import type  { Store } from '../common/types';
+
+
+interface SessionFilesInfo {
+  startedAt: number
+  sessionId: string
+  
+  domURL: string[]
+  devtoolsURL: string[]
+  mobsUrl: string[] // back-compatibility. TODO: Remove in the 1.11.0
+  fileKey: string  | null
+}
+
+type State = typeof MessageLoader.INITIAL_STATE
+
+export default class MessageLoader {
+  static INITIAL_STATE = {
+    firstFileLoading: false,
+    domLoading: false,
+    devtoolsLoading: false,
+  }
+
+  private lastMessageInFileTime: number = 0;
+
+	constructor(
+    private readonly session: SessionFilesInfo,
+    private store: Store<State>,
+    private distributeMessage: (msg: Message, index: number) => void,
+  ) {}
+  
+  private createNewParser(shouldDecrypt=true) {
+    const fKey = this.session.fileKey
+    const decrypt = shouldDecrypt && fKey
+      ? (b: Uint8Array) => decryptSessionBytes(b, fKey)
+      : (b: Uint8Array) => Promise.resolve(b)
+    // Each time called - new fileReader created. TODO: reuseable decryptor instance
+    const fileReader = new MFileReader(new Uint8Array(), this.session.startedAt)
+    return (b: Uint8Array) => decrypt(b).then(b => {
+      fileReader.append(b)
+      const msgs: Array<Message> = []
+      for (let msg = fileReader.readNext();msg !== null;msg = fileReader.readNext()) {
+        this.distributeMessage(msg, msg._index)
+        msgs.push(msg)
+      }
+
+      logger.info("Messages loaded: ", msgs.length, msgs)
+
+      //this._sortMessagesHack(fileReader) // TODO
+      this.store.update({ firstFileLoading: false }) // How to do it more explicit: on the first file loading?
+    })
+  }
+
+  requestFallbackDOM = () => 
+    requestEFSDom(this.session.sessionId)
+    .then(this.createNewParser(false))
+
+	loadDOM() {
+    this.store.update({ 
+      domLoading: true,
+      firstFileLoading: true,
+    })
+
+    const loadMethod = this.session.domURL && this.session.domURL.length > 0
+      ? { url: this.session.domURL, needsDecryption: true }
+      : { url: this.session.mobsUrl, needsDecryption: false }
+
+    return loadFiles(loadMethod.url, this.createNewParser(loadMethod.needsDecryption))
+      // EFS fallback
+      .catch((e) => this.requestFallbackDOM())
+      .finally(() => this.store.update({ domLoading: false }))
+  }
+
+  loadDevtools() {
+    this.store.update({ devtoolsLoading: true })
+    return loadFiles(this.session.devtoolsURL, this.createNewParser())
+      // EFS fallback
+      .catch(() =>
+        requestEFSDevtools(this.session.sessionId)
+          .then(this.createNewParser(false))
+      )
+      .finally(() => this.store.update({ devtoolsLoading: false }))
+  }
+}

frontend/app/player/web/MessageManager.ts

@@ -6,8 +6,6 @@ import { TYPES as EVENT_TYPES } from 'Types/session/event';
 import { Log } from './types/log';
 import { Resource, ResourceType, getResourceFromResourceTiming, getResourceFromNetworkRequest } from './types/resource'
 
-import { toast } from 'react-toastify';
-
 import type  { Store, Timed } from '../common/types';
 import ListWalker from '../common/ListWalker';
 
@@ -36,7 +34,6 @@ import { decryptSessionBytes } from './network/crypto';
 import Lists, { INITIAL_STATE as LISTS_INITIAL_STATE, State as ListsState } from './Lists';
 
 import Screen, {
-  INITIAL_STATE as SCREEN_INITIAL_STATE,
   State as ScreenState,
 } from './Screen/Screen';
 
@@ -57,13 +54,9 @@ export interface State extends ScreenState, ListsState {
   domContentLoadedTime?:  { time: number, value: number },
   domBuildingTime?: number,
   loadTime?: { time: number, value: number },
-  error: boolean,
-  devtoolsLoading: boolean,
 
-  messagesLoading: boolean,
   cssLoading: boolean,
 
-  ready: boolean,
   lastMessageTime: number,
 }
 
@@ -80,16 +73,12 @@ const visualChanges = [
 
 export default class MessageManager {
   static INITIAL_STATE: State = {
-    ...SCREEN_INITIAL_STATE,
+    ...Screen.INITIAL_STATE,
     ...LISTS_INITIAL_STATE,
     performanceChartData: [],
     skipIntervals: [],
-    error: false,
-    devtoolsLoading: false,
 
-    messagesLoading: false,
     cssLoading: false,
-    ready: false,
     lastMessageTime: 0,
   }
 
@@ -137,117 +126,6 @@ export default class MessageManager {
     this.activityManager = new ActivityManager(this.session.duration.milliseconds) // only if not-live
   }
 
-  private setCSSLoading = (cssLoading: boolean) => {
-    this.screen.displayFrame(!cssLoading)
-    this.state.update({ cssLoading, ready: !this.state.get().messagesLoading && !cssLoading })
-  }
-
-  private _sortMessagesHack(msgs: Message[]) {
-    // @ts-ignore Hack for upet (TODO: fix ordering in one mutation in tracker(removes first))
-    const headChildrenIds = msgs.filter(m => m.parentID === 1).map(m => m.id);
-    this.pagesManager.sortPages((m1, m2) => {
-      if (m1.time === m2.time) {
-        if (m1.tp === MType.RemoveNode && m2.tp !== MType.RemoveNode) {
-          if (headChildrenIds.includes(m1.id)) {
-            return -1;
-          }
-        } else if (m2.tp === MType.RemoveNode && m1.tp !== MType.RemoveNode) {
-          if (headChildrenIds.includes(m2.id)) {
-            return 1;
-          }
-        }  else if (m2.tp === MType.RemoveNode && m1.tp === MType.RemoveNode) {
-          const m1FromHead = headChildrenIds.includes(m1.id);
-          const m2FromHead = headChildrenIds.includes(m2.id);
-          if (m1FromHead && !m2FromHead) {
-            return -1;
-          } else if (m2FromHead && !m1FromHead) {
-            return 1;
-          }
-        }
-      }
-      return 0;
-    })
-  }
-
-  private waitingForFiles: boolean = false
-  private onFileReadSuccess = () => {
-    const stateToUpdate : Partial<State>= {
-      performanceChartData: this.performanceTrackManager.chartData,
-      performanceAvaliability: this.performanceTrackManager.avaliability,
-      ...this.lists.getFullListsState(),
-    }
-    if (this.activityManager) {
-      this.activityManager.end()
-      stateToUpdate.skipIntervals = this.activityManager.list
-    }
-    this.state.update(stateToUpdate)
-  }
-  private onFileReadFailed = (e: any) => {
-    logger.error(e)
-    this.state.update({ error: true })
-    toast.error('Error requesting a session file')
-  }
-  private onFileReadFinally = () => {
-    this.waitingForFiles = false
-    // this.setMessagesLoading(false)
-    // this.state.update({ filesLoaded: true })
-  }
-
-  async loadMessages(isClickmap: boolean = false) {
-    this.setMessagesLoading(true)
-    // TODO: reusable decryptor instance
-    const createNewParser = (shouldDecrypt = true) => {
-      const decrypt = shouldDecrypt && this.session.fileKey
-        ? (b: Uint8Array) => decryptSessionBytes(b, this.session.fileKey)
-        : (b: Uint8Array) => Promise.resolve(b)
-      // Each time called - new fileReader created
-      const fileReader = new MFileReader(new Uint8Array(), this.sessionStart)
-      return (b: Uint8Array) => decrypt(b).then(b => {
-        fileReader.append(b)
-        const msgs: Array<Message> = []
-        for (let msg = fileReader.readNext();msg !== null;msg = fileReader.readNext()) {
-          this.distributeMessage(msg, msg._index)
-          msgs.push(msg)
-        }
-
-        logger.info("Messages count: ", msgs.length, msgs)
-        this._sortMessagesHack(msgs)
-        this.setMessagesLoading(false)
-      })
-    }
-
-    this.waitingForFiles = true
-
-    const loadMethod = this.session.domURL && this.session.domURL.length > 0
-      ? { url: this.session.domURL, parser: createNewParser }
-      : { url: this.session.mobsUrl, parser: () => createNewParser(false)}
-
-    loadFiles(loadMethod.url, loadMethod.parser())
-      // EFS fallback
-      .catch((e) =>
-        requestEFSDom(this.session.sessionId)
-          .then(createNewParser(false))
-      )
-      .then(this.onFileReadSuccess)
-      .catch(this.onFileReadFailed)
-      .finally(this.onFileReadFinally);
-
-    // load devtools (TODO: start after the first DOM file download)
-    if (isClickmap) return;
-    this.state.update({ devtoolsLoading: true })
-    loadFiles(this.session.devtoolsURL, createNewParser())
-    // EFS fallback
-    .catch(() =>
-      requestEFSDevtools(this.session.sessionId)
-        .then(createNewParser(false))
-    )
-    .then(() => {
-      this.state.update(this.lists.getFullListsState()) // TODO: also in case of dynamic update through assist
-    })
-    .catch(e => logger.error("Can not download the devtools file", e))
-    .finally(() => this.state.update({ devtoolsLoading: false }))
-  }
-
   resetMessageManagers() {
     this.locationEventManager = new ListWalker();
     this.locationManager = new ListWalker();
@@ -327,10 +205,6 @@ export default class MessageManager {
         this.screen.cursor.click();
       }
     })
-
-    if (this.waitingForFiles && this.lastMessageTime <= t && t !== this.session.duration.milliseconds) {
-      this.setMessagesLoading(true)
-    }
   }
 
   private decodeStateMessage(msg: any, keys: Array<string>) {
@@ -347,7 +221,8 @@ export default class MessageManager {
     return { ...msg, ...decoded };
   }
 
-  distributeMessage(msg: Message, index: number): void {
+  distributeMessage = (msg: Message, index: number): void  => {
+    this._handeleForSortHack(msg)
     const lastMessageTime =  Math.max(msg.time, this.lastMessageTime)
     this.lastMessageTime = lastMessageTime
     this.state.update({ lastMessageTime })
@@ -471,9 +346,54 @@ export default class MessageManager {
     }
   }
 
-  setMessagesLoading(messagesLoading: boolean) {
+  private _heeadChildrenID: number[] = []
-    this.screen.display(!messagesLoading);
+  private _handeleForSortHack(m: Message) {
-    this.state.update({ messagesLoading, ready: !messagesLoading && !this.state.get().cssLoading });
+    // @ts-ignore
+    m.parentID === 1 && this._heeadChildrenID.push(m.id)
+  }
+  // Hack for upet (TODO: fix ordering in one mutation in tracker(removes first))
+  private _sortMessagesHack() {
+    const headChildrenIds = this._heeadChildrenID
+    this.pagesManager.sortPages((m1, m2) => {
+      if (m1.time === m2.time) {
+        if (m1.tp === MType.RemoveNode && m2.tp !== MType.RemoveNode) {
+          if (headChildrenIds.includes(m1.id)) {
+            return -1;
+          }
+        } else if (m2.tp === MType.RemoveNode && m1.tp !== MType.RemoveNode) {
+          if (headChildrenIds.includes(m2.id)) {
+            return 1;
+          }
+        }  else if (m2.tp === MType.RemoveNode && m1.tp === MType.RemoveNode) {
+          const m1FromHead = headChildrenIds.includes(m1.id);
+          const m2FromHead = headChildrenIds.includes(m2.id);
+          if (m1FromHead && !m2FromHead) {
+            return -1;
+          } else if (m2FromHead && !m1FromHead) {
+            return 1;
+          }
+        }
+      }
+      return 0;
+    })
+  }
+
+  onMessagesLoaded = () => {
+    const stateToUpdate : Partial<State>= {
+      performanceChartData: this.performanceTrackManager.chartData,
+      performanceAvaliability: this.performanceTrackManager.avaliability,
+      ...this.lists.getFullListsState(),
+    }
+    if (this.activityManager) {
+      this.activityManager.end()
+      stateToUpdate.skipIntervals = this.activityManager.list
+    }
+    this.state.update(stateToUpdate)
+    this._sortMessagesHack()
+  }
+
+  private setCSSLoading = (cssLoading: boolean) => {
+    this.state.update({ cssLoading })
   }
 
   private setSize({ height, width }: { height: number, width: number }) {

frontend/app/player/web/Screen/Screen.ts

@@ -6,19 +6,12 @@ import type { Point, Dimensions } from './types';
 
 export type State  = Dimensions
 
-export const INITIAL_STATE: State = {
-  width: 0,
-  height: 0,
-}
-
-
 export enum ScaleMode {
   Embed,
   //AdjustParentWidth
   AdjustParentHeight,
 }
 
-
 function getElementsFromInternalPoint(doc: Document, { x, y }: Point): Element[] {
   // @ts-ignore (IE, Edge)
   if (typeof doc.msElementsFromRect === 'function') {
@@ -57,6 +50,11 @@ function isIframe(el: Element): el is HTMLIFrameElement {
 }
 
 export default class Screen {
+  static INITIAL_STATE: State = {
+    width: 0,
+    height: 0,
+  }
+
   readonly overlay: HTMLDivElement
   readonly cursor: Cursor
 
@@ -91,20 +89,6 @@ export default class Screen {
 
     parentElement.appendChild(this.screen);
     this.parentElement = parentElement;
-
-    /* == For the Inspecting Document content  == */
-    this.overlay.addEventListener('contextmenu', () => {
-      this.overlay.style.display = 'none'
-      const doc = this.document
-      if (!doc) { return }
-      const returnOverlay = () => {
-        this.overlay.style.display = 'block'
-        doc.removeEventListener('mousemove', returnOverlay)
-        doc.removeEventListener('mouseclick', returnOverlay) // TODO: prevent default in case of input selection
-      }
-      doc.addEventListener('mousemove', returnOverlay)
-      doc.addEventListener('mouseclick', returnOverlay)
-    })
   }
 
   getParentElement():  HTMLElement | null {
@@ -131,7 +115,7 @@ export default class Screen {
   private getBoundingClientRect(): DOMRect {
      if (this.boundingRect === null) {
        // TODO: use this.screen instead in order to separate overlay functionality
-      return this.boundingRect = this.overlay.getBoundingClientRect() // expensive operation?
+      return this.boundingRect = this.screen.getBoundingClientRect() // expensive operation?
     }
     return this.boundingRect
   }
@@ -139,7 +123,7 @@ export default class Screen {
   getInternalViewportCoordinates({ x, y }: Point): Point {
     const { x: overlayX, y: overlayY, width } = this.getBoundingClientRect();
 
-    const screenWidth = this.overlay.offsetWidth;
+    const screenWidth = this.screen.offsetWidth;
 
     const scale = screenWidth / width;
     const screenX = (x - overlayX) * scale;
@@ -240,7 +224,7 @@ export default class Screen {
       width: width + 'px',
     })
 
-    this.boundingRect = this.overlay.getBoundingClientRect();
+    this.boundingRect = this.screen.getBoundingClientRect();
   }
 
 }

frontend/app/player/web/WebFilePlayer.ts

@@ -0,0 +1 @@
+// separate Player from File & WebLive player

frontend/app/player/web/WebLivePlayer.ts

@@ -15,6 +15,10 @@ export default class WebLivePlayer extends WebPlayer {
     ...WebPlayer.INITIAL_STATE,
     ...AssistManager.INITIAL_STATE,
     liveTimeTravel: false,
+    assistLoading: false,
+    // get ready() { // TODO TODO TODO how to extend state here?
+    //   return this.assistLoading && super.ready
+    // }
   }
 
   assistManager: AssistManager // public so far
@@ -23,12 +27,12 @@ export default class WebLivePlayer extends WebPlayer {
   private lastMessageInFileTime = 0
   private lastMessageInFileIndex = 0
 
-  constructor(wpState: Store<typeof WebLivePlayer.INITIAL_STATE>, private session:any, config: RTCIceServer[]) {
+  constructor(wpStore: Store<typeof WebLivePlayer.INITIAL_STATE>, private session:any, config: RTCIceServer[]) {
-    super(wpState, session, true)
+    super(wpStore, session, true)
 
     this.assistManager = new AssistManager(
       session,
-      f => this.messageManager.setMessagesLoading(f),
+      assistLoading => wpStore.update({ assistLoading }),
       (msg, idx) => {
         this.incomingMessages.push(msg)
         if (!this.historyFileIsLoading) {
@@ -38,31 +42,27 @@ export default class WebLivePlayer extends WebPlayer {
       },
       this.screen,
       config,
-      wpState,
+      wpStore,
     )
     this.assistManager.connect(session.agentToken)
   }
 
   toggleTimetravel = async () => {
-    if (this.wpState.get().liveTimeTravel) {
+    // TODO: implement via jump() API rewritten instead
+    if (this.wpStore.get().liveTimeTravel) {
       return
     }
     let result = false;
     this.historyFileIsLoading = true
-    this.messageManager.setMessagesLoading(true) // do it in one place. update unique  loading states each time instead
     this.messageManager.resetMessageManagers()
 
     try {
-      const bytes = await requestEFSDom(this.session.sessionId)
+      await this.messageLoader.requestFallbackDOM()
-      const fileReader = new MFileReader(bytes, this.session.startedAt)
+      this.wpStore.update({
-      for (let msg = fileReader.readNext();msg !== null;msg = fileReader.readNext()) {
-        this.messageManager.distributeMessage(msg, msg._index)
-      }
-      this.wpState.update({
         liveTimeTravel: true,
       })
+      this.messageManager.onMessagesLoaded()
       result = true
-      // here we need to update also lists state, if we gonna use them this.messageManager.onFileReadSuccess
     } catch(e) {
       toast.error('Error requesting a session file')
       console.error("EFS file download error:", e)
@@ -75,16 +75,14 @@ export default class WebLivePlayer extends WebPlayer {
     this.incomingMessages.length = 0
 
     this.historyFileIsLoading = false
-    this.messageManager.setMessagesLoading(false)
     return result;
   }
 
   jumpToLive = () => {
-    this.wpState.update({
+    this.wpStore.update({
-      live: true,
       livePlay: true,
     })
-    this.jump(this.wpState.get().lastMessageTime)
+    this.jump(this.wpStore.get().lastMessageTime)
   }
 
   clean = () => {

frontend/app/player/web/WebPlayer.ts

@@ -1,31 +1,38 @@
 import { Log, LogLevel } from './types/log'
+import { toast } from 'react-toastify';
+import logger from 'App/logger';
 
-import type { Store } from 'App/player'
+import type { Store } from '../common/types'
+import StorSubscriber from '../common/StoreSubscriber'
 import Player from '../player/Player'
 
 import MessageManager from './MessageManager'
 import InspectorController from './addons/InspectorController'
 import TargetMarker from './addons/TargetMarker'
 import Screen, { ScaleMode } from './Screen/Screen'
+import MessageLoader from './MessageLoader'
 
 
-// export type State = typeof WebPlayer.INITIAL_STATE
-
 export default class WebPlayer extends Player {
   static readonly INITIAL_STATE = {
     ...Player.INITIAL_STATE,
     ...TargetMarker.INITIAL_STATE,
     ...MessageManager.INITIAL_STATE,
 
+    ...MessageLoader.INITIAL_STATE,
+  
+    ready: true,
     inspectorMode: false,
   }
 
   private readonly inspectorController: InspectorController
   protected readonly screen: Screen
+  protected readonly messageLoader: MessageLoader
   protected readonly messageManager: MessageManager
 
   private targetMarker: TargetMarker
 
+
   constructor(protected wpState: Store<typeof WebPlayer.INITIAL_STATE>, session: any, live: boolean, isClickMap = false) {
     let initialLists = live ? {} : {
       event: session.events || [],
@@ -39,13 +46,32 @@ export default class WebPlayer extends Player {
       ) || [],
     }
 
+    const store = new StorSubscriber(wpState)
     const screen = new Screen(session.isMobile, isClickMap ? ScaleMode.AdjustParentHeight : ScaleMode.Embed)
-    const messageManager = new MessageManager(session, wpState, screen, initialLists)
+    const messageManager = new MessageManager(session, store, screen, initialLists)
-    super(wpState, messageManager)
+    //TODO: same for scaling
+    store.subscribe(state => state.cssLoading, cssLoading => this.screen.displayFrame(!cssLoading))
+    store.subscribe(state => state.domLoading, domLoading => this.screen.display(!domLoading))
+    store.subscribe(state => {
+      const notReady = state.cssLoading || (state.domLoading && state.time >= state.lastMessageTime)
+      return !notReady
+    }, ready => store.update({ ready }))
+
+    super(store, messageManager)
     this.screen = screen
     this.messageManager = messageManager
+    this.messageLoader = new MessageLoader(session, wpState, messageManager.distributeMessage)
+
     if (!live) { // hack. TODO: split OfflinePlayer class
-      void messageManager.loadMessages(isClickMap)
+      this.messageLoader.loadDOM()
+      .then(() => messageManager.onMessagesLoaded())
+      .catch((e: any) => {
+        logger.error(e)
+        toast.error('Error requesting a session file') // TODO: outside of the player lib
+      })
+      this.messageLoader.loadDevtools()
+      .then(() => messageManager.onMessagesLoaded())
+      .catch(e => logger.error("Can not download the devtools file", e))
     }
 
     this.targetMarker = new TargetMarker(this.screen, wpState)
@@ -127,6 +153,7 @@ export default class WebPlayer extends Player {
 
   clean = () => {
     super.clean()
+    this.messageManager.clean()
     window.removeEventListener('resize', this.scale)
   }
 }

frontend/app/player/web/addons/InspectorController.ts

@@ -8,7 +8,20 @@ export default class InspectorController {
   private substitutor: Screen | null = null
   private inspector: Inspector | null = null
   marker: Marker | null = null
-  constructor(private screen: Screen) {}
+  constructor(private screen: Screen) {
+    screen.overlay.addEventListener('contextmenu', () => {
+      screen.overlay.style.display = 'none'
+      const doc = screen.document
+      if (!doc) { return }
+      const returnOverlay = () => {
+        screen.overlay.style.display = 'block'
+        doc.removeEventListener('mousemove', returnOverlay)
+        doc.removeEventListener('mouseclick', returnOverlay) // TODO: prevent default in case of input selection
+      }
+      doc.addEventListener('mousemove', returnOverlay)
+      doc.addEventListener('mouseclick', returnOverlay)
+    })
+  }
 
   scale(dims: Dimensions) {
     if (this.substitutor) {

frontend/app/player/web/managers/types.ts

@@ -0,0 +1,3 @@
+export interface MessageManager {
+	handleMessage(msg: Message): void
+}

frontend/app/player/web/messageLoader/MessageLoader.ts

@@ -0,0 +1,163 @@
+import { toast } from 'react-toastify';
+import logger from 'App/logger';
+
+import { decryptSessionBytes } from '../network/crypto';
+import MFileReader from '../messages/MFileReader';
+import { loadFiles, requestEFSDom, requestEFSDevtools } from './loadFiles';
+import type {
+  Message,
+} from '../messages';
+
+
+
+export default class MessageLoader {
+  private lastMessageInFileTime: number = 0;
+
+	constructor(
+    private readonly session: any /*Session*/,
+    private setMessagesLoading: (flag: boolean) => void,
+    private distributeMessage: (msg: Message, index: number) => void,
+  ) {}
+
+  requestEFSFile() {
+    this.setMessagesLoading(true)
+    this.waitingForFiles = true
+    const onData = (byteArray: Uint8Array) => {
+      const onMessage = (msg: Message) => { this.lastMessageInFileTime = msg.time }
+      this.parseAndDistributeMessages(new MFileReader(byteArray, this.session.startedAt), onMessage)
+    }
+
+    // assist will pause and skip messages to prevent timestamp related errors
+    // ----> this.reloadMessageManagers()
+    // ---> this.windowNodeCounter.reset()
+
+    
+
+    return requestEFSDom(this.session.sessionId)
+      .then(onData)
+      // --->.then(this.onFileReadSuccess)
+      // --->.catch(this.onFileReadFailed)
+      .finally(this.onFileReadFinally)
+  }
+
+  private parseAndDistributeMessages(fileReader: MFileReader, onMessage?: (msg: Message) => void) {
+    const msgs: Array<Message> = []
+    let next: ReturnType<MFileReader['next']>
+    while (next = fileReader.next()) {
+      const [msg, index] = next
+      this.distributeMessage(msg, index)
+      msgs.push(msg)
+      onMessage?.(msg)
+    }
+
+
+    logger.info("Messages loaded: ", msgs.length, msgs)
+
+    // @ts-ignore Hack for upet (TODO: fix ordering in one mutation in tracker(removes first))
+    //--->// const headChildrenIds = msgs.filter(m => m.parentID === 1).map(m => m.id);
+    // this.pagesManager.sortPages((m1, m2) => {
+    //   if (m1.time === m2.time) {
+    //     if (m1.tp === MType.RemoveNode && m2.tp !== MType.RemoveNode) {
+    //       if (headChildrenIds.includes(m1.id)) {
+    //         return -1;
+    //       }
+    //     } else if (m2.tp === MType.RemoveNode && m1.tp !== MType.RemoveNode) {
+    //       if (headChildrenIds.includes(m2.id)) {
+    //         return 1;
+    //       }
+    //     }  else if (m2.tp === MType.RemoveNode && m1.tp === MType.RemoveNode) {
+    //       const m1FromHead = headChildrenIds.includes(m1.id);
+    //       const m2FromHead = headChildrenIds.includes(m2.id);
+    //       if (m1FromHead && !m2FromHead) {
+    //         return -1;
+    //       } else if (m2FromHead && !m1FromHead) {
+    //         return 1;
+    //       }
+    //     }
+    //   }
+    //   return 0;
+    // })
+  }
+
+  private waitingForFiles: boolean = false
+  //--->// private onFileReadSuccess = () => {
+  //   const stateToUpdate : Partial<State>= {
+  //     performanceChartData: this.performanceTrackManager.chartData,
+  //     performanceAvaliability: this.performanceTrackManager.avaliability,
+  //     ...this.lists.getFullListsState(),
+  //   }
+  //   if (this.activityManager) {
+  //     this.activityManager.end()
+  //     stateToUpdate.skipIntervals = this.activityManager.list
+  //   }
+  //   this.state.update(stateToUpdate)
+  // }
+  //---> private onFileReadFailed = (e: any) => {
+  //   this.state.update({ error: true })
+  //   logger.error(e)
+  //   toast.error('Error requesting a session file')
+  // }
+  private onFileReadFinally = () => {
+    //--->// this.incomingMessages
+    //   .filter(msg => msg.time >= this.lastMessageInFileTime)
+    //   .forEach(msg => this.distributeMessage(msg, 0))
+
+    this.waitingForFiles = false
+    this.setMessagesLoading(false)
+  }
+
+  
+  private createNewParser(shouldDecrypt=true) {
+    const decrypt = shouldDecrypt && this.session.fileKey
+      ? (b: Uint8Array) => decryptSessionBytes(b, this.session.fileKey)
+      : (b: Uint8Array) => Promise.resolve(b)
+    // Each time called - new fileReader created. TODO: reuseable decryptor instance
+    const fileReader = new MFileReader(new Uint8Array(), this.session.startedAt)
+    return (b: Uint8Array) => decrypt(b).then(b => {
+      fileReader.append(b)
+      this.parseAndDistributeMessages(fileReader)
+      this.setMessagesLoading(false)
+    })
+  }
+
+	loadDOM() {
+    this.setMessagesLoading(true)
+    this.waitingForFiles = true
+
+    let fileReadPromise = this.session.domURL && this.session.domURL.length > 0
+      ? loadFiles(this.session.domURL, this.createNewParser())
+      : Promise.reject()
+    
+    return fileReadPromise
+    // EFS fallback
+    .catch(() => requestEFSDom(this.session.sessionId).then(this.createNewParser(false)))
+    // old url fallback
+    .catch(e => {
+      logger.error('Can not get normal session replay file:', e)
+      // back compat fallback to an old mobsUrl
+      return loadFiles(this.session.mobsUrl, this.createNewParser(false))
+    })
+    // --->.then(this.onFileReadSuccess)
+    // --->.catch(this.onFileReadFailed)
+    .finally(this.onFileReadFinally)
+  }
+
+  loadDevtools() {
+    // load devtools
+    if (this.session.devtoolsURL.length) {
+      // ---> this.state.update({ devtoolsLoading: true })
+      return loadFiles(this.session.devtoolsURL, this.createNewParser())
+      .catch(() =>
+        requestEFSDevtools(this.session.sessionId)
+          .then(this.createNewParser(false))
+      )
+      //--->// .then(() => {
+      //   this.state.update(this.lists.getFullListsState())
+      // })
+      // --->.catch(e => logger.error("Can not download the devtools file", e))
+      // --->//.finally(() => this.state.update({ devtoolsLoading: false }))
+    }
+    return Promise.resolve()
+  }
+
+}

frontend/app/player/web/messageLoader/loadFiles.ts

@@ -0,0 +1,60 @@
+import APIClient from 'App/api_client';
+
+const NO_NTH_FILE = "nnf"
+const NO_UNPROCESSED_FILES = "nuf"
+
+export async function* loadFiles(
+  urls: string[],
+){
+  const firstFileURL = urls[0]
+  urls = urls.slice(1)
+  if (!firstFileURL) {
+    throw "No urls provided"
+  }
+  try {
+    yield await window.fetch(firstFileURL)
+      .then(r => processAPIStreamResponse(r, true))
+    for(const url in urls) {
+      yield await window.fetch(url)
+        .then(r =>  processAPIStreamResponse(r, false))
+    }
+  } catch(e) {
+    if (e === NO_NTH_FILE) {
+      return
+    }
+    throw e
+  }
+}
+
+
+export async function requestEFSDom(sessionId: string) {
+  return await requestEFSMobFile(sessionId + "/dom.mob")
+}
+
+export async function requestEFSDevtools(sessionId: string) {
+  return await requestEFSMobFile(sessionId + "/devtools.mob")
+}
+
+async function requestEFSMobFile(filename: string) {
+  const api = new APIClient()
+  const res = await api.fetch('/unprocessed/' + filename)
+  if (res.status >= 400) {
+    throw NO_UNPROCESSED_FILES
+  }
+  return await processAPIStreamResponse(res, false)
+}
+
+const processAPIStreamResponse = (response: Response, isFirstFile: boolean) => {
+  return new Promise<ArrayBuffer>((res, rej) => {
+    if (response.status === 404 && !isFirstFile) {
+      return rej(NO_NTH_FILE)
+    }
+    if (response.status >= 400) {
+      return rej(
+        isFirstFile ? `no start file. status code ${ response.status }`
+        : `Bad endfile status code ${response.status}`
+      )
+    }
+    res(response.arrayBuffer())
+  }).then(buffer => new Uint8Array(buffer))
+}

frontend/app/player/web/messages/MFileReader.ts

@@ -16,6 +16,10 @@ export default class MFileReader extends RawMessageReader {
     super(data)
   }
 
+  getPosition() {
+    return this.p
+  }
+
   private needSkipMessage(): boolean {
     if (this.p === 0) return false
     for (let i = 7; i >= 0; i--) {

read-file.ts

@@ -0,0 +1,173 @@
+import MFileReader from './frontend/app/player/web/messages/MFileReader';
+import {
+  MType,
+} from './frontend/app/player/web/messages/raw.gen';
+
+import fs from 'fs'
+
+
+// silent logger
+// const logger = {
+//     log(){},
+//     error(){},
+//     warn(){},
+//     group(){},
+// }
+
+/**
+ * For reading big files by chunks
+*/
+function readBytes(fd, sharedBuffer) {
+  return new Promise((resolve, reject) => {
+    fs.read(
+      fd, 
+      sharedBuffer,
+      0,
+      sharedBuffer.length,
+      null,
+      (err) => {
+          if(err) { return reject(err); }
+          resolve(void)
+      }
+    )
+  })
+}
+async function* readByChunks(filePath, size) {
+  const sharedBuffer = Buffer.alloc(size);
+  const stats = fs.statSync(filePath); // file details
+  const fd = fs.openSync(filePath); // file descriptor
+  let bytesRead = 0; // how many bytes were read
+  let end = size; 
+  
+  for(let i = 0; i < Math.ceil(stats.size / size); i++) {
+    await readBytes(fd, sharedBuffer);
+    bytesRead = (i + 1) * size;
+    if(bytesRead > stats.size) {
+       // When we reach the end of file, 
+       // we have to calculate how many bytes were actually read
+       end = size - (bytesRead - stats.size);
+    }
+    yield sharedBuffer.slice(0, end);
+  }
+}
+/* ==== end chunk-reader === */
+
+/*== Message generators ==*/
+async function* readBigFileMessages(file: string, chunkSize: number) {
+  let i = 0
+  const fileReader = new MFileReader(new Uint8Array(), 0)
+  for await(const chunk of readByChunks(file, chunkSize)) {
+    i++
+    fileReader.append(chunk)
+    let msg
+    while (msg = fileReader.readNext()) {
+      yield msg
+    }
+    console.log("Read chunk: ", i)
+  }
+}
+function* readMessagesTwoFiles(filename1: string, filename2: string) {
+  const file1 = fs.readFileSync(filename1)
+  const file2 = fs.readFileSync(filename2)
+  console.log("First file: ", file1.length, " bytes")
+  const fileReader = new MFileReader(file1, 0 )
+  fileReader.append(file2)
+  let msg
+  while (msg = fileReader.readNext()) {
+    yield [ msg, fileReader.getPosition() ]
+  }
+}
+/*== end message generators ==*/
+
+
+
+function addToMap(map, key, n=1) {
+    map[key] = map[key] ? map[key] + n : n
+}
+
+function estimateSize(msg) {
+  return Object.values(msg).reduce((prevSum: number, val: any) => {
+    if (typeof val === "string") {
+        return prevSum + val.length + 1
+    }
+    if (typeof val ==="number") {
+        return prevSum + 2
+    }
+    return prevSum
+  }, 0)
+}
+
+const mapByTp = {}
+const mapBySize = {}
+const stringRepeatMapAttrs = {}
+const stringRepeatMapAttrsNodes = {}
+const stringRepeatMapOthers = {}
+function updateStringsMap(map, msg){
+    Object.values(msg).forEach(val => {
+        if (typeof val === "string") {
+            addToMap(map, val)
+        }
+    })
+}
+
+
+const CHUNK_SIZE = 100000000; // 100MB
+const FILE =  "../decrypted0.mob"
+
+let lastI = 0
+let currentPageUrl = ""
+async function main() {  
+  for (const msg of  readBigFileMessages(FILE, CHUNK_SIZE)) {
+    console.log(msg)
+
+    // const index = msg._index
+    // lastI = isNaN(index) ? lastI : index
+
+    // addToMap(mapByTp, msg.tp)
+    // addToMap(mapBySize, msg.tp, estimateSize(msg))
+  
+    // if (msg.tp === 4) {
+    //   currentPageUrl = msg.url
+    // }
+
+    // if (msg.tp === 12) {
+
+    //   if (!stringRepeatMapAttrsNodes[msg.name]) {
+    //       stringRepeatMapAttrsNodes[msg.name] = {}
+    //   }
+    //   if (!stringRepeatMapAttrsNodes[msg.value]) {
+    //       stringRepeatMapAttrsNodes[msg.value] = {}
+    //   }
+
+    //   addToMap(stringRepeatMapAttrsNodes[msg.name], msg.id)
+    //   addToMap(stringRepeatMapAttrsNodes[msg.value], msg.id)
+
+    //   updateStringsMap(stringRepeatMapAttrs, msg)
+    // } else { updateStringsMap(stringRepeatMapOthers, msg)}  
+ 
+  }
+}
+function calcStrMapStats(strMap){
+  const topStringMap = {}
+  const stringEntries = Object.entries(strMap)
+  stringEntries
+      .sort(([k1, v1], [k2, v2]) => v1*k1.length - v2*k2.length)
+      .slice(-10)
+      .forEach(([key, val]) => topStringMap[key]=val)
+
+  const keySize = Math.log10(stringEntries.length)
+
+  const strSum = stringEntries
+      .reduce((s, [k, v]) => s+k.length*v, 0)
+  const redSum = stringEntries
+      .reduce((s, [k, v]) => s+v*keySize, 0)
+  const dictSize = stringEntries
+      .reduce((s, [k, v]) => s+k.length +keySize, 0)
+  return [ strSum, redSum, dictSize, keySize, topStringMap ]
+}
+const statsAttr = calcStrMapStats(stringRepeatMapAttrs)
+const statsOthers = calcStrMapStats(stringRepeatMapOthers)
+
+
+// main()
+

tracker/tracker/src/main/plugin.ts

@@ -0,0 +1,56 @@
+import type Message from './app/messages.gen.js'
+import type App from './app/index.js'
+//import { PluginPayload } from './app/messages.gen.js'
+
+// TODO: sendState(name, action, state) for state plugins;
+
+export interface PluginDescription {
+  name: string
+  version: string
+  requiredTrackerVersion: string
+  onStart: () => void
+  onStop: () => void
+  onNode: (node: Node, id: number) => void
+  onContext: (context: typeof globalThis) => void
+}
+
+type PluginWrapper = (app: AppForPlugins) => Partial<PluginDescription>
+
+interface AppForPlugins {
+  sendMessage(m: Message): void
+  send(name: string, payload: Object): void
+  // ...
+}
+
+export function applyPlugin<T>(app: App, pluginWrapper: PluginWrapper) {
+  function send(name: string, second?: Object) {
+    const paload = app.safe(() => JSON.stringify(second))() || ''
+    // app.send(PluginPayload(name, payload)) // send PluginPayload message
+    return
+  }
+  function sendMessage(msg: Message) {
+    app.send(msg)
+  }
+  const plugin = pluginWrapper({
+    send,
+    sendMessage,
+    //...
+  })
+
+  if (plugin.onStart) {
+    app.attachStartCallback(plugin.onStart)
+  }
+  if (plugin.onStop) {
+    app.attachStopCallback(plugin.onStop)
+  }
+  if (plugin.onNode) {
+    app.nodes.attachNodeCallback((node) => {
+      const id = app.nodes.getID(node)
+      if (id !== undefined) {
+        plugin.onNode(node, id)
+      }
+    })
+  }
+
+  return plugin
+}