feat: canvas support [assist] (#1641)

* feat(tracker/ui): start canvas support

* feat(tracker): slpeer -> peerjs for canvas streams

* fix(ui): fix agent canvas peer id

* fix(ui): fix agent canvas peer id

* fix(ui): fix peer removal

* feat(tracker): canvas recorder

* feat(tracker): canvas recorder

* feat(tracker): canvas recorder

* feat(tracker): canvas recorder

* feat(ui): canvas support for ui

* fix(tracker): fix falling tests

* feat(ui): replay canvas in video

* feat(ui): refactor video streaming to draw on canvas

* feat(ui): 10hz check for canvas replay

* feat(ui): fix for tests

* feat(ui): fix for tests

* feat(ui): fix for tests

* feat(ui): fix for tests cov

* feat(ui): mroe test coverage

* fix(ui): styling

* fix(tracker): support backend settings for canvas

.github/workflows/tracker-tests.yaml

@@ -47,10 +47,6 @@ jobs:
         run: |
           cd tracker/tracker
           bun install
-      - name: (TA) Setup Testing packages
-        run: |
-          cd tracker/tracker-assist
-          bun install
       - name: Jest tests
         run: |
           cd tracker/tracker
@@ -59,6 +55,10 @@ jobs:
         run: |
           cd tracker/tracker
           bun run build
+      - name: (TA) Setup Testing packages
+        run: |
+          cd tracker/tracker-assist
+          bun install
       - name: (TA) Jest tests
         run: |
           cd tracker/tracker-assist

backend/pkg/messages/filters.go

@@ -10,5 +10,5 @@ func IsIOSType(id int) bool {
 }
 
 func IsDOMType(id int) bool {
-	return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 114 == id || 117 == id || 118 == id || 93 == id || 96 == id || 100 == id || 101 == id || 102 == id || 103 == id || 104 == id || 105 == id || 106 == id || 111 == id
+	return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 114 == id || 117 == id || 118 == id || 119 == id || 93 == id || 96 == id || 100 == id || 101 == id || 102 == id || 103 == id || 104 == id || 105 == id || 106 == id || 111 == id
 }

backend/pkg/messages/messages.go

@@ -84,6 +84,7 @@ const (
 	MsgResourceTiming              = 116
 	MsgTabChange                   = 117
 	MsgTabData                     = 118
+	MsgCanvasNode                  = 119
 	MsgIssueEvent                  = 125
 	MsgSessionEnd                  = 126
 	MsgSessionSearch               = 127
@@ -2245,6 +2246,29 @@ func (msg *TabData) TypeID() int {
 	return 118
 }
 
+type CanvasNode struct {
+	message
+	NodeId    string
+	Timestamp uint64
+}
+
+func (msg *CanvasNode) Encode() []byte {
+	buf := make([]byte, 21+len(msg.NodeId))
+	buf[0] = 119
+	p := 1
+	p = WriteString(msg.NodeId, buf, p)
+	p = WriteUint(msg.Timestamp, buf, p)
+	return buf[:p]
+}
+
+func (msg *CanvasNode) Decode() Message {
+	return msg
+}
+
+func (msg *CanvasNode) TypeID() int {
+	return 119
+}
+
 type IssueEvent struct {
 	message
 	MessageID     uint64

backend/pkg/messages/read-message.go

@@ -1365,6 +1365,18 @@ func DecodeTabData(reader BytesReader) (Message, error) {
 	return msg, err
 }
 
+func DecodeCanvasNode(reader BytesReader) (Message, error) {
+	var err error = nil
+	msg := &CanvasNode{}
+	if msg.NodeId, err = reader.ReadString(); err != nil {
+		return nil, err
+	}
+	if msg.Timestamp, err = reader.ReadUint(); err != nil {
+		return nil, err
+	}
+	return msg, err
+}
+
 func DecodeIssueEvent(reader BytesReader) (Message, error) {
 	var err error = nil
 	msg := &IssueEvent{}
@@ -1993,6 +2005,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) {
 		return DecodeTabChange(reader)
 	case 118:
 		return DecodeTabData(reader)
+	case 119:
+		return DecodeCanvasNode(reader)
 	case 125:
 		return DecodeIssueEvent(reader)
 	case 126:

codecov.yml

@@ -0,0 +1,10 @@
+ignore:
+  - "**/*/*.gen.ts"
+  - "**/*/coverage.xml"
+  - "**/*/coverage-final.json"
+  - "**/*/coverage/**"
+  - "**/*/node_modules/**"
+  - "**/*/dist/**"
+  - "**/*/build/**"
+  - "**/*/.test.*"
+  - "**/*/version.ts"

ee/connectors/msgcodec/messages.py

@@ -788,6 +788,14 @@ class TabData(Message):
         self.tab_id = tab_id
 
 
+class CanvasNode(Message):
+    __id__ = 119
+
+    def __init__(self, node_id, timestamp):
+        self.node_id = node_id
+        self.timestamp = timestamp
+
+
 class IssueEvent(Message):
     __id__ = 125
 

ee/connectors/msgcodec/messages.pyx

@@ -1164,6 +1164,17 @@ cdef class TabData(PyMessage):
         self.tab_id = tab_id
 
 
+cdef class CanvasNode(PyMessage):
+    cdef public int __id__
+    cdef public str node_id
+    cdef public unsigned long timestamp
+
+    def __init__(self, str node_id, unsigned long timestamp):
+        self.__id__ = 119
+        self.node_id = node_id
+        self.timestamp = timestamp
+
+
 cdef class IssueEvent(PyMessage):
     cdef public int __id__
     cdef public unsigned long message_id

ee/connectors/msgcodec/msgcodec.py

@@ -711,6 +711,12 @@ class MessageCodec(Codec):
                 tab_id=self.read_string(reader)
             )
 
+        if message_id == 119:
+            return CanvasNode(
+                node_id=self.read_string(reader),
+                timestamp=self.read_uint(reader)
+            )
+
         if message_id == 125:
             return IssueEvent(
                 message_id=self.read_uint(reader),

ee/connectors/msgcodec/msgcodec.pyx

@@ -809,6 +809,12 @@ cdef class MessageCodec:
                 tab_id=self.read_string(reader)
             )
 
+        if message_id == 119:
+            return CanvasNode(
+                node_id=self.read_string(reader),
+                timestamp=self.read_uint(reader)
+            )
+
         if message_id == 125:
             return IssueEvent(
                 message_id=self.read_uint(reader),

frontend/.babelrc

@@ -6,10 +6,10 @@
     ],
     "plugins": [
         "babel-plugin-react-require",
-        [ "@babel/plugin-proposal-private-property-in-object", { "loose": true } ],
+        ["@babel/plugin-transform-private-property-in-object", { "loose":true } ],
         [ "@babel/plugin-transform-runtime", { "regenerator": true } ],
         [ "@babel/plugin-proposal-decorators", { "legacy":true } ],
-        [ "@babel/plugin-proposal-class-properties", { "loose":true } ],
+        [ "@babel/plugin-transform-class-properties", { "loose":true } ],
-        [ "@babel/plugin-proposal-private-methods", { "loose": true }]
+        [ "@babel/plugin-transform-private-methods", { "loose": true }]
     ]
 }

frontend/app/player/web/MessageManager.ts

@@ -236,6 +236,10 @@ export default class MessageManager {
     }
   }
 
+  public getNode(id: number) {
+    return this.tabs[this.activeTab]?.getNode(id);
+  }
+
   public changeTab(tabId: string) {
     this.activeTab = tabId;
     this.state.update({ currentTab: tabId });

frontend/app/player/web/Screen/Screen.ts

@@ -120,7 +120,7 @@ export default class Screen {
     return this.parentElement
   }
 
-  setBorderStyle(style: { border: string }) {
+  setBorderStyle(style: { outline: string }) {
     return Object.assign(this.screen.style, style)
   }
 

frontend/app/player/web/TabManager.ts

@@ -1,23 +1,28 @@
+import type { Store } from "Player";
+import { getResourceFromNetworkRequest, getResourceFromResourceTiming, Log, ResourceType } from "Player";
 import ListWalker from "Player/common/ListWalker";
+import Lists, { INITIAL_STATE as LISTS_INITIAL_STATE, InitialLists, State as ListsState } from "Player/web/Lists";
+import CanvasManager from "Player/web/managers/CanvasManager";
+import { VElement } from "Player/web/managers/DOM/VirtualDOM";
+import PagesManager from "Player/web/managers/PagesManager";
+import PerformanceTrackManager from "Player/web/managers/PerformanceTrackManager";
+import WindowNodeCounter from "Player/web/managers/WindowNodeCounter";
 import {
+  CanvasNode,
   ConnectionInformation,
-  Message, MType, ResourceTiming,
+  Message,
+  MType,
+  ResourceTiming,
   SetPageLocation,
   SetViewportScroll,
   SetViewportSize
 } from "Player/web/messages";
-import PerformanceTrackManager from "Player/web/managers/PerformanceTrackManager";
+import { isDOMType } from "Player/web/messages/filters.gen";
-import WindowNodeCounter from "Player/web/managers/WindowNodeCounter";
+import Screen from "Player/web/Screen/Screen";
-import PagesManager from "Player/web/managers/PagesManager";
 // @ts-ignore
 import { Decoder } from "syncod";
-import Lists, { InitialLists, INITIAL_STATE as LISTS_INITIAL_STATE, State as ListsState } from "Player/web/Lists";
-import type  { Store } from 'Player';
-import Screen from "Player/web/Screen/Screen";
 import { TYPES as EVENT_TYPES } from "Types/session/event";
-import type { PerformanceChartPoint } from './managers/PerformanceTrackManager';
+import type { PerformanceChartPoint } from "./managers/PerformanceTrackManager";
-import { getResourceFromNetworkRequest, getResourceFromResourceTiming, Log, ResourceType } from "Player";
-import { isDOMType } from "Player/web/messages/filters.gen";
 
 export interface TabState extends ListsState {
   performanceAvailability?: PerformanceTrackManager['availability']
@@ -57,6 +62,8 @@ export default class TabSessionManager {
   public readonly decoder = new Decoder();
   private lists: Lists;
   private navigationStartOffset = 0
+  private canvasManagers: { [key: string]: { manager: CanvasManager, start: number, running: boolean } } = {}
+  private canvasReplayWalker: ListWalker<CanvasNode> = new ListWalker();
 
   constructor(
     private readonly session: any,
@@ -76,6 +83,10 @@ export default class TabSessionManager {
     })
   }
 
+  public getNode = (id: number) => {
+    return this.pagesManager.getNode(id)
+  }
+
   public updateLists(lists: Partial<InitialLists>) {
     Object.keys(lists).forEach((key: 'event' | 'stack' | 'exceptions') => {
       const currentList = this.lists.lists[key]
@@ -141,6 +152,23 @@ export default class TabSessionManager {
 
   distributeMessage(msg: Message): void {
     switch (msg.tp) {
+      case MType.CanvasNode:
+        const managerId = `${msg.timestamp}_${msg.nodeId}`;
+        if (!this.canvasManagers[managerId]) {
+          const filename = `${managerId}.mp4`;
+          const delta = msg.timestamp - this.sessionStart;
+          const fileUrl = this.session.canvasURL.find((url: string) => url.includes(filename));
+          const manager = new CanvasManager(
+            msg.nodeId,
+            delta,
+            fileUrl,
+            this.getNode as (id: number) => VElement | undefined
+          );
+          this.canvasManagers[managerId] = { manager, start: msg.timestamp, running: false };
+          this.canvasReplayWalker.append(msg);
+
+        }
+        break;
       case MType.SetPageLocation:
         this.locationManager.append(msg);
         if (msg.navigationStart > 0) {
@@ -289,6 +317,16 @@ export default class TabSessionManager {
       if (!!lastScroll && this.screen.window) {
         this.screen.window.scrollTo(lastScroll.x, lastScroll.y);
       }
+      const canvasMsg = this.canvasReplayWalker.moveGetLast(t)
+      if (canvasMsg) {
+        this.canvasManagers[`${canvasMsg.timestamp}_${canvasMsg.nodeId}`].manager.startVideo();
+        this.canvasManagers[`${canvasMsg.timestamp}_${canvasMsg.nodeId}`].running = true;
+      }
+      const runningManagers = Object.keys(this.canvasManagers).filter((key) => this.canvasManagers[key].running);
+      runningManagers.forEach((key) => {
+        const manager = this.canvasManagers[key].manager;
+        manager.move(t);
+      })
     })
   }
 

frontend/app/player/web/WebLivePlayer.ts

@@ -42,6 +42,7 @@ export default class WebLivePlayer extends WebPlayer {
       this.screen,
       config,
       wpState,
+      (id) => this.messageManager.getNode(id),
       uiErrorHandler,
     )
     this.assistManager.connect(session.agentToken!, agentId, projectId)

frontend/app/player/web/assist/AssistManager.ts

@@ -1,19 +1,16 @@
+import MessageManager from 'Player/web/MessageManager';
 import type { Socket } from 'socket.io-client';
 import type Screen from '../Screen/Screen';
-import type { Store } from '../../common/types'
+import type { Store } from '../../common/types';
 import type { Message } from '../messages';
 import MStreamReader from '../messages/MStreamReader';
-import JSONRawMessageReader from '../messages/JSONRawMessageReader'
+import JSONRawMessageReader from '../messages/JSONRawMessageReader';
 import Call, { CallingState } from './Call';
-import RemoteControl, { RemoteControlStatus } from './RemoteControl'
+import RemoteControl, { RemoteControlStatus } from './RemoteControl';
-import ScreenRecording,  { SessionRecordingStatus } from './ScreenRecording'
+import ScreenRecording, { SessionRecordingStatus } from './ScreenRecording';
+import CanvasReceiver from 'Player/web/assist/CanvasReceiver';
 
-
+export { RemoteControlStatus, SessionRecordingStatus, CallingState };
-export {
-  RemoteControlStatus,
-  SessionRecordingStatus,
-  CallingState,
-}
 
 export enum ConnectionStatus {
   Connecting,
@@ -25,29 +22,30 @@ export enum ConnectionStatus {
   Closed,
 }
 
-type StatsEvent = 's_call_started'
+type StatsEvent =
+  | 's_call_started'
   | 's_call_ended'
   | 's_control_started'
   | 's_control_ended'
   | 's_recording_started'
-  | 's_recording_ended'
+  | 's_recording_ended';
 
 export function getStatusText(status: ConnectionStatus): string {
   switch (status) {
     case ConnectionStatus.Closed:
       return 'Closed...';
     case ConnectionStatus.Connecting:
-      return "Connecting...";
+      return 'Connecting...';
     case ConnectionStatus.Connected:
-      return "";
+      return '';
     case ConnectionStatus.Inactive:
-      return "Client tab is inactive";
+      return 'Client tab is inactive';
     case ConnectionStatus.Disconnected:
-      return "Disconnected";
+      return 'Disconnected';
     case ConnectionStatus.Error:
-      return "Something went wrong. Try to reload the page.";
+      return 'Something went wrong. Try to reload the page.';
     case ConnectionStatus.WaitingMessages:
-      return "Connected. Waiting for the data... (The tab might be inactive)"
+      return 'Connected. Waiting for the data... (The tab might be inactive)';
   }
 }
 
@@ -59,14 +57,16 @@ export function getStatusText(status: ConnectionStatus): string {
 const MAX_RECONNECTION_COUNT = 4;
 
 export default class AssistManager {
-  assistVersion = 1
+  assistVersion = 1;
+  private canvasReceiver: CanvasReceiver;
   static readonly INITIAL_STATE = {
     peerConnectionStatus: ConnectionStatus.Connecting,
     assistStart: 0,
     ...Call.INITIAL_STATE,
     ...RemoteControl.INITIAL_STATE,
     ...ScreenRecording.INITIAL_STATE,
-  }
+  };
+
   // TODO: Session type
   constructor(
     private session: any,
@@ -75,26 +75,31 @@ export default class AssistManager {
     private screen: Screen,
     private config: RTCIceServer[] | null,
     private store: Store<typeof AssistManager.INITIAL_STATE>,
-    public readonly uiErrorHandler?: { error: (msg: string) => void }
+    private getNode: MessageManager['getNode'],
+    public readonly uiErrorHandler?: {
+      error: (msg: string) => void;
+    }
   ) {}
 
-  public getAssistVersion = () => this.assistVersion
+  public getAssistVersion = () => this.assistVersion;
 
   private get borderStyle() {
-    const { recordingState, remoteControl } = this.store.get()
+    const { recordingState, remoteControl } = this.store.get();
 
-    const isRecordingActive = recordingState === SessionRecordingStatus.Recording
+    const isRecordingActive = recordingState === SessionRecordingStatus.Recording;
-    const isControlActive = remoteControl === RemoteControlStatus.Enabled
+    const isControlActive = remoteControl === RemoteControlStatus.Enabled;
     // recording gets priority here
-    if (isRecordingActive) return { outline: '2px dashed red' }
+    if (isRecordingActive) return { outline: '2px dashed red' };
-    if (isControlActive) return { outline: '2px dashed blue' }
+    if (isControlActive) return { outline: '2px dashed blue' };
-    return { outline: 'unset' }
+    return { outline: 'unset' };
   }
 
   private setStatus(status: ConnectionStatus) {
-    if (this.store.get().peerConnectionStatus === ConnectionStatus.Disconnected &&
+    if (
-      status !== ConnectionStatus.Connected) {
+      this.store.get().peerConnectionStatus === ConnectionStatus.Disconnected &&
-      return
+      status !== ConnectionStatus.Connected
+    ) {
+      return;
     }
 
     if (status === ConnectionStatus.Connecting) {
@@ -111,145 +116,154 @@ export default class AssistManager {
   }
 
   private get peerID(): string {
-    return `${this.session.projectKey}-${this.session.sessionId}`
+    return `${this.session.projectKey}-${this.session.sessionId}`;
   }
 
-  private socketCloseTimeout: ReturnType<typeof setTimeout> | undefined
+  private socketCloseTimeout: ReturnType<typeof setTimeout> | undefined;
   private onVisChange = () => {
-    this.socketCloseTimeout && clearTimeout(this.socketCloseTimeout)
+    this.socketCloseTimeout && clearTimeout(this.socketCloseTimeout);
     if (document.hidden) {
       this.socketCloseTimeout = setTimeout(() => {
-        const state = this.store.get()
+        const state = this.store.get();
-        if (document.hidden &&
+        if (
+          document.hidden &&
           // TODO: should it be RemoteControlStatus.Disabled? (check)
-          (state.calling === CallingState.NoCall && state.remoteControl === RemoteControlStatus.Enabled)) {
+          state.calling === CallingState.NoCall &&
-          this.socket?.close()
+          state.remoteControl === RemoteControlStatus.Enabled
+        ) {
+          this.socket?.close();
         }
-      }, 30000)
+      }, 30000);
     } else {
-      this.socket?.open()
+      this.socket?.open();
-    }
     }
+  };
+
+  private socket: Socket | null = null;
+  private disconnectTimeout: ReturnType<typeof setTimeout> | undefined;
+  private inactiveTimeout: ReturnType<typeof setTimeout> | undefined;
+  private inactiveTabs: string[] = [];
 
-  private socket: Socket | null = null
-  private disconnectTimeout: ReturnType<typeof setTimeout> | undefined
-  private inactiveTimeout: ReturnType<typeof setTimeout> | undefined
-  private inactiveTabs: string[] = []
   private clearDisconnectTimeout() {
-    this.disconnectTimeout && clearTimeout(this.disconnectTimeout)
+    this.disconnectTimeout && clearTimeout(this.disconnectTimeout);
-    this.disconnectTimeout = undefined
+    this.disconnectTimeout = undefined;
   }
-  private clearInactiveTimeout() {
-    this.inactiveTimeout && clearTimeout(this.inactiveTimeout)
-    this.inactiveTimeout = undefined
-  }
-  connect(agentToken: string, agentId: number, projectId: number) {
-    const jmr = new JSONRawMessageReader()
-    const reader = new MStreamReader(jmr, this.session.startedAt)
-    let waitingForMessages = true
 
-    const now = +new Date()
+  private clearInactiveTimeout() {
-    this.store.update({ assistStart: now })
+    this.inactiveTimeout && clearTimeout(this.inactiveTimeout);
+    this.inactiveTimeout = undefined;
+  }
+
+  connect(agentToken: string, agentId: number, projectId: number) {
+    const jmr = new JSONRawMessageReader();
+    const reader = new MStreamReader(jmr, this.session.startedAt);
+    let waitingForMessages = true;
+
+    const now = +new Date();
+    this.store.update({ assistStart: now });
 
     // @ts-ignore
     import('socket.io-client').then(({ default: io }) => {
-      if (this.socket != null || this.cleaned) { return }
+      if (this.socket != null || this.cleaned) {
+        return;
+      }
       // @ts-ignore
-      const urlObject = new URL(window.env.API_EDP || window.location.origin) // does it handle ssl automatically?
+      const urlObject = new URL(window.env.API_EDP || window.location.origin); // does it handle ssl automatically?
 
-      const socket: Socket = this.socket = io(urlObject.origin, {
+      const socket: Socket = (this.socket = io(urlObject.origin, {
         withCredentials: true,
         multiplex: true,
         transports: ['websocket'],
         path: '/ws-assist/socket',
         auth: {
-          token: agentToken
+          token: agentToken,
         },
         query: {
           peerId: this.peerID,
           projectId,
-          identity: "agent",
+          identity: 'agent',
           agentInfo: JSON.stringify({
             ...this.session.agentInfo,
             id: agentId,
+            peerId: this.peerID,
             query: document.location.search,
-          })
+          }),
-        }
+        },
-      })
+      }));
-      socket.on("connect", () => {
+      socket.on('connect', () => {
-        waitingForMessages = true
+        waitingForMessages = true;
-        this.setStatus(ConnectionStatus.WaitingMessages) // TODO: reconnect happens frequently on bad network
+        this.setStatus(ConnectionStatus.WaitingMessages); // TODO: reconnect happens frequently on bad network
-      })
+      });
 
-      socket.on('messages', messages => {
+      socket.on('messages', (messages) => {
-        const isOldVersion = messages.meta.version === 1
+        const isOldVersion = messages.meta.version === 1;
-        this.assistVersion = isOldVersion ? 1 : 2
+        this.assistVersion = isOldVersion ? 1 : 2;
 
-        const data = messages.data || messages
+        const data = messages.data || messages;
-        jmr.append(data) // as RawMessage[]
+        jmr.append(data); // as RawMessage[]
         if (waitingForMessages) {
-          waitingForMessages = false // TODO: more explicit
+          waitingForMessages = false; // TODO: more explicit
-          this.setStatus(ConnectionStatus.Connected)
+          this.setStatus(ConnectionStatus.Connected);
         }
         if (messages.meta.tabId !== this.store.get().currentTab) {
-          this.clearDisconnectTimeout()
+          this.clearDisconnectTimeout();
           if (isOldVersion) {
-            reader.currentTab = messages.meta.tabId
+            reader.currentTab = messages.meta.tabId;
-            this.store.update({ currentTab: messages.meta.tabId })
+            this.store.update({ currentTab: messages.meta.tabId });
           }
         }
 
         for (let msg = reader.readNext(); msg !== null; msg = reader.readNext()) {
-          this.handleMessage(msg, msg._index)
+          this.handleMessage(msg, msg._index);
         }
-      })
+      });
 
       socket.on('SESSION_RECONNECTED', () => {
-        this.clearDisconnectTimeout()
+        this.clearDisconnectTimeout();
-        this.clearInactiveTimeout()
+        this.clearInactiveTimeout();
-        this.setStatus(ConnectionStatus.Connected)
+        this.setStatus(ConnectionStatus.Connected);
-      })
+      });
 
       socket.on('UPDATE_SESSION', (evData) => {
-        const { meta = {}, data = {} } = evData
+        const { meta = {}, data = {} } = evData;
-        const { tabId } = meta
+        const { tabId } = meta;
-        const usedData = this.assistVersion === 1 ? evData : data
+        const usedData = this.assistVersion === 1 ? evData : data;
-        const { active } = usedData
+        const { active } = usedData;
-        const currentTab = this.store.get().currentTab
+        const currentTab = this.store.get().currentTab;
-        this.clearDisconnectTimeout()
+        this.clearDisconnectTimeout();
-        !this.inactiveTimeout && this.setStatus(ConnectionStatus.Connected)
+        !this.inactiveTimeout && this.setStatus(ConnectionStatus.Connected);
-        if (typeof active === "boolean") {
+        if (typeof active === 'boolean') {
-          this.clearInactiveTimeout()
+          this.clearInactiveTimeout();
           if (active) {
-            this.setStatus(ConnectionStatus.Connected)
+            this.setStatus(ConnectionStatus.Connected);
-            this.inactiveTabs = this.inactiveTabs.filter(t => t !== tabId)
+            this.inactiveTabs = this.inactiveTabs.filter((t) => t !== tabId);
           } else {
             if (!this.inactiveTabs.includes(tabId)) {
-              this.inactiveTabs.push(tabId)
+              this.inactiveTabs.push(tabId);
             }
             if (tabId === undefined || tabId === currentTab) {
               this.inactiveTimeout = setTimeout(() => {
                 // @ts-ignore
-                const tabs = this.store.get().tabs
+                const tabs = this.store.get().tabs;
                 if (this.inactiveTabs.length === tabs.size) {
-                  this.setStatus(ConnectionStatus.Inactive)
+                  this.setStatus(ConnectionStatus.Inactive);
                 }
-              }, 10000)
+              }, 10000);
             }
           }
         }
-      })
+      });
-      socket.on('SESSION_DISCONNECTED', e => {
+      socket.on('SESSION_DISCONNECTED', (e) => {
-        waitingForMessages = true
+        waitingForMessages = true;
-        this.clearDisconnectTimeout()
+        this.clearDisconnectTimeout();
         this.disconnectTimeout = setTimeout(() => {
-          this.setStatus(ConnectionStatus.Disconnected)
+          this.setStatus(ConnectionStatus.Disconnected);
-        }, 30000)
+        }, 30000);
-      })
+      });
-      socket.on('error', e => {
+      socket.on('error', (e) => {
-        console.warn("Socket error: ", e )
+        console.warn('Socket error: ', e);
         this.setStatus(ConnectionStatus.Error);
-      })
+      });
 
       // Maybe  do lazy initialization for all?
       // TODO: socket proxy (depend on interfaces)
@@ -259,88 +273,93 @@ export default class AssistManager {
         this.config,
         this.peerID,
         this.getAssistVersion
-      )
+      );
       this.remoteControl = new RemoteControl(
         this.store,
         socket,
         this.screen,
         this.session.agentInfo,
         () => this.screen.setBorderStyle(this.borderStyle),
-        this.getAssistVersion,
+        this.getAssistVersion
-      )
+      );
       this.screenRecording = new ScreenRecording(
         this.store,
         socket,
         this.session.agentInfo,
         () => this.screen.setBorderStyle(this.borderStyle),
         this.uiErrorHandler,
-        this.getAssistVersion,
+        this.getAssistVersion
-      )
+      );
+      this.canvasReceiver = new CanvasReceiver(this.peerID, this.config, this.getNode, {
+            ...this.session.agentInfo,
+            id: agentId,
+          });
 
-      document.addEventListener('visibilitychange', this.onVisChange)
+      document.addEventListener('visibilitychange', this.onVisChange);
-    })
+    });
   }
 
   public ping(event: StatsEvent, id: number) {
-    this.socket?.emit(event, id)
+    this.socket?.emit(event, id);
   }
 
-
   /* ==== ScreenRecording ==== */
-  private screenRecording: ScreenRecording | null = null
+  private screenRecording: ScreenRecording | null = null;
   requestRecording = (...args: Parameters<ScreenRecording['requestRecording']>) => {
-    return this.screenRecording?.requestRecording(...args)
+    return this.screenRecording?.requestRecording(...args);
-  }
+  };
   stopRecording = (...args: Parameters<ScreenRecording['stopRecording']>) => {
-    return this.screenRecording?.stopRecording(...args)
+    return this.screenRecording?.stopRecording(...args);
-  }
+  };
-
 
   /* ==== RemoteControl ==== */
-  private remoteControl: RemoteControl | null = null
+  private remoteControl: RemoteControl | null = null;
-  requestReleaseRemoteControl = (...args: Parameters<RemoteControl['requestReleaseRemoteControl']>) => {
+  requestReleaseRemoteControl = (
-    return this.remoteControl?.requestReleaseRemoteControl(...args)
+    ...args: Parameters<RemoteControl['requestReleaseRemoteControl']>
-  }
+  ) => {
+    return this.remoteControl?.requestReleaseRemoteControl(...args);
+  };
   setRemoteControlCallbacks = (...args: Parameters<RemoteControl['setCallbacks']>) => {
-    return this.remoteControl?.setCallbacks(...args)
+    return this.remoteControl?.setCallbacks(...args);
-  }
+  };
   releaseRemoteControl = (...args: Parameters<RemoteControl['releaseRemoteControl']>) => {
-    return this.remoteControl?.releaseRemoteControl(...args)
+    return this.remoteControl?.releaseRemoteControl(...args);
-  }
+  };
   toggleAnnotation = (...args: Parameters<RemoteControl['toggleAnnotation']>) => {
-    return this.remoteControl?.toggleAnnotation(...args)
+    return this.remoteControl?.toggleAnnotation(...args);
-  }
+  };
 
   /* ==== Call  ==== */
-  private callManager: Call | null = null
+  private callManager: Call | null = null;
   initiateCallEnd = async (...args: Parameters<Call['initiateCallEnd']>) => {
-    return this.callManager?.initiateCallEnd(...args)
+    return this.callManager?.initiateCallEnd(...args);
-  }
+  };
   setCallArgs = (...args: Parameters<Call['setCallArgs']>) => {
-    return this.callManager?.setCallArgs(...args)
+    return this.callManager?.setCallArgs(...args);
-  }
+  };
   call = (...args: Parameters<Call['call']>) => {
-    return this.callManager?.call(...args)
+    return this.callManager?.call(...args);
-  }
+  };
   toggleVideoLocalStream = (...args: Parameters<Call['toggleVideoLocalStream']>) => {
-    return this.callManager?.toggleVideoLocalStream(...args)
+    return this.callManager?.toggleVideoLocalStream(...args);
-  }
+  };
   addPeerCall = (...args: Parameters<Call['addPeerCall']>) => {
-    return this.callManager?.addPeerCall(...args)
+    return this.callManager?.addPeerCall(...args);
-  }
+  };
-
 
   /* ==== Cleaning ==== */
-  private cleaned = false
+  private cleaned = false;
+
   clean() {
-    this.cleaned = true // sometimes cleaned before modules loaded
+    this.cleaned = true; // sometimes cleaned before modules loaded
-    this.remoteControl?.clean()
+    this.remoteControl?.clean();
-    this.callManager?.clean()
+    this.callManager?.clean();
-    this.socket?.close()
+    this.canvasReceiver?.clear();
-    this.socket = null
+    this.socket?.close();
-    this.clearDisconnectTimeout()
+    this.socket = null;
-    this.clearInactiveTimeout()
+    this.clearDisconnectTimeout();
-    this.socketCloseTimeout && clearTimeout(this.socketCloseTimeout)
+    this.clearInactiveTimeout();
-    document.removeEventListener('visibilitychange', this.onVisChange)
+    this.socketCloseTimeout && clearTimeout(this.socketCloseTimeout);
+    document.removeEventListener('visibilitychange', this.onVisChange);
   }
 }

frontend/app/player/web/assist/Call.ts

@@ -32,7 +32,7 @@ export default class Call {
   private videoStreams: Record<string, MediaStreamTrack> = {};
 
   constructor(
-    private store: Store<State>,
+    private store: Store<State & { tabs: Set<string> }>,
     private socket: Socket,
     private config: RTCIceServer[] | null,
     private peerID: string,
@@ -253,7 +253,7 @@ export default class Call {
       this.getAssistVersion() === 1
         ? this.peerID
         : `${this.peerID}-${tab || Object.keys(this.store.get().tabs)[0]}`;
-    console.log(peerId, this.getAssistVersion());
+
     void this._peerConnection(peerId);
     this.emitData('_agent_name', appStore.getState().getIn(['user', 'account', 'name']));
   }

frontend/app/player/web/assist/CanvasReceiver.ts

@@ -0,0 +1,162 @@
+import Peer from 'peerjs';
+import { VElement } from 'Player/web/managers/DOM/VirtualDOM';
+import MessageManager from 'Player/web/MessageManager';
+
+let frameCounter = 0;
+
+function draw(
+  video: HTMLVideoElement,
+  canvas: HTMLCanvasElement,
+  canvasCtx: CanvasRenderingContext2D
+) {
+  if (frameCounter % 4 === 0) {
+    canvasCtx.drawImage(video, 0, 0, canvas.width, canvas.height);
+  }
+  frameCounter++;
+  requestAnimationFrame(() => draw(video, canvas, canvasCtx));
+}
+
+export default class CanvasReceiver {
+  private streams: Map<string, MediaStream> = new Map();
+  private peer: Peer | null = null;
+
+  constructor(
+    private readonly peerIdPrefix: string,
+    private readonly config: RTCIceServer[] | null,
+    private readonly getNode: MessageManager['getNode'],
+    private readonly agentInfo: Record<string, any>
+  ) {
+    // @ts-ignore
+    const urlObject = new URL(window.env.API_EDP || window.location.origin);
+    const peerOpts: Peer.PeerJSOption = {
+      host: urlObject.hostname,
+      path: '/assist',
+      port:
+        urlObject.port === ''
+          ? location.protocol === 'https:'
+            ? 443
+            : 80
+          : parseInt(urlObject.port),
+    };
+    if (this.config) {
+      peerOpts['config'] = {
+        iceServers: this.config,
+        //@ts-ignore
+        sdpSemantics: 'unified-plan',
+        iceTransportPolicy: 'all',
+      };
+    }
+    const id = `${this.peerIdPrefix}-${this.agentInfo.id}-canvas`;
+    const canvasPeer = new Peer(id, peerOpts);
+    this.peer = canvasPeer;
+    canvasPeer.on('error', (err) => console.error('canvas peer error', err));
+    canvasPeer.on('call', (call) => {
+      call.answer();
+      const canvasId = call.peer.split('-')[2];
+      call.on('stream', (stream) => {
+        this.streams.set(canvasId, stream);
+        setTimeout(() => {
+          const node = this.getNode(parseInt(canvasId, 10));
+          const videoEl = spawnVideo(
+            this.streams.get(canvasId)?.clone() as MediaStream,
+            node as VElement
+          );
+          if (node) {
+            draw(
+              videoEl,
+              node.node as HTMLCanvasElement,
+              (node.node as HTMLCanvasElement).getContext('2d') as CanvasRenderingContext2D
+            );
+          }
+        }, 500);
+      });
+      call.on('error', (err) => console.error('canvas call error', err));
+    });
+  }
+
+  clear() {
+    if (this.peer) {
+      // otherwise it calls reconnection on data chan close
+      const peer = this.peer;
+      this.peer = null;
+      peer.disconnect();
+      peer.destroy();
+    }
+  }
+}
+
+function spawnVideo(stream: MediaStream, node: VElement) {
+  const videoEl = document.createElement('video');
+
+  videoEl.srcObject = stream
+  videoEl.setAttribute('autoplay', 'true');
+  videoEl.setAttribute('muted', 'true');
+  videoEl.setAttribute('playsinline', 'true');
+  videoEl.setAttribute('crossorigin', 'anonymous');
+  void videoEl.play();
+
+  return videoEl;
+}
+
+function spawnDebugVideo(stream: MediaStream, node: VElement) {
+  const video = document.createElement('video');
+  video.id = 'canvas-or-testing';
+  video.style.border = '1px solid red';
+  video.setAttribute('autoplay', 'true');
+  video.setAttribute('muted', 'true');
+  video.setAttribute('playsinline', 'true');
+  video.setAttribute('crossorigin', 'anonymous');
+
+  const coords = node.node.getBoundingClientRect();
+
+  Object.assign(video.style, {
+    position: 'absolute',
+    left: `${coords.left}px`,
+    top: `${coords.top}px`,
+    width: `${coords.width}px`,
+    height: `${coords.height}px`,
+  });
+  video.width = coords.width;
+  video.height = coords.height;
+  video.srcObject = stream;
+
+  document.body.appendChild(video);
+  video
+    .play()
+    .then(() => {
+      console.log('started streaming canvas');
+    })
+    .catch((e) => {
+      console.error(e);
+      const waiter = () => {
+        void video.play();
+        document.removeEventListener('click', waiter);
+      };
+      document.addEventListener('click', waiter);
+    });
+}
+
+/** simple peer example
+ * // @ts-ignore
+ *     const peer = new SLPeer({ initiator: false })
+ *     socket.on('c_signal', ({ data }) => {
+ *       console.log('got signal', data)
+ *       peer.signal(data.data);
+ *       peer.canvasId = data.id;
+ *     });
+ *
+ *     peer.on('signal', (data: any) => {
+ *       socket.emit('c_signal', data);
+ *     });
+ *     peer.on('stream', (stream: MediaStream) => {
+ *       console.log('stream ready', stream, peer.canvasId);
+ *       this.streams.set(peer.canvasId, stream)
+ *       setTimeout(() => {
+ *         const node = this.getNode(peer.canvasId)
+ *         console.log(peer.canvasId, this.streams, node)
+ *         spawnVideo(this.streams.get(peer.canvasId)?.clone(), node, this.screen)
+ *       }, 500)
+ *     })
+ *     peer.on('error', console.error)
+ *
+ * */

frontend/app/player/web/managers/CanvasManager.ts

@@ -0,0 +1,53 @@
+import { VElement } from "Player/web/managers/DOM/VirtualDOM";
+
+export default class CanvasManager {
+  private fileData: string | undefined;
+  private canvasEl: HTMLVideoElement
+  private canvasCtx: CanvasRenderingContext2D | null = null;
+  private videoTag = document.createElement('video')
+  private lastTs = 0;
+
+  constructor(
+    private readonly nodeId: string,
+    private readonly delta: number,
+    private readonly filename: string,
+    private readonly getNode: (id: number) => VElement | undefined) {
+      // getting mp4 file composed from canvas snapshot images
+      fetch(this.filename).then((r) => {
+        if (r.status === 200) {
+          r.blob().then((blob) => {
+            this.fileData = URL.createObjectURL(blob);
+          })
+        } else {
+          return Promise.reject(`File ${this.filename} not found`)
+        }
+      }).catch(console.error)
+  }
+
+  startVideo = () => {
+    if (!this.fileData) return;
+    this.videoTag.setAttribute('autoplay', 'true');
+    this.videoTag.setAttribute('muted', 'true');
+    this.videoTag.setAttribute('playsinline', 'true');
+    this.videoTag.setAttribute('crossorigin', 'anonymous');
+    this.videoTag.src = this.fileData;
+    this.videoTag.currentTime = 0;
+
+    const node = this.getNode(parseInt(this.nodeId, 10)) as unknown as VElement
+    this.canvasCtx = (node.node as HTMLCanvasElement).getContext('2d');
+    this.canvasEl = node.node as HTMLVideoElement;
+  }
+
+  move(t: number) {
+    if (t - this.lastTs < 100) return;
+    this.lastTs = t;
+    const playTime = t - this.delta
+    if (playTime > 0) {
+      if (!this.videoTag.paused) {
+          void this.videoTag.pause()
+      }
+      this.videoTag.currentTime = playTime/1000;
+      this.canvasCtx?.drawImage(this.videoTag, 0, 0, this.canvasEl.width, this.canvasEl.height);
+    }
+  }
+}

frontend/app/player/web/managers/DOM/DOMManager.ts

@@ -113,6 +113,10 @@ export default class DOMManager extends ListWalker<Message> {
     return false;
   }
 
+  public getNode(id: number) {
+    return this.vElements.get(id) || this.vTexts.get(id)
+  }
+
   private insertNode({ parentID, id, index }: { parentID: number, id: number, index: number }): void {
     const child = this.vElements.get(id) || this.vTexts.get(id)
     if (!child) {
@@ -208,7 +212,8 @@ export default class DOMManager extends ListWalker<Message> {
         return
       }
       case MType.CreateElementNode: {
-        const vElem = new VElement(msg.tag, msg.svg)
+        // if (msg.tag.toLowerCase() === 'canvas') msg.tag = 'video'
+        const vElem = new VElement(msg.tag, msg.svg, msg.index)
         if (['STYLE', 'style', 'LINK'].includes(msg.tag)) {
           vElem.prioritized = true
         }

frontend/app/player/web/managers/DOM/VirtualDOM.ts

@@ -144,7 +144,7 @@ export class VElement extends VParent<Element> {
 	parentNode: VParent | null = null /** Should be modified only by he parent itself */
 	private newAttributes: Map<string, string | false> = new Map()
 
-	constructor(readonly tagName: string, readonly isSVG = false) { super() }
+	constructor(readonly tagName: string, readonly isSVG = false, public readonly index: number) { super() }
 	protected createNode() {
 		try {
 			return this.isSVG

frontend/app/player/web/managers/PagesManager.ts

@@ -56,6 +56,10 @@ export default class PagesManager extends ListWalker<DOMManager> {
 		this.forEach(page => page.sort(comparator))
 	}
 
+	public getNode(id: number) {
+		return this.currentPage?.getNode(id)
+	}
+
 	moveReady(t: number): Promise<void> {
 		const requiredPage = this.moveGetLast(t)
 		if (requiredPage != null) {

frontend/app/player/web/messages/RawMessageReader.gen.ts

@@ -713,6 +713,16 @@ export default class RawMessageReader extends PrimitiveReader {
       };
     }
 
+    case 119: {
+      const nodeId = this.readString(); if (nodeId === null) { return resetPointer() }
+      const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() }
+      return {
+        tp: MType.CanvasNode,
+        nodeId,
+        timestamp,
+      };
+    }
+
     case 93: {
       const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() }
       const length = this.readUint(); if (length === null) { return resetPointer() }

frontend/app/player/web/messages/filters.gen.ts

@@ -4,7 +4,7 @@
 import { MType } from './raw.gen'
 
 const IOS_TYPES = [90,91,92,93,94,95,96,97,98,100,101,102,103,104,105,106,107,110,111]
-const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,113,114,117,118]
+const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,113,114,117,118,119]
 export function isDOMType(t: MType) {
 	return DOM_TYPES.includes(t)
 }

frontend/app/player/web/messages/message.gen.ts

@@ -61,6 +61,7 @@ import type {
   RawResourceTiming,
   RawTabChange,
   RawTabData,
+  RawCanvasNode,
   RawIosEvent,
   RawIosScreenChanges,
   RawIosClickEvent,
@@ -190,6 +191,8 @@ export type TabChange = RawTabChange & Timed
 
 export type TabData = RawTabData & Timed
 
+export type CanvasNode = RawCanvasNode & Timed
+
 export type IosEvent = RawIosEvent & Timed
 
 export type IosScreenChanges = RawIosScreenChanges & Timed

frontend/app/player/web/messages/raw.gen.ts

@@ -59,6 +59,7 @@ export const enum MType {
   ResourceTiming = 116,
   TabChange = 117,
   TabData = 118,
+  CanvasNode = 119,
   IosEvent = 93,
   IosScreenChanges = 96,
   IosClickEvent = 100,
@@ -476,6 +477,12 @@ export interface RawTabData {
   tabId: string,
 }
 
+export interface RawCanvasNode {
+  tp: MType.CanvasNode,
+  nodeId: string,
+  timestamp: number,
+}
+
 export interface RawIosEvent {
   tp: MType.IosEvent,
   timestamp: number,
@@ -568,4 +575,4 @@ export interface RawIosIssueEvent {
 }
 
 
-export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequestDeprecated | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTimingDeprecated | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawNetworkRequest | RawSelectionChange | RawMouseThrashing | RawResourceTiming | RawTabChange | RawTabData | RawIosEvent | RawIosScreenChanges | RawIosClickEvent | RawIosInputEvent | RawIosPerformanceEvent | RawIosLog | RawIosInternalError | RawIosNetworkCall | RawIosSwipeEvent | RawIosIssueEvent;
+export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequestDeprecated | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTimingDeprecated | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawNetworkRequest | RawSelectionChange | RawMouseThrashing | RawResourceTiming | RawTabChange | RawTabData | RawCanvasNode | RawIosEvent | RawIosScreenChanges | RawIosClickEvent | RawIosInputEvent | RawIosPerformanceEvent | RawIosLog | RawIosInternalError | RawIosNetworkCall | RawIosSwipeEvent | RawIosIssueEvent;

frontend/app/player/web/messages/tracker-legacy.gen.ts

@@ -60,6 +60,7 @@ export const TP_MAP = {
   116: MType.ResourceTiming,
   117: MType.TabChange,
   118: MType.TabData,
+  119: MType.CanvasNode,
   93: MType.IosEvent,
   96: MType.IosScreenChanges,
   100: MType.IosClickEvent,

frontend/app/player/web/messages/tracker.gen.ts

@@ -493,8 +493,14 @@ type TrTabData = [
   tabId: string,
 ]
 
+type TrCanvasNode = [
+  type: 119,
+  nodeId: string,
+  timestamp: number,
+]
 
-export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequestDeprecated | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTimingDeprecated | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrNetworkRequest | TrInputChange | TrSelectionChange | TrMouseThrashing | TrUnbindNodes | TrResourceTiming | TrTabChange | TrTabData
+
+export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequestDeprecated | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTimingDeprecated | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrNetworkRequest | TrInputChange | TrSelectionChange | TrMouseThrashing | TrUnbindNodes | TrResourceTiming | TrTabChange | TrTabData | TrCanvasNode
 
 export default function translate(tMsg: TrackerMessage): RawMessage | null {
   switch(tMsg[0]) {
@@ -992,6 +998,14 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null {
       }
     }
     
+    case 119: {
+      return {
+        tp: MType.CanvasNode,
+        nodeId: tMsg[1],
+        timestamp: tMsg[2],
+      }
+    }
+    
     default:
       return null
   }

frontend/app/types/session/session.ts

@@ -79,6 +79,7 @@ export interface ISession {
   metadata: [];
   favorite: boolean;
   filterId?: string;
+  canvasURL: string[];
   domURL: string[];
   devtoolsURL: string[];
   /**
@@ -148,6 +149,7 @@ const emptyValues = {
   devtoolsURL: [],
   mobsUrl: [],
   notes: [],
+  canvasURL: [],
   metadata: {},
   startedAt: 0,
   platform: 'web',
@@ -160,6 +162,7 @@ export default class Session {
   siteId: ISession['siteId'];
   projectKey: ISession['projectKey'];
   peerId: ISession['peerId'];
+  canvasURL: ISession['canvasURL'];
   live: ISession['live'];
   startedAt: ISession['startedAt'];
   duration: ISession['duration'];
@@ -234,6 +237,7 @@ export default class Session {
       mobsUrl = [],
       crashes = [],
       notes = [],
+      canvasURL = [],
       ...session
     } = sessionData;
     const duration = Duration.fromMillis(session.duration < 1000 ? 1000 : session.duration);
@@ -325,6 +329,7 @@ export default class Session {
       domURL,
       devtoolsURL,
       notes,
+      canvasURL,
       notesWithEvents: mixedEventsWithIssues,
       frustrations: frustrationList,
     });

frontend/app/window.d.ts

@@ -0,0 +1,14 @@
+declare global {
+  interface Window {
+    env: {
+      NODE_ENV: string
+      ORIGIN: string
+      ASSETS_HOST: string
+      API_EDP: string
+      VERSION: string
+      TRACKER_VERSION: string
+      TRACKER_HOST: string
+      SOURCEMAP: boolean
+    }
+  }
+}

frontend/package.json

@@ -25,6 +25,7 @@
   },
   "dependencies": {
     "@ant-design/icons": "^5.2.5",
+    "@babel/plugin-transform-private-methods": "^7.23.3",
     "@floating-ui/react-dom-interactions": "^0.10.3",
     "@sentry/browser": "^5.21.1",
     "@svg-maps/world": "^1.0.1",
@@ -89,7 +90,7 @@
     "@babel/plugin-proposal-decorators": "^7.23.2",
     "@babel/plugin-proposal-private-methods": "^7.18.6",
     "@babel/plugin-syntax-bigint": "^7.8.3",
-    "@babel/plugin-transform-class-properties": "^7.22.5",
+    "@babel/plugin-transform-class-properties": "^7.23.3",
     "@babel/plugin-transform-private-property-in-object": "^7.22.11",
     "@babel/plugin-transform-runtime": "^7.17.12",
     "@babel/preset-env": "^7.23.2",

frontend/yarn.lock

@@ -1626,6 +1626,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@babel/plugin-transform-class-properties@npm:^7.23.3":
+  version: 7.23.3
+  resolution: "@babel/plugin-transform-class-properties@npm:7.23.3"
+  dependencies:
+    "@babel/helper-create-class-features-plugin": ^7.22.15
+    "@babel/helper-plugin-utils": ^7.22.5
+  peerDependencies:
+    "@babel/core": ^7.0.0-0
+  checksum: bca30d576f539eef216494b56d610f1a64aa9375de4134bc021d9660f1fa735b1d7cc413029f22abc0b7cb737e3a57935c8ae9d8bd1730921ccb1deebce51bfd
+  languageName: node
+  linkType: hard
+
 "@babel/plugin-transform-class-static-block@npm:^7.22.11":
   version: 7.22.11
   resolution: "@babel/plugin-transform-class-static-block@npm:7.22.11"
@@ -2227,6 +2239,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@babel/plugin-transform-private-methods@npm:^7.23.3":
+  version: 7.23.3
+  resolution: "@babel/plugin-transform-private-methods@npm:7.23.3"
+  dependencies:
+    "@babel/helper-create-class-features-plugin": ^7.22.15
+    "@babel/helper-plugin-utils": ^7.22.5
+  peerDependencies:
+    "@babel/core": ^7.0.0-0
+  checksum: 745a655edcd111b7f91882b921671ca0613079760d8c9befe336b8a9bc4ce6bb49c0c08941831c950afb1b225b4b2d3eaac8842e732db095b04db38efd8c34f4
+  languageName: node
+  linkType: hard
+
 "@babel/plugin-transform-private-property-in-object@npm:^7.22.11":
   version: 7.22.11
   resolution: "@babel/plugin-transform-private-property-in-object@npm:7.22.11"
@@ -19630,7 +19654,8 @@ __metadata:
     "@babel/plugin-proposal-decorators": ^7.23.2
     "@babel/plugin-proposal-private-methods": ^7.18.6
     "@babel/plugin-syntax-bigint": ^7.8.3
-    "@babel/plugin-transform-class-properties": ^7.22.5
+    "@babel/plugin-transform-class-properties": ^7.23.3
+    "@babel/plugin-transform-private-methods": ^7.23.3
     "@babel/plugin-transform-private-property-in-object": ^7.22.11
     "@babel/plugin-transform-runtime": ^7.17.12
     "@babel/preset-env": ^7.23.2

mobs/messages.rb

@@ -504,6 +504,11 @@ message 118, 'TabData' do
     string 'TabId'
 end
 
+message 119, 'CanvasNode' do
+    string 'NodeId'
+    uint 'Timestamp'
+end
+
 ## Backend-only
 message 125, 'IssueEvent', :replayer => false, :tracker => false do
   uint 'MessageID'

tracker/.husky/pre-commit

@@ -1,7 +1,7 @@
 #!/usr/bin/env sh
 . "$(dirname -- "$0")/_/husky.sh"
 
-if git diff --cached --name-only | grep --quiet '^tracker/tracker/'
+if git diff --cached --name-only | grep -v '\.gen\.ts$' | grep --quiet '^tracker/tracker/'
 then
   echo "tracker"
   pwd
@@ -12,7 +12,7 @@ then
   cd ../../
 fi
 
-if git diff --cached --name-only | grep --quiet '^tracker/tracker-assist/'
+if git diff --cached --name-only | grep -v '\.gen\.ts$' | grep --quiet '^tracker/tracker-assist/'
 then
   echo "tracker-assist"
   cd tracker/tracker-assist

tracker/tracker-assist/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openreplay/tracker-assist",
   "description": "Tracker plugin for screen assistance through the WebRTC",
-  "version": "6.0.3",
+  "version": "6.0.4-57",
   "keywords": [
     "WebRTC",
     "assistance",
@@ -34,7 +34,7 @@
     "socket.io-client": "^4.7.2"
   },
   "peerDependencies": {
-    "@openreplay/tracker": ">=8.0.0"
+    "@openreplay/tracker": ">=10.0.3"
   },
   "devDependencies": {
     "@openreplay/tracker": "file:../tracker",

tracker/tracker-assist/src/Assist.ts

@@ -1,4 +1,5 @@
 /* eslint-disable @typescript-eslint/no-empty-function */
+import {hasTag,} from '@openreplay/tracker/lib/app/guards'
 import type { Socket, } from 'socket.io-client'
 import { connect, } from 'socket.io-client'
 import Peer, { MediaConnection, } from 'peerjs'
@@ -14,6 +15,7 @@ import { callConfirmDefault, } from './ConfirmWindow/defaults.js'
 import type { Options as ConfirmOptions, } from './ConfirmWindow/defaults.js'
 import ScreenRecordingState from './ScreenRecordingState.js'
 import { pkgVersion, } from './version.js'
+import Canvas from './Canvas.js'
 
 // TODO: fully specified strict check with no-any (everywhere)
 // @ts-ignore
@@ -21,6 +23,14 @@ const safeCastedPeer = Peer.default || Peer
 
 type StartEndCallback = (agentInfo?: Record<string, any>) => ((() => any) | void)
 
+interface AgentInfo {
+  email: string;
+  id: number
+  name: string
+  peerId: string
+  query: string
+}
+
 export interface Options {
   onAgentConnect: StartEndCallback;
   onCallStart: StartEndCallback;
@@ -58,7 +68,7 @@ type OptionalCallback = (()=>Record<string, unknown>) | void
 type Agent = {
   onDisconnect?: OptionalCallback,
   onControlReleased?: OptionalCallback,
-  agentInfo: Record<string, string> | undefined
+  agentInfo: AgentInfo | undefined
   //
 }
 
@@ -68,12 +78,15 @@ export default class Assist {
 
   private socket: Socket | null = null
   private peer: Peer | null = null
+  private canvasPeer: Peer | null = null
   private assistDemandedRestart = false
   private callingState: CallingState = CallingState.False
   private remoteControl: RemoteControl | null = null;
 
   private agents: Record<string, Agent> = {}
   private readonly options: Options
+  private readonly canvasMap: Map<number, Canvas> = new Map()
+
   constructor(
     private readonly app: App,
     options?: Partial<Options>,
@@ -151,13 +164,14 @@ export default class Assist {
     }
     return ''
   }
+
   private onStart() {
     const app = this.app
     const sessionId = app.getSessionID()
     // Common for all incoming call requests
     let callUI: CallWindow | null = null
     let annot: AnnotationCanvas | null = null
-    // TODO: incapsulate
+    // TODO: encapsulate
     let callConfirmWindow: ConfirmWindow | null = null
     let callConfirmAnswer: Promise<boolean> | null = null
     let callEndCallback: ReturnType<StartEndCallback> | null = null
@@ -190,7 +204,7 @@ export default class Assist {
       app.debug.log('Socket:', ...args)
     })
 
-    const onGrand = (id) => {
+    const onGrand = (id: string) => {
       if (!callUI) {
         callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
       }
@@ -203,7 +217,7 @@ export default class Assist {
       annot.mount()
       return callingAgents.get(id)
     }
-    const onRelease = (id, isDenied) => {
+    const onRelease = (id?: string | null, isDenied?: boolean) => {
       {
         if (id) {
           const cb = this.agents[id].onControlReleased
@@ -237,7 +251,7 @@ export default class Assist {
     const onAcceptRecording = () => {
       socket.emit('recording_accepted')
     }
-    const onRejectRecording = (agentData) => {
+    const onRejectRecording = (agentData: AgentInfo) => {
       socket.emit('recording_rejected')
 
       this.options.onRecordingDeny?.(agentData || {})
@@ -276,7 +290,7 @@ export default class Assist {
     socket.on('startAnnotation', (id, event) => processEvent(id, event, (_,  d) => annot?.start(d)))
     socket.on('stopAnnotation', (id, event) => processEvent(id, event, annot?.stop))
 
-    socket.on('NEW_AGENT', (id: string, info) => {
+    socket.on('NEW_AGENT', (id: string, info: AgentInfo) => {
       this.agents[id] = {
         onDisconnect: this.options.onAgentConnect?.(info),
         agentInfo: info, // TODO ?
@@ -386,7 +400,7 @@ export default class Assist {
       host: this.getHost(),
       path: this.getBasePrefixUrl()+'/assist',
       port: location.protocol === 'http:' && this.noSecureMode ? 80 : 443,
-      //debug: appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs.
+      debug: 2, //appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs.
     }
     if (this.options.config) {
       peerOptions['config'] = this.options.config
@@ -547,6 +561,38 @@ export default class Assist {
         app.debug.log(reason)
       })
     })
+
+    app.nodes.attachNodeCallback((node) => {
+      const id = app.nodes.getID(node)
+      if (id && hasTag(node, 'canvas')) {
+        const canvasPId = `${app.getProjectKey()}-${sessionId}-${id}`
+        if (!this.canvasPeer) this.canvasPeer = new safeCastedPeer(canvasPId, peerOptions) as Peer
+        const canvasHandler = new Canvas(
+          node as unknown as HTMLCanvasElement,
+          id,
+          30,
+          (stream: MediaStream) => {
+            Object.values(this.agents).forEach(agent => {
+              if (agent.agentInfo) {
+                const target = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas`
+                const connection = this.canvasPeer?.connect(target)
+                connection?.on('open', () => {
+                  if (agent.agentInfo) {
+                    const pCall = this.canvasPeer?.call(target, stream)
+                    pCall?.on('error', app.debug.error)
+                  }
+                })
+                connection?.on('error', app.debug.error)
+                this.canvasPeer?.on('error', app.debug.error)
+              } else {
+                app.debug.error('Assist: cant establish canvas peer to agent, no agent info')
+              }
+            })
+          },
+        )
+        this.canvasMap.set(id, canvasHandler)
+      }
+    })
   }
 
   private playNotificationSound() {
@@ -572,3 +618,16 @@ export default class Assist {
     }
   }
 }
+
+/** simple peers impl
+ * const slPeer = new SLPeer({ initiator: true, stream: stream, })
+ *               // slPeer.on('signal', (data: any) => {
+ *               //   this.emit('c_signal', { data, id, })
+ *               // })
+ *               // this.socket?.on('c_signal', (tab: string, data: any) => {
+ *               //   console.log(data)
+ *               //   slPeer.signal(data)
+ *               // })
+ *               // slPeer.on('error', console.error)
+ *               // this.emit('canvas_stream', { canvasId, })
+ * */

tracker/tracker-assist/src/Canvas.ts

@@ -0,0 +1,61 @@
+
+export default class CanvasRecorder {
+  stream: MediaStream | null;
+
+  constructor(
+    private readonly canvas: HTMLCanvasElement,
+    private readonly canvasId: number,
+    private readonly fps: number,
+    private readonly onStream: (stream: MediaStream) => void
+  ) {
+    this.canvas.getContext('2d', { alpha: true, })
+    const stream = this.canvas.captureStream(this.fps)
+    this.emitStream(stream)
+  }
+
+  restart() {
+    // this.stop()
+    const stream = this.canvas.captureStream(this.fps)
+    this.stream = stream
+    this.emitStream(stream)
+  }
+
+  toggleLocal(stream: MediaStream) {
+    const possibleVideoEl = document.getElementById('canvas-or-testing')
+    if (possibleVideoEl) {
+      document.body.removeChild(possibleVideoEl)
+    }
+    const video = document.createElement('video')
+    video.width = 520
+    video.height = 400
+    video.id = 'canvas-or-testing'
+    video.setAttribute('autoplay', 'true')
+    video.setAttribute('muted', 'true')
+    video.setAttribute('playsinline', 'true')
+    video.crossOrigin = 'anonymous'
+    document.body.appendChild(video)
+
+    video.srcObject = stream
+
+    void video.play()
+    video.addEventListener('error', (e) => {
+      console.error('Video error:', e)
+    })
+  }
+
+  emitStream(stream?: MediaStream) {
+    if (stream) {
+      return this.onStream(stream)
+    }
+    if (this.stream) {
+      this.onStream(this.stream)
+    } else {
+      console.error('no stream for canvas', this.canvasId)
+    }
+  }
+
+  stop() {
+    this.stream?.getTracks().forEach((track) => track.stop())
+    this.stream = null
+  }
+}

tracker/tracker-assist/src/version.ts

@@ -1 +1 @@
-export const pkgVersion = '6.0.3'
+export const pkgVersion = '6.0.4-57'

tracker/tracker/package.json

@@ -1,14 +1,15 @@
 {
   "name": "@openreplay/tracker",
   "description": "The OpenReplay tracker main package",
-  "version": "10.0.2",
+  "version": "10.0.3-43",
   "keywords": [
     "logging",
     "replay"
   ],
   "author": "Alex Tsokurov",
   "contributors": [
-    "Aleksandr K <alex@openreplay.com>"
+    "Aleksandr K <alex@openreplay.com>",
+    "Nikita D <nikita@openreplay.com>"
   ],
   "license": "MIT",
   "type": "module",

tracker/tracker/src/common/messages.gen.ts

@@ -71,6 +71,7 @@ export declare const enum Type {
   ResourceTiming = 116,
   TabChange = 117,
   TabData = 118,
+  CanvasNode = 119,
 }
 
 
@@ -562,6 +563,12 @@ export type TabData = [
   /*tabId:*/ string,
 ]
 
+export type CanvasNode = [
+  /*type:*/ Type.CanvasNode,
+  /*nodeId:*/ string,
+  /*timestamp:*/ number,
+]
 
-type Message =  Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequestDeprecated | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTimingDeprecated | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | NetworkRequest | InputChange | SelectionChange | MouseThrashing | UnbindNodes | ResourceTiming | TabChange | TabData
+
+type Message =  Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequestDeprecated | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTimingDeprecated | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | NetworkRequest | InputChange | SelectionChange | MouseThrashing | UnbindNodes | ResourceTiming | TabChange | TabData | CanvasNode
 export default Message

tracker/tracker/src/main/app/canvas.ts

@@ -0,0 +1,130 @@
+import App from '../app/index.js'
+import { hasTag } from './guards.js'
+import Message, { CanvasNode } from './messages.gen.js'
+
+interface CanvasSnapshot {
+  images: { data: string; id: number }[]
+  createdAt: number
+}
+
+interface Options {
+  fps: number
+  quality: 'low' | 'medium' | 'high'
+}
+
+class CanvasRecorder {
+  private snapshots: Record<number, CanvasSnapshot> = {}
+  private readonly intervals: NodeJS.Timeout[] = []
+  private readonly interval: number
+
+  constructor(
+    private readonly app: App,
+    private readonly options: Options,
+  ) {
+    this.interval = 1000 / options.fps
+  }
+
+  startTracking() {
+    this.app.nodes.attachNodeCallback((node: Node): void => {
+      const id = this.app.nodes.getID(node)
+      if (!id || !hasTag(node, 'canvas') || this.snapshots[id]) {
+        return
+      }
+      const ts = this.app.timestamp()
+      this.snapshots[id] = {
+        images: [],
+        createdAt: ts,
+      }
+      const canvasMsg = CanvasNode(id.toString(), ts)
+      this.app.send(canvasMsg as Message)
+      const int = setInterval(() => {
+        const cid = this.app.nodes.getID(node)
+        const canvas = cid ? this.app.nodes.getNode(cid) : undefined
+        if (!canvas || !hasTag(canvas, 'canvas') || canvas !== node) {
+          console.log('Canvas element not in sync')
+          clearInterval(int)
+        } else {
+          const snapshot = captureSnapshot(canvas, this.options.quality)
+          this.snapshots[id].images.push({ id: this.app.timestamp(), data: snapshot })
+          if (this.snapshots[id].images.length > 9) {
+            this.sendSnaps(this.snapshots[id].images, id, this.snapshots[id].createdAt)
+            this.snapshots[id].images = []
+          }
+        }
+      }, this.interval)
+      this.intervals.push(int)
+    })
+  }
+
+  sendSnaps(images: { data: string; id: number }[], canvasId: number, createdAt: number) {
+    if (Object.keys(this.snapshots).length === 0) {
+      console.log(this.snapshots)
+      return
+    }
+    const formData = new FormData()
+    images.forEach((snapshot) => {
+      const blob = dataUrlToBlob(snapshot.data)[0]
+      formData.append('snapshot', blob, `${createdAt}_${canvasId}_${snapshot.id}.jpeg`)
+      // saveImageData(snapshot.data, `${createdAt}_${canvasId}_${snapshot.id}.jpeg`)
+    })
+
+    fetch(this.app.options.ingestPoint + '/v1/web/images', {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${this.app.session.getSessionToken() ?? ''}`,
+      },
+      body: formData,
+    })
+      .then((r) => {
+        console.log('done', r)
+      })
+      .catch((e) => {
+        console.error('error saving canvas', e)
+      })
+  }
+
+  clear() {
+    console.log('cleaning up')
+    this.intervals.forEach((int) => clearInterval(int))
+    this.snapshots = {}
+  }
+}
+
+const qualityInt = {
+  low: 0.33,
+  medium: 0.55,
+  high: 0.8,
+}
+
+function captureSnapshot(canvas: HTMLCanvasElement, quality: 'low' | 'medium' | 'high' = 'medium') {
+  const imageFormat = 'image/jpeg' // or /png'
+  return canvas.toDataURL(imageFormat, qualityInt[quality])
+}
+
+function dataUrlToBlob(dataUrl: string): [Blob, Uint8Array] {
+  const [header, base64] = dataUrl.split(',')
+  // @ts-ignore
+  const mime = header.match(/:(.*?);/)[1]
+  const blobStr = atob(base64)
+  let n = blobStr.length
+  const u8arr = new Uint8Array(n)
+
+  while (n--) {
+    u8arr[n] = blobStr.charCodeAt(n)
+  }
+
+  return [new Blob([u8arr], { type: mime }), u8arr]
+}
+
+function saveImageData(imageDataUrl: string, name: string) {
+  const link = document.createElement('a')
+  link.href = imageDataUrl
+  link.download = name
+  link.style.display = 'none'
+
+  document.body.appendChild(link)
+  link.click()
+  document.body.removeChild(link)
+}
+
+export default CanvasRecorder

tracker/tracker/src/main/app/guards.ts

@@ -38,6 +38,7 @@ type TagTypeMap = {
   iframe: HTMLIFrameElement
   style: HTMLStyleElement | SVGStyleElement
   link: HTMLLinkElement
+  canvas: HTMLCanvasElement
 }
 export function hasTag<T extends keyof TagTypeMap>(
   el: Node,

tracker/tracker/src/main/app/index.ts

@@ -23,6 +23,7 @@ import type { Options as SanitizerOptions } from './sanitizer.js'
 import type { Options as LoggerOptions } from './logger.js'
 import type { Options as SessOptions } from './session.js'
 import type { Options as NetworkOptions } from '../modules/network.js'
+import CanvasRecorder from './canvas.js'
 
 import type {
   Options as WebworkerOptions,
@@ -142,6 +143,12 @@ export default class App {
   private readonly bc: BroadcastChannel | null = null
   private readonly contextId
   public attributeSender: AttributeSender
+  private canvasRecorder: CanvasRecorder | null = null
+  private canvasOptions = {
+    canvasEnabled: false,
+    canvasQuality: 'medium',
+    canvasFPS: 1,
+  }
 
   constructor(projectKey: string, sessionToken: string | undefined, options: Partial<Options>) {
     // if (options.onStart !== undefined) {
@@ -631,6 +638,9 @@ export default class App {
           userDevice,
           userOS,
           userState,
+          canvasEnabled,
+          canvasQuality,
+          canvasFPS,
         } = r
         if (
           typeof token !== 'string' ||
@@ -675,13 +685,19 @@ export default class App {
         })
 
         this.compressionThreshold = compressionThreshold
-
         const onStartInfo = { sessionToken: token, userUUID, sessionID }
 
         // TODO: start as early as possible (before receiving the token)
         this.startCallbacks.forEach((cb) => cb(onStartInfo)) // MBTODO: callbacks after DOM "mounted" (observed)
         this.observer.observe()
         this.ticker.start()
+
+        if (canvasEnabled) {
+          this.canvasRecorder =
+            this.canvasRecorder ??
+            new CanvasRecorder(this, { fps: canvasFPS, quality: canvasQuality })
+          this.canvasRecorder.startTracking()
+        }
         this.activityState = ActivityState.Active
 
         this.notify.log('OpenReplay tracking started.')
@@ -752,6 +768,7 @@ export default class App {
         if (this.worker && stopWorker) {
           this.worker.postMessage('stop')
         }
+        this.canvasRecorder?.clear()
       } finally {
         this.activityState = ActivityState.NotActive
       }

tracker/tracker/src/main/app/messages.gen.ts

@@ -912,3 +912,14 @@ export function TabData(
   ]
 }
 
+export function CanvasNode(
+  nodeId: string,
+  timestamp: number,
+): Messages.CanvasNode {
+  return [
+    Messages.Type.CanvasNode,
+    nodeId,
+    timestamp,
+  ]
+}
+

tracker/tracker/src/tests/guards.test.ts

@@ -0,0 +1,71 @@
+import { describe, beforeEach, expect, test } from '@jest/globals'
+import {
+  isNode,
+  isSVGElement,
+  isElementNode,
+  isCommentNode,
+  isTextNode,
+  isDocument,
+  isRootNode,
+  hasTag,
+} from '../main/app/guards'
+
+describe('DOM utility functions', () => {
+  let elementNode: Element
+  let commentNode: Comment
+  let textNode: Text
+  let documentNode: Document
+  let fragmentNode: DocumentFragment
+  let svgElement: SVGElement
+
+  beforeEach(() => {
+    elementNode = document.createElement('div')
+    commentNode = document.createComment('This is a comment')
+    textNode = document.createTextNode('This is text')
+    documentNode = document
+    fragmentNode = document.createDocumentFragment()
+    svgElement = document.createElementNS('http://www.w3.org/2000/svg', 'svg')
+  })
+
+  test('isNode', () => {
+    expect(isNode(elementNode)).toBeTruthy()
+    expect(isNode(null)).toBeFalsy()
+  })
+
+  test('isSVGElement', () => {
+    expect(isSVGElement(svgElement)).toBeTruthy()
+    expect(isSVGElement(elementNode)).toBeFalsy()
+  })
+
+  test('isElementNode', () => {
+    expect(isElementNode(elementNode)).toBeTruthy()
+    expect(isElementNode(textNode)).toBeFalsy()
+  })
+
+  test('isCommentNode', () => {
+    expect(isCommentNode(commentNode)).toBeTruthy()
+    expect(isCommentNode(elementNode)).toBeFalsy()
+  })
+
+  test('isTextNode', () => {
+    expect(isTextNode(textNode)).toBeTruthy()
+    expect(isTextNode(elementNode)).toBeFalsy()
+  })
+
+  test('isDocument', () => {
+    expect(isDocument(documentNode)).toBeTruthy()
+    expect(isDocument(elementNode)).toBeFalsy()
+  })
+
+  test('isRootNode', () => {
+    expect(isRootNode(documentNode)).toBeTruthy()
+    expect(isRootNode(fragmentNode)).toBeTruthy()
+    expect(isRootNode(elementNode)).toBeFalsy()
+  })
+
+  test('hasTag', () => {
+    const imgElement = document.createElement('img')
+    expect(hasTag(imgElement, 'img')).toBeTruthy()
+    expect(hasTag(elementNode, 'img')).toBeFalsy()
+  })
+})

tracker/tracker/src/tests/nodes.unit.test.ts

@@ -1,5 +1,5 @@
-import Nodes from './nodes'
+import Nodes from '../main/app/nodes'
-import { describe, beforeEach, expect, it, jest } from '@jest/globals'
+import { describe, beforeEach, expect, test, jest } from '@jest/globals'
 
 describe('Nodes', () => {
   let nodes: Nodes
@@ -11,13 +11,13 @@ describe('Nodes', () => {
     mockCallback.mockClear()
   })
 
-  it('attachNodeCallback', () => {
+  test('attachNodeCallback', () => {
     nodes.attachNodeCallback(mockCallback)
     nodes.callNodeCallbacks(document.createElement('div'), true)
     expect(mockCallback).toHaveBeenCalled()
   })
 
-  it('attachNodeListener is listening to events', () => {
+  test('attachNodeListener is listening to events', () => {
     const node = document.createElement('div')
     const mockListener = jest.fn()
     document.body.appendChild(node)
@@ -26,7 +26,7 @@ describe('Nodes', () => {
     node.dispatchEvent(new Event('click'))
     expect(mockListener).toHaveBeenCalled()
   })
-  it('attachNodeListener is calling native method', () => {
+  test('attachNodeListener is calling native method', () => {
     const node = document.createElement('div')
     const mockListener = jest.fn()
     const addEventListenerSpy = jest.spyOn(node, 'addEventListener')
@@ -36,55 +36,55 @@ describe('Nodes', () => {
     expect(addEventListenerSpy).toHaveBeenCalledWith('click', mockListener, true)
   })
 
-  it('registerNode', () => {
+  test('registerNode', () => {
     const node = document.createElement('div')
     const [id, isNew] = nodes.registerNode(node)
     expect(id).toBeDefined()
     expect(isNew).toBe(true)
   })
 
-  it('unregisterNode', () => {
+  test('unregisterNode', () => {
     const node = document.createElement('div')
     const [id] = nodes.registerNode(node)
     const unregisteredId = nodes.unregisterNode(node)
     expect(unregisteredId).toBe(id)
   })
 
-  it('cleanTree', () => {
+  test('cleanTree', () => {
     const node = document.createElement('div')
     nodes.registerNode(node)
     nodes.cleanTree()
     expect(nodes.getNodeCount()).toBe(0)
   })
 
-  it('callNodeCallbacks', () => {
+  test('callNodeCallbacks', () => {
     nodes.attachNodeCallback(mockCallback)
     const node = document.createElement('div')
     nodes.callNodeCallbacks(node, true)
     expect(mockCallback).toHaveBeenCalledWith(node, true)
   })
 
-  it('getID', () => {
+  test('getID', () => {
     const node = document.createElement('div')
     const [id] = nodes.registerNode(node)
     const fetchedId = nodes.getID(node)
     expect(fetchedId).toBe(id)
   })
 
-  it('getNode', () => {
+  test('getNode', () => {
     const node = document.createElement('div')
     const [id] = nodes.registerNode(node)
     const fetchedNode = nodes.getNode(id)
     expect(fetchedNode).toBe(node)
   })
 
-  it('getNodeCount', () => {
+  test('getNodeCount', () => {
     expect(nodes.getNodeCount()).toBe(0)
     nodes.registerNode(document.createElement('div'))
     expect(nodes.getNodeCount()).toBe(1)
   })
 
-  it('clear', () => {
+  test('clear', () => {
     nodes.registerNode(document.createElement('div'))
     nodes.clear()
     expect(nodes.getNodeCount()).toBe(0)

tracker/tracker/src/webworker/MessageEncoder.gen.ts

@@ -286,6 +286,10 @@ export default class MessageEncoder extends PrimitiveEncoder {
       return  this.string(msg[1])
     break
 
+    case Messages.Type.CanvasNode:
+      return  this.string(msg[1]) && this.uint(msg[2])
+    break
+
     }
   }