fix(tracker): fix multi canvas streaming (#1945)

* fix(tracker): fix multi canvas streaming

* fix(tracker): fix canvas streaming

* fix(ui): fix canvas receiver interaction waiter

* fix(ui): remove console log

frontend/app/player/web/assist/CanvasReceiver.ts

@@ -33,10 +33,10 @@ export default class CanvasReceiver {
       path: '/assist',
       port:
         urlObject.port === ''
-          ? location.protocol === 'https:'
-            ? 443
-            : 80
-          : parseInt(urlObject.port),
+        ? location.protocol === 'https:'
+          ? 443
+          : 80
+        : parseInt(urlObject.port),
     };
     if (this.config) {
       peerOpts['config'] = {
@@ -68,7 +68,7 @@ export default class CanvasReceiver {
               (node.node as HTMLCanvasElement).getContext('2d') as CanvasRenderingContext2D
             );
           }
-        }, 500);
+        }, 250);
       });
       call.on('error', (err) => console.error('canvas call error', err));
     });
@@ -93,7 +93,28 @@ function spawnVideo(stream: MediaStream, node: VElement) {
   videoEl.setAttribute('muted', 'true');
   videoEl.setAttribute('playsinline', 'true');
   videoEl.setAttribute('crossorigin', 'anonymous');
-  void videoEl.play();
+
+  videoEl.play()
+    .then(() => true)
+    .catch(() => {
+      // we allow that if user just reloaded the page
+    })
+
+  const clearListeners = () => {
+    document.removeEventListener('click', startStream)
+    videoEl.removeEventListener('playing', clearListeners)
+  }
+  videoEl.addEventListener('playing', clearListeners)
+
+  const startStream = () => {
+    videoEl.play()
+      .then(() => console.log('unpaused'))
+      .catch(() => {
+        // we allow that if user just reloaded the page
+      })
+    document.removeEventListener('click', startStream)
+  }
+  document.addEventListener('click', startStream)
 
   return videoEl;
 }

tracker/tracker-assist/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 8.0.4
+
+- fix for multiple canvas streaming
+
 ## 8.0.3
 
 - make assist canvas layer hidden from canvas capturer

tracker/tracker-assist/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openreplay/tracker-assist",
   "description": "Tracker plugin for screen assistance through the WebRTC",
-  "version": "8.0.3",
+  "version": "8.0.4",
   "keywords": [
     "WebRTC",
     "assistance",

tracker/tracker-assist/src/Assist.ts

@@ -5,7 +5,7 @@ import Peer, { MediaConnection, } from 'peerjs'
 import type { Properties, } from 'csstype'
 import { App, } from '@openreplay/tracker'
 
-import RequestLocalStream, { LocalStream, } from './LocalStream.js'
+import RequestLocalStream, { LocalStream } from './LocalStream.js';
 import {hasTag,} from './guards.js'
 import RemoteControl, { RCStatus, } from './RemoteControl.js'
 import CallWindow from './CallWindow.js'
@@ -79,7 +79,7 @@ export default class Assist {
 
   private socket: Socket | null = null
   private peer: Peer | null = null
-  private canvasPeer: Peer | null = null
+  private canvasPeers: Record<number, Peer | null> = {}
   private assistDemandedRestart = false
   private callingState: CallingState = CallingState.False
   private remoteControl: RemoteControl | null = null;
@@ -590,32 +590,41 @@ export default class Assist {
       })
     })
 
+
+    const startCanvasStream = (stream: MediaStream, id: number) => {
+      const canvasPID = `${app.getProjectKey()}-${sessionId}-${id}`
+      if (!this.canvasPeers[id]) {
+        this.canvasPeers[id] = new safeCastedPeer(canvasPID, peerOptions) as Peer
+      }
+      this.canvasPeers[id]?.on('error', (e) => app.debug.error(e))
+
+      Object.values(this.agents).forEach(agent => {
+        if (agent.agentInfo) {
+          const target = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas`
+          const connection = this.canvasPeers[id]?.connect(target)
+          connection?.on('open', () => {
+            if (agent.agentInfo) {
+              const call = this.canvasPeers[id]?.call(target, stream.clone())
+              call?.on('error', app.debug.error)
+            }
+          })
+          connection?.on('error',  (e) => app.debug.error(e))
+        } else {
+          app.debug.error('Assist: cant establish canvas peer to agent, no agent info')
+        }
+      })
+    }
+
     app.nodes.attachNodeCallback((node) => {
       const id = app.nodes.getID(node)
       if (id && hasTag(node, 'canvas')) {
-        const canvasPId = `${app.getProjectKey()}-${sessionId}-${id}`
-        if (!this.canvasPeer) this.canvasPeer = new safeCastedPeer(canvasPId, peerOptions) as Peer
+        app.debug.log(`Creating stream for canvas ${id}`)
         const canvasHandler = new Canvas(
           node as unknown as HTMLCanvasElement,
           id,
           30,
           (stream: MediaStream) => {
-            Object.values(this.agents).forEach(agent => {
-              if (agent.agentInfo) {
-                const target = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas`
-                const connection = this.canvasPeer?.connect(target)
-                connection?.on('open', () => {
-                  if (agent.agentInfo) {
-                    const pCall = this.canvasPeer?.call(target, stream)
-                    pCall?.on('error', app.debug.error)
-                  }
-                })
-                connection?.on('error', app.debug.error)
-                this.canvasPeer?.on('error', app.debug.error)
-              } else {
-                app.debug.error('Assist: cant establish canvas peer to agent, no agent info')
-              }
-            })
+            startCanvasStream(stream, id)
           },
           app.debug.error,
         )
@@ -627,10 +636,10 @@ export default class Assist {
   private playNotificationSound() {
     if ('Audio' in window) {
       new Audio('https://static.openreplay.com/tracker-assist/notification.mp3')
-      .play()
-      .catch(e => {
-        this.app.debug.warn(e)
-      })
+        .play()
+        .catch(e => {
+          this.app.debug.warn(e)
+        })
     }
   }
 

tracker/tracker-assist/src/version.ts

@@ -1 +1 @@
-export const pkgVersion = '8.0.2'
+export const pkgVersion = "8.0.4";

tracker/tracker/src/main/app/index.ts

@@ -123,6 +123,7 @@ type AppOptions = {
   forceSingleTab?: boolean
   disableStringDict?: boolean
   assistSocketHost?: string
+  disableCanvas?: boolean
 
   /** @deprecated */
   onStart?: StartCallback
@@ -213,6 +214,7 @@ export default class App {
         forceSingleTab: false,
         assistSocketHost: '',
         fixedCanvasScaling: false,
+        disableCanvas: false,
       },
       options,
     )
@@ -1058,7 +1060,7 @@ export default class App {
         await this.tagWatcher.fetchTags(this.options.ingestPoint, token)
         this.activityState = ActivityState.Active
 
-        if (canvasEnabled) {
+        if (canvasEnabled && !this.options.disableCanvas) {
           this.canvasRecorder =
             this.canvasRecorder ??
             new CanvasRecorder(this, {