fix(player/tracker): fix camera and socket connection

frontend/app/components/Assist/ChatWindow/ChatWindow.tsx

@@ -1,29 +1,35 @@
-import React, { useState, useEffect } from 'react'
-import VideoContainer from '../components/VideoContainer'
-import cn from 'classnames'
-import Counter from 'App/components/shared/SessionItem/Counter'
-import stl from './chatWindow.module.css'
-import ChatControls from '../ChatControls/ChatControls'
+import React, { useState, useEffect } from 'react';
+import VideoContainer from '../components/VideoContainer';
+import cn from 'classnames';
+import Counter from 'App/components/shared/SessionItem/Counter';
+import stl from './chatWindow.module.css';
+import ChatControls from '../ChatControls/ChatControls';
 import Draggable from 'react-draggable';
 import type { LocalStream } from 'Player/MessageDistributor/managers/LocalStream';
+import { toggleVideoLocalStream } from 'Player'
 
 export interface Props {
-  incomeStream: MediaStream[] | null,
-  localStream: LocalStream | null,
-  userId: string,
+  incomeStream: MediaStream[] | null;
+  localStream: LocalStream | null;
+  userId: string;
   isPrestart?: boolean;
-  endCall: () => void
+  endCall: () => void;
 }
 
 function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }: Props) {
-  const [localVideoEnabled, setLocalVideoEnabled] = useState(false)
-  const [anyRemoteEnabled, setRemoteEnabled] = useState(false)
+  const [localVideoEnabled, setLocalVideoEnabled] = useState(false);
+  const [anyRemoteEnabled, setRemoteEnabled] = useState(false);
+
+  const onlyLocalEnabled = localVideoEnabled && !anyRemoteEnabled;
+
+  useEffect(() => {
+    toggleVideoLocalStream(localVideoEnabled)
+  }, [localVideoEnabled])
 
-  const onlyLocalEnabled = localVideoEnabled && !anyRemoteEnabled
   return (
     <Draggable handle=".handle" bounds="body" defaultPosition={{ x: 50, y: 200 }}>
       <div
-        className={cn(stl.wrapper, "fixed radius bg-white shadow-xl mt-16")}
+        className={cn(stl.wrapper, 'fixed radius bg-white shadow-xl mt-16')}
         style={{ width: '280px' }}
       >
         <div className="handle flex items-center p-2 cursor-move select-none border-b">
@@ -32,21 +38,39 @@ function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }:
             <br />
             {incomeStream && incomeStream.length > 2 ? ' (+ other agents in the call)' : ''}
           </div>
-          <Counter startTime={new Date().getTime() } className="text-sm ml-auto" />
+          <Counter startTime={new Date().getTime()} className="text-sm ml-auto" />
         </div>
-        <div className={cn(stl.videoWrapper, 'relative')} style={{ minHeight: onlyLocalEnabled ? 210 : 'unset'}}>
-          {incomeStream
-            ? incomeStream.map(stream => <React.Fragment key={stream.id}><VideoContainer stream={ stream } setRemoteEnabled={setRemoteEnabled} /></React.Fragment>) : (
+        <div
+          className={cn(stl.videoWrapper, 'relative')}
+          style={{ minHeight: onlyLocalEnabled ? 210 : 'unset' }}
+        >
+          {incomeStream ? (
+            incomeStream.map((stream) => (
+              <React.Fragment key={stream.id}>
+                <VideoContainer stream={stream} setRemoteEnabled={setRemoteEnabled} />
+              </React.Fragment>
+            ))
+          ) : (
             <div className={stl.noVideo}>Error obtaining incoming streams</div>
           )}
-          <div className={cn("absolute bottom-0 right-0 z-50", localVideoEnabled ? "" : "!hidden")}>
-            <VideoContainer stream={ localStream ? localStream.stream : null } muted height={anyRemoteEnabled ? 50 : 'unset'} />
+          <div className={cn('absolute bottom-0 right-0 z-50', localVideoEnabled ? '' : '!hidden')}>
+            <VideoContainer
+              stream={localStream ? localStream.stream : null}
+              muted
+              height={anyRemoteEnabled ? 50 : 'unset'}
+            />
           </div>
         </div>
-        <ChatControls videoEnabled={localVideoEnabled} setVideoEnabled={setLocalVideoEnabled} stream={localStream} endCall={endCall} isPrestart={isPrestart} />
+        <ChatControls
+          videoEnabled={localVideoEnabled}
+          setVideoEnabled={setLocalVideoEnabled}
+          stream={localStream}
+          endCall={endCall}
+          isPrestart={isPrestart}
+        />
       </div>
     </Draggable>
-  )
+  );
 }
 
-export default ChatWindow
+export default ChatWindow;

frontend/app/player/MessageDistributor/managers/AssistManager.ts

@@ -205,9 +205,12 @@ export default class AssistManager {
         }
       })
       socket.on('videofeed', ({ streamId, enabled }) => {
+        console.log(streamId, enabled)
+        console.log(this.videoStreams)
         if (this.videoStreams[streamId]) {
           this.videoStreams[streamId].enabled = enabled
         }
+        console.log(this.videoStreams)
       })
       socket.on('SESSION_DISCONNECTED', e => {
         waitingForMessages = true
@@ -370,10 +373,8 @@ export default class AssistManager {
           })
 
           call.on('stream', stream => {
+            this.videoStreams[call.peer] = stream.getVideoTracks()[0]
             this.callArgs && this.callArgs.onStream(stream)
-            stream.getVideoTracks().forEach(track => {
-              this.videoStreams[track.id] = track
-            })
           });
           // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
 
@@ -505,9 +506,8 @@ export default class AssistManager {
       call.on('stream', stream => {
         getState().calling !== CallingState.OnCall && update({ calling: CallingState.OnCall })
 
-        stream.getVideoTracks().forEach(track => {
-          this.videoStreams[track.id] = track
-        })
+        this.videoStreams[call.peer] = stream.getVideoTracks()[0]
+
         this.callArgs && this.callArgs.onStream(stream)
       });
       // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
@@ -562,6 +562,12 @@ export default class AssistManager {
     }
   }
 
+  toggleVideoLocalStream(enabled: boolean) {
+    this.getPeer().then((peer) => {
+      this.socket.emit('videofeed', { streamId: peer.id, enabled })
+    })
+  }
+
   private annot: AnnotationCanvas | null = null
 
   /* ==== Cleaning ==== */

frontend/app/player/singletone.js

@@ -86,6 +86,8 @@ export const jumpToLive = initCheck((...args) => instance.jumpToLive(...args))
 export const toggleUserName = initCheck((...args) => instance.toggleUserName(...args))
 export const injectNotes = initCheck((...args) => instance.injectNotes(...args))
 export const filterOutNote = initCheck((...args) => instance.filterOutNote(...args))
+/** @type {Player.assistManager.toggleVideoLocalStream} */
+export const toggleVideoLocalStream = initCheck((...args) => instance.assistManager.toggleVideoLocalStream(...args))
 
 export const Controls = {
   jump,

tracker/tracker-assist/src/Assist.ts

@@ -248,8 +248,8 @@ export default class Assist {
       callingAgents.set(id, name)
       updateCallerNames()
     })
-    socket.on('videofeed', ({ streamId, enabled, }) => {
-      callUI?.toggleVideoStream({ streamId, enabled, })
+    socket.on('videofeed', (id, feedState) => {
+      callUI?.toggleVideoStream(feedState)
     })
 
     const callingAgents: Map<string, string> = new Map() // !! uses socket.io ID
@@ -341,7 +341,7 @@ export default class Assist {
       this.emit('call_end')
       handleCallEnd()
     }
-    const updateVideoFeed = ({ streamId, enabled, }) => this.emit('videofeed', { streamId, enabled, })
+    const updateVideoFeed = ({ enabled, }) => this.emit('videofeed', { streamId: this.peer?.id, enabled, })
 
     peer.on('call', (call) => {
       app.debug.log('Incoming call: ', call)
@@ -399,7 +399,7 @@ export default class Assist {
           initiateCallEnd()
         })
         call.on('stream', (rStream) => {
-          callUI?.addRemoteStream(rStream)
+          callUI?.addRemoteStream(rStream, call.peer)
           const onInteraction = () => { // do only if document.hidden ?
             callUI?.playRemote()
             document.removeEventListener('click', onInteraction)

tracker/tracker-assist/src/CallWindow.ts

@@ -4,6 +4,7 @@ import attachDND from './dnd.js'
 const SS_START_TS_KEY = '__openreplay_assist_call_start_ts'
 
 export default class CallWindow {
+	private remoteVideoId: string
 	private readonly iframe: HTMLIFrameElement
 	private vRemote: HTMLVideoElement | null = null
 	private vLocal: HTMLVideoElement | null = null
@@ -30,7 +31,7 @@ export default class CallWindow {
 			position: 'fixed',
 			zIndex: 2147483647 - 1,
 			border: 'none',
-			bottom: '10px',
+			bottom: '50px',
 			right: '10px',
 			height: '200px',
 			width: '200px',
@@ -127,13 +128,14 @@ export default class CallWindow {
 
 	private checkRemoteVideoInterval: ReturnType<typeof setInterval>
 	private audioContainer: HTMLDivElement | null = null
-	addRemoteStream(rStream: MediaStream) {
+	addRemoteStream(rStream: MediaStream, peerId: string) {
 		this.load
 			.then(() => {
 				// Video
 				if (this.vRemote && !this.vRemote.srcObject) {
 					this.vRemote.srcObject = rStream
 					this.remoteVideo = rStream.getVideoTracks()[0]
+					this.remoteVideoId = peerId
 					if (this.vPlaceholder) {
 						this.vPlaceholder.innerText =
 							'Video has been paused. Click anywhere to resume.'
@@ -177,6 +179,8 @@ export default class CallWindow {
 					this.remoteVideoOn = enable
 					if (enable) {
 						this.videoContainer.classList.add('remote')
+					} else {
+						this.videoContainer.classList.remove('remote')
 					}
 					this.adjustIframeSize()
 				}
@@ -320,9 +324,10 @@ export default class CallWindow {
 		this.localStreams = []
 	}
 
-	toggleVideoStream({ streamId, enabled, }: { streamId: string, enabled: boolean}) {
-		if (this.remoteVideo.id === streamId) {
+	toggleVideoStream({ streamId, enabled, }: { streamId: string, enabled: boolean }) {
+		if (this.remoteVideoId === streamId) {
 			this.remoteVideo.enabled = enabled
+			this.toggleRemoteVideoUI(enabled)
 		}
 	}
 }