fix(ui): fix assist multi call videos

frontend/app/components/Assist/ChatWindow/ChatWindow.tsx

@@ -17,22 +17,6 @@ export interface Props {
 
 function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }: Props) {
   const [localVideoEnabled, setLocalVideoEnabled] = useState(false)
-  const [remoteVideoEnabled, setRemoteVideoEnabled] = useState(false)
-
-  useEffect(() => {
-    if (!incomeStream || incomeStream.length === 0) { return }
-    const iid = setInterval(() => {
-      const settings = incomeStream.map(stream => stream.getVideoTracks()[0]?.getSettings()).filter(Boolean)
-      const isDummyVideoTrack = settings.length > 0 ? (settings.every(s => s.width === 2 || s.frameRate === 0 || s.frameRate === undefined)) : true
-      const shouldBeEnabled = !isDummyVideoTrack
-      if (shouldBeEnabled !== localVideoEnabled) {
-        setRemoteVideoEnabled(shouldBeEnabled)
-      }
-    }, 1000)
-    return () => clearInterval(iid)
-  }, [ incomeStream, localVideoEnabled ])
-
-  const minimize = !localVideoEnabled && !remoteVideoEnabled
 
   return (
     <Draggable handle=".handle" bounds="body">
@@ -43,15 +27,18 @@ function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }:
         <div className="handle flex items-center p-2 cursor-move select-none border-b">
           <div className={stl.headerTitle}>
             <b>Talking to </b> {userId ? userId : 'Anonymous User'}
+            <br />
             {incomeStream && incomeStream.length > 2 ? ' (+ other agents in the call)' : ''}
           </div>
           <Counter startTime={new Date().getTime() } className="text-sm ml-auto" />
         </div>
-        <div className={cn(stl.videoWrapper, {'hidden' : minimize}, 'relative')}>
-          {!incomeStream && <div className={stl.noVideo}>Error obtaining incoming streams</div>}
-          {incomeStream && incomeStream.map(stream => <VideoContainer stream={ stream } />)}
-          <div className="absolute bottom-0 right-0 z-50">
-            <VideoContainer stream={ localStream ? localStream.stream : null } muted width={50} />
+        <div className={cn(stl.videoWrapper, 'relative')} style={{ minHeight: localVideoEnabled ? 52 : undefined}}>
+          {incomeStream
+            ? incomeStream.map(stream => <React.Fragment key={stream.id}><VideoContainer stream={ stream } /></React.Fragment>) : (
+            <div className={stl.noVideo}>Error obtaining incoming streams</div>
+          )}
+          <div className={cn("absolute bottom-0 right-0 z-50", localVideoEnabled ? "" : "!hidden")}>
+            <VideoContainer stream={ localStream ? localStream.stream : null } muted height={50} />
           </div>
         </div>
         <ChatControls videoEnabled={localVideoEnabled} setVideoEnabled={setLocalVideoEnabled} stream={localStream} endCall={endCall} isPrestart={isPrestart} />

frontend/app/components/Assist/ChatWindow/chatWindow.module.css

@@ -2,7 +2,7 @@
   background-color: white;
   border: solid thin $gray-light;
   border-radius: 3px;
-  position: fixed;  
+  position: fixed;
   width: 300px;
   box-shadow: 1px 1px 1px rgba(0, 0, 0, 0.1);
 }
@@ -16,7 +16,7 @@
 }
 
 .videoWrapper {
-  height: 180px;
-  overflow: hidden;
+  max-height: 280px;
+  display: flex;
   background-color: #000;
-}
+}

frontend/app/components/Assist/components/AssistActions/AssistActions.tsx

@@ -25,7 +25,7 @@ function onError(e) {
 }
 
 interface Props {
-    userId: String;
+    userId: string;
     toggleChatWindow: (state) => void;
     calling: CallingState;
     annotating: boolean;
@@ -69,7 +69,12 @@ function AssistActions({
     }, [peerConnectionStatus]);
 
     const addIncomeStream = (stream: MediaStream) => {
-        setIncomeStream(oldState => [...oldState, stream]);
+        setIncomeStream(oldState => {
+            if (!oldState.find(existingStream => existingStream.id === stream.id)) {
+                return [...oldState, stream]
+            }
+            return oldState
+        });
     }
 
     function call(agentIds?: string[]) {

frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx

@@ -6,18 +6,30 @@ interface Props {
   width?: number
 }
 
-function VideoContainer({ stream, muted = false, width = 280 }: Props) {  
+function VideoContainer({ stream, muted = false, height = 280 }: Props) {
   const ref = useRef<HTMLVideoElement>(null);
+  const [isEnabled, setEnabled] = React.useState(false);
 
   useEffect(() => {
     if (ref.current) {
       ref.current.srcObject = stream;
     }
-  }, [ ref.current, stream ])  
+  }, [ ref.current, stream, stream.getVideoTracks()[0]?.getSettings().width ])
+
+  useEffect(() => {
+    if (!stream) { return }
+    const iid = setInterval(() => {
+      const settings = stream.getVideoTracks()[0]?.getSettings()
+      const isDummyVideoTrack = settings.width === 2 || settings.frameRate === 0
+      const shouldBeEnabled = !isDummyVideoTrack
+      isEnabled !== shouldBeEnabled ? setEnabled(shouldBeEnabled) : null;
+    }, 1000)
+    return () => clearInterval(iid)
+  }, [ stream, isEnabled ])
 
   return (
-    <div>
-      <video autoPlay ref={ ref } muted={ muted } style={{ width: width }} />
+    <div className="flex-1" style={{ display: isEnabled ? undefined : 'none', border: "1px solid grey" }}>
+      <video autoPlay ref={ ref } muted={ muted } style={{ height: height }} />
     </div>
   )
 }

frontend/app/player/MessageDistributor/managers/AssistManager.ts

@@ -54,7 +54,7 @@ export function getStatusText(status: ConnectionStatus): string {
       return "Connected. Waiting for the data... (The tab might be inactive)"
   }
 }
- 
+
 export interface State {
   calling: CallingState;
   peerConnectionStatus: ConnectionStatus;
@@ -77,11 +77,11 @@ const MAX_RECONNECTION_COUNT = 4;
 export default class AssistManager {
   private timeTravelJump = false;
   private jumped = false;
-  
+
   constructor(private session: any, private md: MessageDistributor, private config: any) {}
 
   private setStatus(status: ConnectionStatus) {
-    if (getState().peerConnectionStatus === ConnectionStatus.Disconnected && 
+    if (getState().peerConnectionStatus === ConnectionStatus.Disconnected &&
       status !== ConnectionStatus.Connected) {
       return
     }
@@ -109,7 +109,7 @@ export default class AssistManager {
     if (document.hidden) {
       this.socketCloseTimeout = setTimeout(() => {
         const state = getState()
-        if (document.hidden && 
+        if (document.hidden &&
           (state.calling === CallingState.NoCall && state.remoteControl === RemoteControlStatus.Enabled)) {
           this.socket?.close()
         }
@@ -238,7 +238,7 @@ export default class AssistManager {
       })
       socket.on('call_end', this.onRemoteCallEnd)
 
-      document.addEventListener('visibilitychange', this.onVisChange)        
+      document.addEventListener('visibilitychange', this.onVisChange)
 
     })
   }
@@ -262,14 +262,14 @@ export default class AssistManager {
   private onMouseClick = (e: MouseEvent): void => {
     if (!this.socket) { return; }
     if (getState().annotating) { return; } // ignore clicks while annotating
-    
+
     const data = this.md.getInternalViewportCoordinates(e)
     // const el = this.md.getElementFromPoint(e); // requires requestiong node_id from domManager
     const el = this.md.getElementFromInternalPoint(data)
     if (el instanceof HTMLElement) {
       el.focus()
       el.oninput = e => {
-        if (el instanceof HTMLTextAreaElement 
+        if (el instanceof HTMLTextAreaElement
           || el instanceof HTMLInputElement
         ) {
           this.socket && this.socket.emit("input", el.value)
@@ -354,7 +354,7 @@ export default class AssistManager {
         console.log('getting call from', call.peer)
           call.answer(this.callArgs.localStream.stream)
           this.callConnection.push(call)
-    
+
           this.callArgs.localStream.onVideoTrack(vTrack => {
             const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video")
             if (!sender) {
@@ -363,12 +363,12 @@ export default class AssistManager {
             }
             sender.replaceTrack(vTrack)
           })
-    
+
           call.on('stream', stream => {
             this.callArgs && this.callArgs.onStream(stream)
           });
           // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
-    
+
           call.on("close", this.onRemoteCallEnd)
           call.on("error", (e) => {
             console.error("PeerJS error (on call):", e)
@@ -385,7 +385,7 @@ export default class AssistManager {
 
         //call-reconnection connected
  //       if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) {
-          // this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT 
+          // this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT
           //   ? ConnectionStatus.Connecting
           //   : ConnectionStatus.Disconnected);
           // Reconnect...
@@ -428,15 +428,15 @@ export default class AssistManager {
     localStream: LocalStream,
     onStream: (s: MediaStream)=>void,
     onCallEnd: () => void,
-    onReject: () => void, 
+    onReject: () => void,
     onError?: ()=> void,
   } | null = null
 
   public setCallArgs(
-    localStream: LocalStream, 
-    onStream: (s: MediaStream)=>void, 
-    onCallEnd: () => void, 
-    onReject: () => void, 
+    localStream: LocalStream,
+    onStream: (s: MediaStream)=>void,
+    onCallEnd: () => void,
+    onReject: () => void,
     onError?: ()=> void,
   ) {
     this.callArgs = {
@@ -459,7 +459,7 @@ export default class AssistManager {
     }
   }
 
-  /** Connecting to the other agents that are already 
+  /** Connecting to the other agents that are already
    *  in the call with the user
    */
   public addPeerCall(thirdPartyPeers: string[]) {

tracker/tracker-assist/src/Assist.ts

@@ -60,11 +60,11 @@ export default class Assist {
   private agents: Record<string, Agent> = {}
   private readonly options: Options
   constructor(
-    private readonly app: App, 
-    options?: Partial<Options>, 
+    private readonly app: App,
+    options?: Partial<Options>,
     private readonly noSecureMode: boolean = false,
   ) {
-    this.options = Object.assign({ 
+    this.options = Object.assign({
         session_calling_peer_key: '__openreplay_calling_peer',
         session_control_peer_key: '__openreplay_control_peer',
         config: null,
@@ -91,12 +91,12 @@ export default class Assist {
     const observer = titleNode && new MutationObserver(() => {
       this.emit('UPDATE_SESSION', { pageTitle: document.title, })
     })
-    app.attachStartCallback(() => { 
+    app.attachStartCallback(() => {
       if (this.assistDemandedRestart) { return }
       this.onStart()
       observer && observer.observe(titleNode, { subtree: true, characterData: true, childList: true, })
     })
-    app.attachStopCallback(() => { 
+    app.attachStopCallback(() => {
       if (this.assistDemandedRestart) { return }
       this.clean()
       observer && observer.disconnect()