Migrate to webrtc (#3051)

* resolved conflicts

* resolved conflicts

* translated comments

* changed console.log message lang

* changed console to logs

* implementing conference call

* add isAgent flag

* add webrtc handlers

* add conference call

* removed conference calls

* fix lint error

---------

Co-authored-by: Andrey Babushkin <a.babushkin@lemon-ai.com>

frontend/app/components/Assist/ChatWindow/ChatWindow.tsx

@@ -9,7 +9,7 @@ import type { LocalStream } from 'Player';
 import { PlayerContext } from 'App/components/Session/playerContext';
 
 export interface Props {
-  incomeStream: MediaStream[] | null;
+  incomeStream: { stream: MediaStream, isAgent: boolean }[] | null;
   localStream: LocalStream | null;
   userId: string;
   isPrestart?: boolean;
@@ -50,8 +50,8 @@ function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }:
         >
           {incomeStream ? (
             incomeStream.map((stream) => (
-              <React.Fragment key={stream.id}>
+              <React.Fragment key={stream.stream.id}>
-                <VideoContainer stream={stream} setRemoteEnabled={setRemoteEnabled} />
+                <VideoContainer stream={stream.stream} setRemoteEnabled={setRemoteEnabled} isAgent={stream.isAgent} />
               </React.Fragment>
             ))
           ) : (
@@ -62,6 +62,7 @@ function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }:
               stream={localStream ? localStream.stream : null}
               muted
               height={anyRemoteEnabled ? 50 : 'unset'}
+              local
             />
           </div>
         </div>

frontend/app/components/Assist/components/AssistActions/AssistActions.tsx

@@ -80,7 +80,7 @@ function AssistActions({
   } = store.get();
 
   const [isPrestart, setPrestart] = useState(false);
-  const [incomeStream, setIncomeStream] = useState<MediaStream[] | null>([]);
+  const [incomeStream, setIncomeStream] = useState<{ stream: MediaStream; isAgent: boolean }[] | null>([]);
   const [localStream, setLocalStream] = useState<LocalStream | null>(null);
   const [callObject, setCallObject] = useState<{ end: () => void } | null>(null);
 
@@ -131,18 +131,25 @@ function AssistActions({
     }
   }, [peerConnectionStatus]);
 
-  const addIncomeStream = (stream: MediaStream) => {
+  const addIncomeStream = (stream: MediaStream, isAgent: boolean) => {
     setIncomeStream((oldState) => {
-      if (oldState === null) return [stream];
+      if (oldState === null) return [{ stream, isAgent }];
-      if (!oldState.find((existingStream) => existingStream.id === stream.id)) {
+      if (!oldState.find((existingStream) => existingStream.stream.id === stream.id)) {
         audioContextManager.mergeAudioStreams(stream);
-        return [...oldState, stream];
+        return [...oldState, { stream, isAgent }];
       }
       return oldState;
     });
   };
 
-  function call(additionalAgentIds?: string[]) {
+  const removeIncomeStream = (stream: MediaStream) => {
+    setIncomeStream((prevState) => {
+      if (!prevState) return [];
+      return prevState.filter((existingStream) => existingStream.stream.id !== stream.id);
+    });
+  };
+
+  function call() {
     RequestLocalStream()
       .then((lStream) => {
         setLocalStream(lStream);
@@ -152,15 +159,16 @@ function AssistActions({
           addIncomeStream,
           () => {
             player.assistManager.ping(AssistActionsPing.call.end, agentId)
-            lStream.stop.bind(lStream);
+            lStream.stop.apply(lStream);
+            removeIncomeStream(lStream.stream);
           },
           onReject,
           onError
         );
         setCallObject(callPeer());
-        if (additionalAgentIds) {
+        // if (additionalAgentIds) {
-          callPeer(additionalAgentIds);
+        //   callPeer(additionalAgentIds);
-        }
+        // }
       })
       .catch(onError);
   }

frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx

@@ -5,9 +5,18 @@ interface Props {
   muted?: boolean;
   height?: number | string;
   setRemoteEnabled?: (isEnabled: boolean) => void;
+  local?: boolean;
+  isAgent?: boolean;
 }
 
-function VideoContainer({ stream, muted = false, height = 280, setRemoteEnabled }: Props) {
+function VideoContainer({
+  stream,
+  muted = false,
+  height = 280,
+  setRemoteEnabled,
+  local,
+  isAgent,
+}: Props) {
   const ref = useRef<HTMLVideoElement>(null);
   const [isEnabled, setEnabled] = React.useState(false);
 
@@ -15,14 +24,14 @@ function VideoContainer({ stream, muted = false, height = 280, setRemoteEnabled
     if (ref.current) {
       ref.current.srcObject = stream;
     }
-  }, [ref.current, stream, stream.getVideoTracks()[0]?.getSettings().width]);
+  }, [ref.current, stream, stream?.getVideoTracks()[0]?.getSettings().width]);
 
   useEffect(() => {
     if (!stream) {
       return;
     }
     const iid = setInterval(() => {
-      const track = stream.getVideoTracks()[0]
+      const track = stream.getVideoTracks()[0];
       const settings = track?.getSettings();
       const isDummyVideoTrack = settings
         ? settings.width === 2 ||
@@ -47,9 +56,19 @@ function VideoContainer({ stream, muted = false, height = 280, setRemoteEnabled
         width: isEnabled ? undefined : '0px!important',
         height: isEnabled ? undefined : '0px!important',
         border: '1px solid grey',
+        transform: local ? 'scaleX(-1)' : undefined,
       }}
     >
       <video autoPlay ref={ref} muted={muted} style={{ height: height }} />
+      {isAgent ? (
+        <div
+          style={{
+            position: 'absolute',
+          }}
+        >
+          Agent
+        </div>
+      ) : null}
     </div>
   );
 }

frontend/app/components/Session/Player/LivePlayer/LivePlayerBlockHeader.tsx

@@ -87,7 +87,7 @@ function LivePlayerBlockHeader({
             </div>
           )}
 
-            <AssistActions userId={userId} isCallActive={isCallActive} agentIds={agentIds} />
+            <AssistActions userId={userId} isCallActive={isCallActive} agentIds={agentIds ?? []} />
         </div>
       </div>
     </div>

frontend/app/player/web/WebLivePlayer.ts

@@ -39,6 +39,7 @@ export default class WebLivePlayer extends WebPlayer {
       config,
       wpState,
       (id) => this.messageManager.getNode(id),
+      agentId,
       uiErrorHandler
     );
     this.assistManager.connect(session.agentToken!, agentId, projectId);

frontend/app/player/web/assist/AssistManager.ts

@@ -67,6 +67,7 @@ export default class AssistManager {
     ...RemoteControl.INITIAL_STATE,
     ...ScreenRecording.INITIAL_STATE,
   };
+  private agentIds: string[] = [];
 
   // TODO: Session type
   constructor(
@@ -77,9 +78,10 @@ export default class AssistManager {
     private config: RTCIceServer[] | null,
     private store: Store<typeof AssistManager.INITIAL_STATE>,
     private getNode: MessageManager['getNode'],
+    public readonly agentId: number,
     public readonly uiErrorHandler?: {
       error: (msg: string) => void;
-    }
+    },
   ) {}
 
   public getAssistVersion = () => this.assistVersion;
@@ -192,6 +194,12 @@ export default class AssistManager {
           }),
         },
       }));
+
+      // socket.onAny((event, ...args) => {
+      //   logger.log(`📩 Socket: ${event}`, args);
+      // });
+      
+
       socket.on('connect', () => {
         waitingForMessages = true;
         // TODO: reconnect happens frequently on bad network
@@ -268,6 +276,10 @@ export default class AssistManager {
             }
           }
         }
+        if (data.agentIds) {
+          const filteredAgentIds = this.agentIds.filter((id: string) => id.split('-')[3] !== agentId.toString());
+          this.agentIds = filteredAgentIds;
+        }
       });
       socket.on('SESSION_DISCONNECTED', (e) => {
         waitingForMessages = true;
@@ -288,7 +300,12 @@ export default class AssistManager {
         socket,
         this.config,
         this.peerID,
-        this.getAssistVersion
+        this.getAssistVersion,
+        {
+          ...this.session.agentInfo,
+          id: agentId,
+        },
+        this.agentIds,
       );
       this.remoteControl = new RemoteControl(
         this.store,
@@ -309,7 +326,7 @@ export default class AssistManager {
       this.canvasReceiver = new CanvasReceiver(this.peerID, this.config, this.getNode, {
         ...this.session.agentInfo,
         id: agentId,
-      });
+      }, socket);
 
       document.addEventListener('visibilitychange', this.onVisChange);
     });
@@ -318,7 +335,7 @@ export default class AssistManager {
   /**
    * Sends event ping to stats service
    * */
-  public ping(event: StatsEvent, id: number) {
+  public ping(event: StatsEvent, id: string) {
     this.socket?.emit(event, id);
   }
 

frontend/app/player/web/assist/Call.ts

@@ -1,10 +1,8 @@
-import type Peer from 'peerjs';
-import type { MediaConnection } from 'peerjs';
-
 import type { LocalStream } from './LocalStream';
 import type { Socket } from './types';
 import type { Store } from '../../common/types';
 import { userStore } from "App/mstore";
+import logger from '@/logger';
 
 export enum CallingState {
   NoCall,
@@ -19,28 +17,59 @@ export interface State {
   currentTab?: string;
 }
 
+const WEBRTC_CALL_AGENT_EVENT_TYPES = {
+  OFFER: 'offer',
+  ANSWER: 'answer',
+  ICE_CANDIDATE: 'ice-candidate',
+}
+
 export default class Call {
   private assistVersion = 1;
   static readonly INITIAL_STATE: Readonly<State> = {
     calling: CallingState.NoCall,
   };
 
-  private _peer: Peer | null = null;
+  private connections: Record<string, RTCPeerConnection> = {};
-  private connectionAttempts: number = 0;
+  private connectAttempts = 0;
-  private callConnection: MediaConnection[] = [];
   private videoStreams: Record<string, MediaStreamTrack> = {};
+  private callID: string;
+  private agentInCallIds: string[] = [];
 
   constructor(
     private store: Store<State & { tabs: Set<string> }>,
     private socket: Socket,
     private config: RTCIceServer[] | null,
     private peerID: string,
-    private getAssistVersion: () => number
+    private getAssistVersion: () => number,
+    private agent: Record<string, any>,
+    private agentIds: string[],
   ) {
+
+    socket.on('WEBRTC_AGENT_CALL', (data) => {
+      switch (data.type) {
+        case WEBRTC_CALL_AGENT_EVENT_TYPES.OFFER:
+          this.handleOffer(data, true);
+          break;
+        case WEBRTC_CALL_AGENT_EVENT_TYPES.ICE_CANDIDATE:
+          this.handleIceCandidate(data);
+          break;
+        case WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER:
+          this.handleAnswer(data, true);
+        default:
+          break;
+      }
+    })
+
+    socket.on('UPDATE_SESSION', (data: { data: { agentIds: string[] }}) => {
+      this.callAgentsInSession({ agentIds: data.data.agentIds });
+    });
+
     socket.on('call_end', () => {
       this.onRemoteCallEnd()
     });
-    socket.on('videofeed', ({ streamId, enabled }) => {
+
+    socket.on('videofeed', (data: { data: { streamId: string; enabled: boolean }}) => {
+      const { streamId, enabled } = data.data;
       if (this.videoStreams[streamId]) {
         this.videoStreams[streamId].enabled = enabled;
       }
@@ -56,14 +85,13 @@ export default class Call {
     });
     socket.on('messages_gz', () => {
       if (reconnecting) {
-        // 'messages' come frequently, so it is better to have Reconnecting
+        // When the connection is restored, we initiate a re-creation of the connection
         this._callSessionPeer();
         reconnecting = false;
       }
     })
     socket.on('messages', () => {
       if (reconnecting) {
-        // 'messages' come frequently, so it is better to have Reconnecting
         this._callSessionPeer();
         reconnecting = false;
       }
@@ -71,111 +99,239 @@ export default class Call {
     socket.on('disconnect', () => {
       this.store.update({ calling: CallingState.NoCall });
     });
+
+    socket.on('webrtc_call_offer', (data: { data: { from: string, offer: RTCSessionDescriptionInit } }) => {
+      this.handleOffer(data.data);
+    });
+
+    socket.on('webrtc_call_answer', (data: { data: { from: string, answer: RTCSessionDescriptionInit } }) => {
+      this.handleAnswer(data.data);
+    });
+    socket.on('webrtc_call_ice_candidate', (data: { data: { from: string, candidate: RTCIceCandidateInit } }) => {
+      this.handleIceCandidate({ candidate: data.data.candidate, from: data.data.from });
+    });
+
     this.assistVersion = this.getAssistVersion();
   }
 
-  private getPeer(): Promise<Peer> {
+  // CREATE A LOCAL PEER
-    if (this._peer && !this._peer.disconnected) {
+  private async createPeerConnection({ remotePeerId, localPeerId, isAgent }: { remotePeerId: string, isAgent?: boolean, localPeerId?: string }): Promise<RTCPeerConnection> {
-      return Promise.resolve(this._peer);
+    // create pc with ice config
+    
+    const pc = new RTCPeerConnection({
+      iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
+    });
+
+    // If there is a local stream, add its tracks to the connection
+    if (this.callArgs && this.callArgs.localStream && this.callArgs.localStream.stream) {
+      this.callArgs.localStream.stream.getTracks().forEach((track) => {
+        pc.addTrack(track, this.callArgs!.localStream.stream);
+      });
     }
 
-    // @ts-ignore
+    // when ice is ready we send it
-    const urlObject = new URL(window.env.API_EDP || window.location.origin);
+    pc.onicecandidate = (event) => {
-
+      if (event.candidate) {
-    // @ts-ignore TODO: set module in ts settings
+        if (isAgent) {
-    return import('peerjs').then(({ default: Peer }) => {
+          this.socket.emit('WEBRTC_AGENT_CALL', { from: localPeerId, candidate: event.candidate, toAgentId: getSocketIdByCallId(remotePeerId), type: WEBRTC_CALL_AGENT_EVENT_TYPES.ICE_CANDIDATE });
-      if (this.cleaned) {
+        } else {
-        return Promise.reject('Already cleaned');
+          this.socket.emit('webrtc_call_ice_candidate', { from: remotePeerId, candidate: event.candidate });
+        }
+      } else {
+        logger.log("ICE candidate gathering complete");
       }
-      const peerOpts: Peer.PeerJSOption = {
-        host: urlObject.hostname,
-        path: '/assist',
-        port:
-          urlObject.port === ''
-            ? location.protocol === 'https:'
-              ? 443
-              : 80
-            : parseInt(urlObject.port),
     };
-      if (this.config) {
-        peerOpts['config'] = {
-          iceServers: this.config,
-          //@ts-ignore
-          sdpSemantics: 'unified-plan',
-          iceTransportPolicy: 'all',
-        };
-      }
-      const peer = (this._peer = new Peer(peerOpts));
-      peer.on('call', (call) => {
-        console.log('getting call from', call.peer);
-        call.answer(this.callArgs?.localStream.stream);
-        this.callConnection.push(call);
 
-        this.callArgs?.localStream.onVideoTrack((vTrack) => {
+    // when we receive a remote track, we write it to videoStreams[peerId]
-          const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video');
+    pc.ontrack = (event) => {
+      const stream = event.streams[0];
+      if (stream && !this.videoStreams[remotePeerId]) {
+        const clonnedStream = stream.clone();
+        this.videoStreams[remotePeerId] = clonnedStream.getVideoTracks()[0];
+        if (this.store.get().calling !== CallingState.OnCall) {
+          this.store.update({ calling: CallingState.OnCall });
+        }
+        if (this.callArgs) {
+          this.callArgs.onStream(stream, remotePeerId !== this.callID && isAgentId(remotePeerId));
+        }
+      }
+    };
+
+    // If the connection is lost, we end the call
+    pc.onconnectionstatechange = () => {
+      if (pc.connectionState === "disconnected" || pc.connectionState === "failed") {
+        this.onRemoteCallEnd();
+      }
+    };
+
+    // Handle track replacement when local video changes
+    if (this.callArgs && this.callArgs.localStream) {
+      this.callArgs.localStream.onVideoTrack((vTrack: MediaStreamTrack) => {
+        const sender = pc.getSenders().find((s) => s.track?.kind === 'video');
         if (!sender) {
-            console.warn('No video sender found');
+          logger.warn('No video sender found');
           return;
         }
         sender.replaceTrack(vTrack);
       });
-
-        call.on('stream', (stream) => {
-          this.videoStreams[call.peer] = stream.getVideoTracks()[0];
-          this.callArgs && this.callArgs.onStream(stream);
-        });
-
-        call.on('close', this.onRemoteCallEnd);
-        call.on('error', (e) => {
-          console.error('PeerJS error (on call):', e);
-          this.initiateCallEnd();
-          this.callArgs && this.callArgs.onError && this.callArgs.onError();
-        });
-      });
-      peer.on('error', (e) => {
-        if (e.type === 'disconnected') {
-          return peer.reconnect();
-        } else if (e.type !== 'peer-unavailable') {
-          console.error(`PeerJS error (on peer). Type ${e.type}`, e);
-        }
-      });
-
-      return new Promise((resolve) => {
-        peer.on('open', () => resolve(peer));
-      });
-    });
     }
 
+    return pc;
+  }
+
+  // ESTABLISHING A CONNECTION
+  private async _peerConnection({ remotePeerId, isAgent, socketId, localPeerId }: { remotePeerId: string, isAgent?: boolean, socketId?: string, localPeerId?: string }) {
+    try {
+      // Create RTCPeerConnection with client
+      const pc = await this.createPeerConnection({ remotePeerId, localPeerId, isAgent });
+      this.connections[remotePeerId] = pc;
+
+      // Create an SDP offer
+      const offer = await pc.createOffer();
+      await pc.setLocalDescription(offer);
+
+      // Sending offer
+      if (isAgent) {
+        this.socket.emit('WEBRTC_AGENT_CALL', { from: localPeerId, offer, toAgentId: socketId, type: WEBRTC_CALL_AGENT_EVENT_TYPES.OFFER });
+      } else {
+        this.socket.emit('webrtc_call_offer', { from: remotePeerId, offer });
+      }
+      this.connectAttempts = 0;
+    } catch (e: any) {
+      logger.error(e);
+      // Trying to reconnect
+      const tryReconnect = async (error: any) => {
+        if (error.type === 'peer-unavailable' && this.connectAttempts < 5) {
+          this.connectAttempts++;
+          logger.log('reconnecting', this.connectAttempts);
+          await new Promise((resolve) => setTimeout(resolve, 250));
+          await this._peerConnection({ remotePeerId });
+        } else {
+          logger.log('error', this.connectAttempts);
+          this.callArgs?.onError?.('Could not establish a connection with the peer after 5 attempts');
+        }
+      };
+      await tryReconnect(e);
+    }
+  }
+
+  // Process the received offer to answer
+  private async handleOffer(data: { from: string, offer: RTCSessionDescriptionInit }, isAgent?: boolean) {
+    // set to remotePeerId data.from
+    logger.log("RECEIVED OFFER", data);
+    const fromCallId = data.from;
+    let pc = this.connections[fromCallId];
+    if (!pc) {
+      if (isAgent) {
+        this.connections[fromCallId] = await this.createPeerConnection({ remotePeerId: fromCallId, isAgent, localPeerId: this.callID });
+        pc = this.connections[fromCallId];
+      } else {
+        logger.error("No connection found for remote peer", fromCallId);
+        return;
+      }
+    }
+    try {
+      // if the connection is not established yet, then set remoteDescription to peer
+      if (!pc.localDescription) {
+        await pc.setRemoteDescription(new RTCSessionDescription(data.offer));
+        const answer = await pc.createAnswer();
+        await pc.setLocalDescription(answer);
+        if (isAgent) {
+          this.socket.emit('WEBRTC_AGENT_CALL', { from: this.callID, answer, toAgentId: getSocketIdByCallId(fromCallId), type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER });
+        } else {
+          this.socket.emit('webrtc_call_answer', { from: fromCallId, answer });
+        }
+      } else {
+        logger.warn("Skipping setRemoteDescription: Already in stable state");
+      }
+    } catch (e) {
+      logger.error("Error setting remote description from answer", e);
+      this.callArgs?.onError?.(e);
+    }
+  }
+
+  // Process the received answer to offer
+  private async handleAnswer(data: { from: string, answer: RTCSessionDescriptionInit }, isAgent?: boolean) {
+    // set to remotePeerId data.from
+    logger.log("RECEIVED ANSWER", data);
+    if (this.agentInCallIds.includes(data.from) && !isAgent) {
+      return;
+    }
+    const callId = data.from;
+    const pc = this.connections[callId];
+    if (!pc) {
+      logger.error("No connection found for remote peer", callId, this.connections);
+      return;
+    }
+    try {
+      // if the connection is not established yet, then set remoteDescription to peer
+      if (pc.signalingState !== "stable") {
+        await pc.setRemoteDescription(new RTCSessionDescription(data.answer));
+      } else {
+        logger.warn("Skipping setRemoteDescription: Already in stable state");
+      }
+    } catch (e) {
+      logger.error("Error setting remote description from answer", e);
+      this.callArgs?.onError?.(e);
+    }
+  }
+
+  // process the received iceCandidate
+  private async handleIceCandidate(data: { from: string, candidate: RTCIceCandidateInit }) {
+    const callId = data.from;
+    const pc = this.connections[callId];
+    if (!pc) return;
+    // if there are ice candidates then add candidate to peer
+    if (data.candidate && (data.candidate.sdpMid || data.candidate.sdpMLineIndex !== null)) {
+      try {
+        await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
+      } catch (e) {
+        logger.error("Error adding ICE candidate", e);
+      }
+    } else {
+      logger.warn("Invalid ICE candidate skipped:", data.candidate);
+    }
+  }
+
+  // handle call ends
   private handleCallEnd() {
-    if (this.store.get().calling !== CallingState.NoCall) this.callArgs && this.callArgs.onCallEnd();
+    // If the call is not completed, then call onCallEnd
+    if (this.store.get().calling !== CallingState.NoCall) {
+      this.callArgs && this.callArgs.onCallEnd();
+    }
+    // change state to NoCall
     this.store.update({ calling: CallingState.NoCall });
-    this.callConnection[0] && this.callConnection[0].close();
+    // Close all created RTCPeerConnection
+    Object.values(this.connections).forEach((pc) => pc.close());
+    this.callArgs?.onCallEnd();
+    // Clear connections
+    this.connections = {};
+    this.callArgs = null;
+    this.videoStreams = {};
     this.callArgs = null;
-    // TODO:  We have it separated, right? (check)
-    //this.toggleAnnotation(false)
   }
 
+  // Call completion event handler by signal
   private onRemoteCallEnd = () => {
     if ([CallingState.Requesting, CallingState.Connecting].includes(this.store.get().calling)) {
+      // If the call has not started yet, then call onReject
       this.callArgs && this.callArgs.onReject();
-      this.callConnection[0] && this.callConnection[0].close();
+     // Close all connections and reset callArgs
+      Object.values(this.connections).forEach((pc) => pc.close());
+      this.connections = {};
+      this.callArgs?.onCallEnd();
       this.store.update({ calling: CallingState.NoCall });
       this.callArgs = null;
     } else {
+      // Call the full call completion handler
       this.handleCallEnd();
     }
   };
 
+  // Ends the call and sends the call_end signal
   initiateCallEnd = async () => {
-    const userName = userStore.account.name;
+    this.emitData('call_end', this.callID);
-    this.emitData('call_end', userName);
     this.handleCallEnd();
-    // TODO:  We have it separated, right? (check)
-    // const remoteControl = this.store.get().remoteControl
-    // if (remoteControl === RemoteControlStatus.Enabled) {
-    //   this.socket.emit("release_control")
-    //   this.toggleRemoteControl(false)
-    // }
   };
 
   private emitData = (event: string, data?: any) => {
@@ -188,7 +344,7 @@ export default class Call {
 
   private callArgs: {
     localStream: LocalStream;
-    onStream: (s: MediaStream) => void;
+    onStream: (s: MediaStream, isAgent: boolean) => void;
     onCallEnd: () => void;
     onReject: () => void;
     onError?: (arg?: any) => void;
@@ -196,7 +352,7 @@ export default class Call {
 
   setCallArgs(
     localStream: LocalStream,
-    onStream: (s: MediaStream) => void,
+    onStream: (s: MediaStream, isAgent: boolean) => void,
     onCallEnd: () => void,
     onReject: () => void,
     onError?: (e?: any) => void
@@ -210,117 +366,88 @@ export default class Call {
     };
   }
 
-  call(thirdPartyPeers?: string[]): { end: () => void } {
+  // Initiates a call
-    if (thirdPartyPeers && thirdPartyPeers.length > 0) {
+  call(): { end: () => void } {
-      this.addPeerCall(thirdPartyPeers);
-    } else {
     this._callSessionPeer();
-    }
+    // this.callAgentsInSession({ agentIds: this.agentInCallIds });
     return {
       end: this.initiateCallEnd,
     };
   }
 
+  // Notify peers of local video state change
   toggleVideoLocalStream(enabled: boolean) {
-    this.getPeer().then((peer) => {
+    this.emitData('videofeed', { streamId: this.callID, enabled });
-      this.emitData('videofeed', { streamId: peer.id, enabled });
-    });
   }
 
-  /** Connecting to the other agents that are already
+  // Calls the method to create a connection with a peer
-   *  in the call with the user
-   */
-  addPeerCall(thirdPartyPeers: string[]) {
-    thirdPartyPeers.forEach((peer) => this._peerConnection(peer));
-  }
-
-  /** Connecting to the app user */
   private _callSessionPeer() {
     if (![CallingState.NoCall, CallingState.Reconnecting].includes(this.store.get().calling)) {
       return;
     }
     this.store.update({ calling: CallingState.Connecting });
     const tab = this.store.get().currentTab;
-    if (!this.store.get().currentTab) {
+    if (!tab) {
-      console.warn('No tab data to connect to peer');
+      logger.warn('No tab data to connect to peer');
     }
-    const peerId =
+
-      this.getAssistVersion() === 1
+   // Generate a peer identifier depending on the assist version
-        ? this.peerID
+    this.callID = this.getCallId();
-        : `${this.peerID}-${tab || Object.keys(this.store.get().tabs)[0]}`;
       
     const userName = userStore.account.name;
     this.emitData('_agent_name', userName);
-    void this._peerConnection(peerId);
+    void this._peerConnection({ remotePeerId: this.callID });
   }
 
-  connectAttempts = 0;
+  private callAgentsInSession({ agentIds }: { agentIds: string[] }) {
-  private async _peerConnection(remotePeerId: string) {
+    if (agentIds) {
-    try {
+      const filteredAgentIds = agentIds.filter((id: string) => id.split('-')[3] !== this.agent.id.toString());
-      const peer = await this.getPeer();
+      const newIds = filteredAgentIds.filter((id: string) => !this.agentInCallIds.includes(id));
-      // let canCall = false
+      const removedIds = this.agentInCallIds.filter((id: string) => !filteredAgentIds.includes(id));
-
+      removedIds.forEach((id: string) => this.agentDisconnected(id));
-      const tryReconnect = async (e: any) => {
+      if (this.store.get().calling === CallingState.OnCall) {
-        peer.off('error', tryReconnect)
+        newIds.forEach((id: string) => {
-        console.log(e.type, this.connectAttempts);
+          const socketId = getSocketIdByCallId(id);
-        if (e.type === 'peer-unavailable' && this.connectAttempts < 5) {
+          this._peerConnection({ remotePeerId: id, isAgent: true, socketId, localPeerId: this.callID });
-          this.connectAttempts++;
-          console.log('reconnecting', this.connectAttempts);
-          await new Promise((resolve) => setTimeout(resolve, 250));
-          await this._peerConnection(remotePeerId);
-        } else {
-          console.log('error', this.connectAttempts);
-          this.callArgs?.onError?.('Could not establish a connection with the peer after 5 attempts');
-        }
-      }
-      const call = peer.call(remotePeerId, this.callArgs!.localStream.stream);
-      peer.on('error', tryReconnect);
-
-      peer.on('connection', () => {
-        this.callConnection.push(call);
-        this.connectAttempts = 0;
-
-        this.callArgs?.localStream.onVideoTrack((vTrack) => {
-          const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video');
-          if (!sender) {
-            console.warn('No video sender found');
-            return;
-          }
-          sender.replaceTrack(vTrack);
         });
-      })
+      }
 
-      call.on('stream', (stream) => {
+      this.agentInCallIds = filteredAgentIds;
-        this.store.get().calling !== CallingState.OnCall &&
-          this.store.update({ calling: CallingState.OnCall });
-
-        this.videoStreams[call.peer] = stream.getVideoTracks()[0];
-
-        this.callArgs && this.callArgs.onStream(stream);
-      });
-
-      call.on('close', this.onRemoteCallEnd);
-      call.on('error', (e) => {
-        console.error('PeerJS error (on call):', e);
-        this.initiateCallEnd();
-        this.callArgs && this.callArgs.onError && this.callArgs.onError();
-      });
-    } catch (e) {
-      console.error(e);
     }
   }
 
-  private cleaned: boolean = false;
+  private getCallId() {
+    const tab = this.store.get().currentTab;
+    if (!tab) {
+      logger.warn('No tab data to connect to peer');
+    }
 
+   // Generate a peer identifier depending on the assist version
+    return `${this.peerID}-${tab || Array.from(this.store.get().tabs)[0]}-${this.agent.id}-${this.socket.id}-agent`;
+  }
+
+  agentDisconnected(agentId: string) {
+    this.connections[agentId]?.close();
+    delete this.connections[agentId];
+  }
+
+  // Method for clearing resources
   clean() {
-    this.cleaned = true; // sometimes cleaned before modules loaded
     void this.initiateCallEnd();
-    if (this._peer) {
+    Object.values(this.connections).forEach((pc) => pc.close());
-      console.log('destroying peer...');
+    this.connections = {};
-      const peer = this._peer; // otherwise it calls reconnection on data chan close
+    this.callArgs?.onCallEnd();
-      this._peer = null;
-      peer.disconnect();
-      peer.destroy();
   }
 }
+
+function isAgentId(id: string): boolean {
+  return id.endsWith('_agent');
+}
+
+function getSocketIdByCallId(callId?: string): string | undefined {
+  const socketIdRegex = /-\d{2}-(.*?)\-agent/;
+  const match = callId?.match(socketIdRegex);
+  if (match) {
+      return match[1];
+  }
 }

frontend/app/player/web/assist/CanvasReceiver.ts

@@ -1,6 +1,6 @@
-import Peer from 'peerjs';
 import { VElement } from 'Player/web/managers/DOM/VirtualDOM';
 import MessageManager from 'Player/web/MessageManager';
+import { Socket } from 'socket.io-client';
 
 let frameCounter = 0;
 
@@ -18,70 +18,102 @@ function draw(
 
 export default class CanvasReceiver {
   private streams: Map<string, MediaStream> = new Map();
-  private peer: Peer | null = null;
+  // Store RTCPeerConnection for each remote peer
+  private connections: Map<string, RTCPeerConnection> = new Map();
+  private cId: string;
 
+  // sendSignal – for sending signals (offer/answer/ICE)
   constructor(
     private readonly peerIdPrefix: string,
     private readonly config: RTCIceServer[] | null,
     private readonly getNode: MessageManager['getNode'],
-    private readonly agentInfo: Record<string, any>
+    private readonly agentInfo: Record<string, any>,
+    private readonly socket: Socket,
   ) {
-    // @ts-ignore
+    // Form an id like in PeerJS
-    const urlObject = new URL(window.env.API_EDP || window.location.origin);
+    this.cId = `${this.peerIdPrefix}-${this.agentInfo.id}-canvas`;
-    const peerOpts: Peer.PeerJSOption = {
+
-      host: urlObject.hostname,
+    this.socket.on('webrtc_canvas_offer', (data: { data: { offer: RTCSessionDescriptionInit, id: string }}) => {
-      path: '/assist',
+      const { offer, id } = data.data;
-      port:
+      if (checkId(id, this.cId)) {
-        urlObject.port === ''
+        this.handleOffer(offer, id);
-        ? location.protocol === 'https:'
-          ? 443
-          : 80
-        : parseInt(urlObject.port),
-    };
-    if (this.config) {
-      peerOpts['config'] = {
-        iceServers: this.config,
-        //@ts-ignore
-        sdpSemantics: 'unified-plan',
-        iceTransportPolicy: 'all',
-      };
       }
-    const id = `${this.peerIdPrefix}-${this.agentInfo.id}-canvas`;
+    });
-    const canvasPeer = new Peer(id, peerOpts);
+    
-    this.peer = canvasPeer;
+    this.socket.on('webrtc_canvas_ice_candidate', (data: { data: { candidate: RTCIceCandidateInit, id: string }}) => {
-    canvasPeer.on('error', (err) => console.error('canvas peer error', err));
+      const {candidate, id } = data.data;
-    canvasPeer.on('call', (call) => {
+      if (checkId(id, this.cId)) {
-      call.answer();
+        this.handleCandidate(candidate, id);
-      const canvasId = call.peer.split('-')[2];
+      }
-      call.on('stream', (stream) => {
+    });
+
+    this.socket.on('webrtc_canvas_restart', () => {
+      this.clear();
+    });
+  }
+
+  async handleOffer(offer: RTCSessionDescriptionInit, id: string): Promise<void> {
+    const pc = new RTCPeerConnection({
+      iceServers: this.config ? this.config : [{ urls: "stun:stun.l.google.com:19302" }],
+    });
+
+    // Save the connection
+    this.connections.set(id, pc);
+
+    pc.onicecandidate = (event) => {
+      if (event.candidate) {
+        this.socket.emit('webrtc_canvas_ice_candidate', ({ candidate: event.candidate, id }));
+      }
+    };
+
+    pc.ontrack = (event) => {
+
+      const stream = event.streams[0];
+      if (stream) {
+        // Detect canvasId from remote peer id
+        const canvasId = id.split('-')[4];
         this.streams.set(canvasId, stream);
         setTimeout(() => {
           const node = this.getNode(parseInt(canvasId, 10));
-          const videoEl = spawnVideo(
+          const videoEl = spawnVideo(stream.clone() as MediaStream, node as VElement);
-            this.streams.get(canvasId)?.clone() as MediaStream,
-            node as VElement
-          );
           if (node) {
             draw(
               videoEl,
               node.node as HTMLCanvasElement,
               (node.node as HTMLCanvasElement).getContext('2d') as CanvasRenderingContext2D
             );
+          } else {
+            console.log('NODE', canvasId, 'IS NOT FOUND');
           }
         }, 250);
-      });
+      }
-      call.on('error', (err) => console.error('canvas call error', err));
+    };
-    });
+
+    await pc.setRemoteDescription(new RTCSessionDescription(offer));
+
+    const answer = await pc.createAnswer();
+    await pc.setLocalDescription(answer);
+
+    this.socket.emit('webrtc_canvas_answer', { answer: answer, id });
+  }
+
+  async handleCandidate(candidate: RTCIceCandidateInit, id: string): Promise<void> {
+    const pc = this.connections.get(id);
+    if (pc) {
+      try {
+        await pc.addIceCandidate(new RTCIceCandidate(candidate));
+      } catch (e) {
+        console.error('Error adding ICE candidate', e);
+      }
+    }
   }
 
   clear() {
-    if (this.peer) {
+    this.connections.forEach((pc) => {
-      // otherwise it calls reconnection on data chan close
+      pc.close();
-      const peer = this.peer;
+    });
-      this.peer = null;
+    this.connections.clear();
-      peer.disconnect();
+    this.streams.clear();
-      peer.destroy();
-    }
   }
 }
 
@@ -157,6 +189,10 @@ function spawnDebugVideo(stream: MediaStream, node: VElement) {
     });
 }
 
+function checkId(id: string, cId: string): boolean {
+  return id.includes(cId);
+}
+
 /** simple peer example
  * // @ts-ignore
  *     const peer = new SLPeer({ initiator: false })

frontend/package.json

@@ -51,7 +51,6 @@
     "mobx": "^6.13.3",
     "mobx-persist-store": "^1.1.5",
     "mobx-react-lite": "^4.0.7",
-    "peerjs": "1.3.2",
     "prismjs": "^1.29.0",
     "rc-time-picker": "^3.7.3",
     "react": "^18.2.0",

tracker/tracker-assist/layout/index-chat.html

@@ -1,5 +1,6 @@
 <!doctype html>
 <html lang="en">
+
 <head>
   <!-- Required meta tags -->
   <meta charset="utf-8">
@@ -13,9 +14,11 @@
       margin: 0;
       padding: 0;
     }
+
     .text-uppercase {
       text-transform: uppercase;
     }
+
     .connecting-message {
       /* margin-top: 50%; */
       font-size: 20px;
@@ -28,6 +31,7 @@
     .status-connecting .connecting-message {
       /* display: block; */
     }
+
     .status-connecting .card {
       /* display: none; */
     }
@@ -92,7 +96,8 @@
       border-bottom: solid thin #ccc;
     }
 
-#agent-name, #duration{
+    #agent-name,
+    #duration {
       cursor: default;
     }
 
@@ -108,15 +113,21 @@
       height: auto;
       object-fit: cover;
     }
-#local-stream, #remote-stream {
+
-  /* display:none;  */ /* TODO uncomment this line  */
+    #local-stream,
+    #remote-stream {
+      /* display:none;  */
+      /* TODO uncomment this line  */
     }
+
     #video-container.remote #remote-stream {
       display: block;
     }
+
     #video-container.local {
       min-height: 100px;
     }
+
     #video-container.local #local-stream {
       display: block;
     }
@@ -136,24 +147,30 @@
     #audio-btn {
       margin-right: 10px;
     }
+
     #audio-btn .bi-mic {
       fill: #CC0000;
     }
+
     #audio-btn .bi-mic-mute {
       display: none;
     }
+
     #audio-btn:after {
       /* text-transform: capitalize; */
       color: #CC0000;
       content: 'Mute';
       padding-left: 5px;
     }
+
     #audio-btn.muted .bi-mic-mute {
       display: inline-block;
     }
+
     #audio-btn.muted .bi-mic {
       display: none;
     }
+
     #audio-btn.muted:after {
       content: 'Unmute';
       padding-left: 5px;
@@ -163,22 +180,27 @@
     #video-btn .bi-camera-video {
       fill: #CC0000;
     }
+
     #video-btn .bi-camera-video-off {
       display: none;
     }
+
     #video-btn:after {
       /* text-transform: capitalize; */
       color: #CC0000;
       content: 'Stop Video';
       padding-left: 5px;
     }
+
     #video-btn.off:after {
       content: 'Start Video';
       padding-left: 5px;
     }
+
     #video-btn.off .bi-camera-video-off {
       display: inline-block;
     }
+
     #video-btn.off .bi-camera-video {
       display: none;
     }
@@ -191,12 +213,29 @@
       background-color: white;
     }
 
-#chat-card .chat-messages { display: none; }
+    #chat-card .chat-messages {
-#chat-card .chat-input { display: none; }
+      display: none;
-#chat-card .chat-header .arrow-state { transform: rotate(180deg); }
+    }
-#chat-card.active .chat-messages { display: flex; }
+
-#chat-card.active .chat-input {  display: flex; }
+    #chat-card .chat-input {
-#chat-card.active .chat-header .arrow-state { transform: rotate(0deg); }
+      display: none;
+    }
+
+    #chat-card .chat-header .arrow-state {
+      transform: rotate(180deg);
+    }
+
+    #chat-card.active .chat-messages {
+      display: flex;
+    }
+
+    #chat-card.active .chat-input {
+      display: flex;
+    }
+
+    #chat-card.active .chat-header .arrow-state {
+      transform: rotate(0deg);
+    }
 
     #chat-card .chat-header {
       border-bottom: solid thin #ccc;
@@ -235,6 +274,7 @@
       /* max-width: 70%; */
       width: fit-content;
     }
+
     #chat-card .message {
       margin-bottom: 15px;
     }
@@ -250,6 +290,7 @@
       font-weight: bold;
       color: #999999;
     }
+
     #chat-card .message .message-time {
       font-size: 12px;
       color: #999999;
@@ -300,9 +341,11 @@
       margin: auto;
       cursor: pointer;
     }
+
     .send-btn:hover {
       background-color: #999;
     }
+
     .send-btn svg {
       fill: #DDDDDD;
     }
@@ -310,6 +353,7 @@
     .confirm-window .title {
       margin-bottom: 10px;
     }
+
     .confirm-window {
       font: 14px 'Roboto', sans-serif;
       padding: 20px;
@@ -320,6 +364,7 @@
       color: #666666;
       display: none;
     }
+
     .confirm-window .actions {
       background-color: white;
       padding: 10px;
@@ -367,8 +412,10 @@
     <div class="title">Answer the call so the agent can assist.</div>
     <div class="actions">
       <button class="text-uppercase btn btn-lg btn-success" style="margin-right: 10px">
-          <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-telephone" viewBox="0 0 16 16">
+        <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-telephone"
-            <path d="M3.654 1.328a.678.678 0 0 0-1.015-.063L1.605 2.3c-.483.484-.661 1.169-.45 1.77a17.568 17.568 0 0 0 4.168 6.608 17.569 17.569 0 0 0 6.608 4.168c.601.211 1.286.033 1.77-.45l1.034-1.034a.678.678 0 0 0-.063-1.015l-2.307-1.794a.678.678 0 0 0-.58-.122l-2.19.547a1.745 1.745 0 0 1-1.657-.459L5.482 8.062a1.745 1.745 0 0 1-.46-1.657l.548-2.19a.678.678 0 0 0-.122-.58L3.654 1.328zM1.884.511a1.745 1.745 0 0 1 2.612.163L6.29 2.98c.329.423.445.974.315 1.494l-.547 2.19a.678.678 0 0 0 .178.643l2.457 2.457a.678.678 0 0 0 .644.178l2.189-.547a1.745 1.745 0 0 1 1.494.315l2.306 1.794c.829.645.905 1.87.163 2.611l-1.034 1.034c-.74.74-1.846 1.065-2.877.702a18.634 18.634 0 0 1-7.01-4.42 18.634 18.634 0 0 1-4.42-7.009c-.362-1.03-.037-2.137.703-2.877L1.885.511z"/>
+          viewBox="0 0 16 16">
+          <path
+            d="M3.654 1.328a.678.678 0 0 0-1.015-.063L1.605 2.3c-.483.484-.661 1.169-.45 1.77a17.568 17.568 0 0 0 4.168 6.608 17.569 17.569 0 0 0 6.608 4.168c.601.211 1.286.033 1.77-.45l1.034-1.034a.678.678 0 0 0-.063-1.015l-2.307-1.794a.678.678 0 0 0-.58-.122l-2.19.547a1.745 1.745 0 0 1-1.657-.459L5.482 8.062a1.745 1.745 0 0 1-.46-1.657l.548-2.19a.678.678 0 0 0-.122-.58L3.654 1.328zM1.884.511a1.745 1.745 0 0 1 2.612.163L6.29 2.98c.329.423.445.974.315 1.494l-.547 2.19a.678.678 0 0 0 .178.643l2.457 2.457a.678.678 0 0 0 .644.178l2.189-.547a1.745 1.745 0 0 1 1.494.315l2.306 1.794c.829.645.905 1.87.163 2.611l-1.034 1.034c-.74.74-1.846 1.065-2.877.702a18.634 18.634 0 0 1-7.01-4.42 18.634 18.634 0 0 1-4.42-7.009c-.362-1.03-.037-2.137.703-2.877L1.885.511z" />
         </svg>
         <span>Answer</span>
       </button>
@@ -386,13 +433,14 @@
         </div>
         <div class="call-duration">
           <!--Call Duration. -->
-                    <span id="duration" class="card-subtitle mb-2 text-muted fw-light"  data-bs-toggle="tooltip" data-bs-placement="bottom" title="Duration">00:00</span>
+          <span id="duration" class="card-subtitle mb-2 text-muted fw-light" data-bs-toggle="tooltip"
+            data-bs-placement="bottom" title="Duration">00:00</span>
         </div>
       </div>
       <div id="video-container" class="card-body bg-dark p-0 d-flex align-items-center position-relative">
-                <div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow">
+        <div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow scale-x-[-1]">
           <!-- <p class="text-white m-auto text-center">Starting video...</p> -->
-                    <video id="video-local" autoplay muted></video>
+          <video id="video-local" autoplay muted class="scale-x-[-1]"></video>
         </div>
 
         <div id="remote-stream" class="ratio ratio-4x3 m-0 p-0">
@@ -404,56 +452,60 @@
       <div class="card-footers">
         <div class="assist-controls">
           <!-- Add class .muted to #audio-btn when user  mutes audio -->
-                    <button
+          <button href="#" id="audio-btn" class="btn btn-light btn-sm text-uppercase me-2">
-                      href="#"
-                      id="audio-btn"
-                      class="btn btn-light btn-sm text-uppercase me-2"
-                    >
             <i>
               <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" class="bi bi-mic" viewBox="0 0 16 16">
-                          <path d="M3.5 6.5A.5.5 0 0 1 4 7v1a4 4 0 0 0 8 0V7a.5.5 0 0 1 1 0v1a5 5 0 0 1-4.5 4.975V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 .5-.5z"/>
+                <path
+                  d="M3.5 6.5A.5.5 0 0 1 4 7v1a4 4 0 0 0 8 0V7a.5.5 0 0 1 1 0v1a5 5 0 0 1-4.5 4.975V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 .5-.5z" />
                 <path d="M10 8a2 2 0 1 1-4 0V3a2 2 0 1 1 4 0v5zM8 0a3 3 0 0 0-3 3v5a3 3 0 0 0 6 0V3a3 3 0 0 0-3-3z" />
               </svg>
               <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" class="bi bi-mic-mute" viewBox="0 0 16 16">
-                          <path d="M13 8c0 .564-.094 1.107-.266 1.613l-.814-.814A4.02 4.02 0 0 0 12 8V7a.5.5 0 0 1 1 0v1zm-5 4c.818 0 1.578-.245 2.212-.667l.718.719a4.973 4.973 0 0 1-2.43.923V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 1 0v1a4 4 0 0 0 4 4zm3-9v4.879l-1-1V3a2 2 0 0 0-3.997-.118l-.845-.845A3.001 3.001 0 0 1 11 3z"/>
+                <path
-                          <path d="m9.486 10.607-.748-.748A2 2 0 0 1 6 8v-.878l-1-1V8a3 3 0 0 0 4.486 2.607zm-7.84-9.253 12 12 .708-.708-12-12-.708.708z"/>
+                  d="M13 8c0 .564-.094 1.107-.266 1.613l-.814-.814A4.02 4.02 0 0 0 12 8V7a.5.5 0 0 1 1 0v1zm-5 4c.818 0 1.578-.245 2.212-.667l.718.719a4.973 4.973 0 0 1-2.43.923V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 1 0v1a4 4 0 0 0 4 4zm3-9v4.879l-1-1V3a2 2 0 0 0-3.997-.118l-.845-.845A3.001 3.001 0 0 1 11 3z" />
+                <path
+                  d="m9.486 10.607-.748-.748A2 2 0 0 1 6 8v-.878l-1-1V8a3 3 0 0 0 4.486 2.607zm-7.84-9.253 12 12 .708-.708-12-12-.708.708z" />
               </svg>
             </i>
           </button>
 
           <!--Add class .off to #video-btn when user stops video -->
-                    <button
+          <button href="#" id="video-btn" class="btn btn-light btn-sm text-uppercase ms-2">
-                      href="#"
-                      id="video-btn"
-                      class="btn btn-light btn-sm text-uppercase ms-2"
-                    >
             <i>
-                        <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-camera-video" viewBox="0 0 16 16">
+              <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
-                          <path fill-rule="evenodd" d="M0 5a2 2 0 0 1 2-2h7.5a2 2 0 0 1 1.983 1.738l3.11-1.382A1 1 0 0 1 16 4.269v7.462a1 1 0 0 1-1.406.913l-3.111-1.382A2 2 0 0 1 9.5 13H2a2 2 0 0 1-2-2V5zm11.5 5.175 3.5 1.556V4.269l-3.5 1.556v4.35zM2 4a1 1 0 0 0-1 1v6a1 1 0 0 0 1 1h7.5a1 1 0 0 0 1-1V5a1 1 0 0 0-1-1H2z"/>
+                class="bi bi-camera-video" viewBox="0 0 16 16">
+                <path fill-rule="evenodd"
+                  d="M0 5a2 2 0 0 1 2-2h7.5a2 2 0 0 1 1.983 1.738l3.11-1.382A1 1 0 0 1 16 4.269v7.462a1 1 0 0 1-1.406.913l-3.111-1.382A2 2 0 0 1 9.5 13H2a2 2 0 0 1-2-2V5zm11.5 5.175 3.5 1.556V4.269l-3.5 1.556v4.35zM2 4a1 1 0 0 0-1 1v6a1 1 0 0 0 1 1h7.5a1 1 0 0 0 1-1V5a1 1 0 0 0-1-1H2z" />
               </svg>
-                        <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-camera-video-off" viewBox="0 0 16 16">
+              <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
-                          <path fill-rule="evenodd" d="M10.961 12.365a1.99 1.99 0 0 0 .522-1.103l3.11 1.382A1 1 0 0 0 16 11.731V4.269a1 1 0 0 0-1.406-.913l-3.111 1.382A2 2 0 0 0 9.5 3H4.272l.714 1H9.5a1 1 0 0 1 1 1v6a1 1 0 0 1-.144.518l.605.847zM1.428 4.18A.999.999 0 0 0 1 5v6a1 1 0 0 0 1 1h5.014l.714 1H2a2 2 0 0 1-2-2V5c0-.675.334-1.272.847-1.634l.58.814zM15 11.73l-3.5-1.555v-4.35L15 4.269v7.462zm-4.407 3.56-10-14 .814-.58 10 14-.814.58z"/>
+                class="bi bi-camera-video-off" viewBox="0 0 16 16">
+                <path fill-rule="evenodd"
+                  d="M10.961 12.365a1.99 1.99 0 0 0 .522-1.103l3.11 1.382A1 1 0 0 0 16 11.731V4.269a1 1 0 0 0-1.406-.913l-3.111 1.382A2 2 0 0 0 9.5 3H4.272l.714 1H9.5a1 1 0 0 1 1 1v6a1 1 0 0 1-.144.518l.605.847zM1.428 4.18A.999.999 0 0 0 1 5v6a1 1 0 0 0 1 1h5.014l.714 1H2a2 2 0 0 1-2-2V5c0-.675.334-1.272.847-1.634l.58.814zM15 11.73l-3.5-1.555v-4.35L15 4.269v7.462zm-4.407 3.56-10-14 .814-.58 10 14-.814.58z" />
               </svg>
             </i>
           </button>
 
         </div>
 
-                <button id="end-call-btn" href="#" class="btn btn-danger btn-sm text-uppercase" style="margin-right: 8px;">End</button>
+        <button id="end-call-btn" href="#" class="btn btn-danger btn-sm text-uppercase"
+          style="margin-right: 8px;">End</button>
       </div>
 
       <!-- CHAT - add .active class to show the messages and input -->
       <div id="chat-card" class="active">
         <div class="chat-header">
           <div class="chat-title">
-                  <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="bi bi-chat" viewBox="0 0 16 16">
+            <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="bi bi-chat"
-                    <path d="M2.678 11.894a1 1 0 0 1 .287.801 10.97 10.97 0 0 1-.398 2c1.395-.323 2.247-.697 2.634-.893a1 1 0 0 1 .71-.074A8.06 8.06 0 0 0 8 14c3.996 0 7-2.807 7-6 0-3.192-3.004-6-7-6S1 4.808 1 8c0 1.468.617 2.83 1.678 3.894zm-.493 3.905a21.682 21.682 0 0 1-.713.129c-.2.032-.352-.176-.273-.362a9.68 9.68 0 0 0 .244-.637l.003-.01c.248-.72.45-1.548.524-2.319C.743 11.37 0 9.76 0 8c0-3.866 3.582-7 8-7s8 3.134 8 7-3.582 7-8 7a9.06 9.06 0 0 1-2.347-.306c-.52.263-1.639.742-3.468 1.105z"/>
+              viewBox="0 0 16 16">
+              <path
+                d="M2.678 11.894a1 1 0 0 1 .287.801 10.97 10.97 0 0 1-.398 2c1.395-.323 2.247-.697 2.634-.893a1 1 0 0 1 .71-.074A8.06 8.06 0 0 0 8 14c3.996 0 7-2.807 7-6 0-3.192-3.004-6-7-6S1 4.808 1 8c0 1.468.617 2.83 1.678 3.894zm-.493 3.905a21.682 21.682 0 0 1-.713.129c-.2.032-.352-.176-.273-.362a9.68 9.68 0 0 0 .244-.637l.003-.01c.248-.72.45-1.548.524-2.319C.743 11.37 0 9.76 0 8c0-3.866 3.582-7 8-7s8 3.134 8 7-3.582 7-8 7a9.06 9.06 0 0 1-2.347-.306c-.52.263-1.639.742-3.468 1.105z" />
             </svg>
             <span>Chat</span>
           </div>
           <div>
-                  <svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" class="bi bi-chevron-up arrow-state" viewBox="0 0 16 16">
+            <svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" class="bi bi-chevron-up arrow-state"
-                    <path fill-rule="evenodd" d="M7.646 4.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1-.708.708L8 5.707l-5.646 5.647a.5.5 0 0 1-.708-.708l6-6z"/>
+              viewBox="0 0 16 16">
+              <path fill-rule="evenodd"
+                d="M7.646 4.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1-.708.708L8 5.707l-5.646 5.647a.5.5 0 0 1-.708-.708l6-6z" />
             </svg>
           </div>
         </div>
@@ -478,8 +530,10 @@
         <div class="chat-input">
           <input type="text" class="input" placeholder="Type a message...">
           <div class="send-btn">
-                  <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" class="bi bi-arrow-right-short" viewBox="0 0 16 16">
+            <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" class="bi bi-arrow-right-short"
-                    <path fill-rule="evenodd" d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z"/>
+              viewBox="0 0 16 16">
+              <path fill-rule="evenodd"
+                d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" />
             </svg>
           </div>
         </div>
@@ -487,4 +541,5 @@
     </div>
   </section>
 </body>
+
 </html>

tracker/tracker-assist/layout/index.html

@@ -152,8 +152,9 @@
         </div>
       </div>
       <div id="video-container" class="card-body bg-dark p-0 d-flex align-items-center position-relative">
-        <div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow">
+        <div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow scale-x-[-1]">
-          <video id="video-local" autoplay muted></video>
+          <!-- fix horizontal mirroring -->
+          <video id="video-local" autoplay muted class="scale-x-[-1]"></video>
         </div>
 
         <div id="remote-stream" class="ratio ratio-4x3 m-0 p-0">

tracker/tracker-assist/package.json

@@ -30,7 +30,6 @@
   "dependencies": {
     "csstype": "^3.0.10",
     "fflate": "^0.8.2",
-    "peerjs": "1.5.4",
     "socket.io-client": "^4.8.1"
   },
   "peerDependencies": {

tracker/tracker-assist/src/Assist.ts

@@ -1,25 +1,21 @@
 /* eslint-disable @typescript-eslint/no-empty-function */
-import type { Socket, } from 'socket.io-client'
+import type { Socket } from 'socket.io-client'
-import { connect, } from 'socket.io-client'
+import { connect } from 'socket.io-client'
-import Peer, { MediaConnection, } from 'peerjs'
+import type { Properties } from 'csstype'
-import type { Properties, } from 'csstype'
+import { App } from '@openreplay/tracker'
-import { App, } from '@openreplay/tracker'
 
-import RequestLocalStream, { LocalStream, } from './LocalStream.js'
+import RequestLocalStream, { LocalStream } from './LocalStream.js'
-import {hasTag,} from './guards.js'
+import { hasTag } from './guards.js'
-import RemoteControl, { RCStatus, } from './RemoteControl.js'
+import RemoteControl, { RCStatus } from './RemoteControl.js'
 import CallWindow from './CallWindow.js'
 import AnnotationCanvas from './AnnotationCanvas.js'
 import ConfirmWindow from './ConfirmWindow/ConfirmWindow.js'
-import { callConfirmDefault, } from './ConfirmWindow/defaults.js'
+import { callConfirmDefault } from './ConfirmWindow/defaults.js'
-import type { Options as ConfirmOptions, } from './ConfirmWindow/defaults.js'
+import type { Options as ConfirmOptions } from './ConfirmWindow/defaults.js'
 import ScreenRecordingState from './ScreenRecordingState.js'
-import { pkgVersion, } from './version.js'
+import { pkgVersion } from './version.js'
 import Canvas from './Canvas.js'
-import { gzip, } from 'fflate'
+import { gzip } from 'fflate'
-// TODO: fully specified strict check with no-any (everywhere)
-// @ts-ignore
-const safeCastedPeer = Peer.default || Peer
 
 type StartEndCallback = (agentInfo?: Record<string, any>) => ((() => any) | void)
 
@@ -52,25 +48,22 @@ export interface Options {
   confirmStyle?: Properties;
 
   config: RTCConfiguration;
-  serverURL: string
+  serverURL: string;
   callUITemplate?: string;
   compressionEnabled: boolean;
   /**
    * Minimum amount of messages in a batch to trigger compression run
    * @default 5000
-   * */
+   */
-  compressionMinBatchSize: number
+  compressionMinBatchSize: number;
 }
 
-
 enum CallingState {
   Requesting,
   True,
   False,
 }
 
-
-// TODO typing????
 type OptionalCallback = (() => Record<string, unknown>) | void
 type Agent = {
   onDisconnect?: OptionalCallback,
@@ -84,8 +77,8 @@ export default class Assist {
   readonly version = pkgVersion
 
   private socket: Socket | null = null
-  private peer: Peer | null = null
+  private calls: Map<string, RTCPeerConnection> = new Map();
-  private canvasPeers: Record<number, Peer | null> = {}
+  private canvasPeers: { [id: number]: RTCPeerConnection | null } = {}
   private canvasNodeCheckers: Map<number, any> = new Map()
   private assistDemandedRestart = false
   private callingState: CallingState = CallingState.False
@@ -257,7 +250,12 @@ export default class Assist {
       if (args[0] === 'messages' || args[0] === 'UPDATE_SESSION') {
         return
       }
+      if (args[0] !== 'webrtc_call_ice_candidate') {
         app.debug.log('Socket:', ...args)
+      };
+      socket.on('close', (e) => {
+        app.debug.warn('Socket closed:', e);
+      })
     })
 
     const onGrand = (id: string) => {
@@ -274,7 +272,6 @@ export default class Assist {
       return callingAgents.get(id)
     }
     const onRelease = (id?: string | null, isDenied?: boolean) => {
-      {
       if (id) {
         const cb = this.agents[id].onControlReleased
         delete this.agents[id].onControlReleased
@@ -295,7 +292,6 @@ export default class Assist {
         this.options.onRemoteControlDeny?.(info || {})
       }
     }
-    }
 
     this.remoteControl = new RemoteControl(
       this.options,
@@ -347,6 +343,7 @@ export default class Assist {
     socket.on('stopAnnotation', (id, event) => processEvent(id, event, annot?.stop))
 
     socket.on('NEW_AGENT', (id: string, info: AgentInfo) => {
+      this.cleanCanvasConnections();
       this.agents[id] = {
         onDisconnect: this.options.onAgentConnect?.(info),
         agentInfo: info, // TODO ?
@@ -369,7 +366,9 @@ export default class Assist {
           })
       }
     })
+
     socket.on('AGENTS_CONNECTED', (ids: string[]) => {
+      this.cleanCanvasConnections();
       ids.forEach(id => {
         const agentInfo = this.agents[id]?.agentInfo
         this.agents[id] = {
@@ -400,37 +399,65 @@ export default class Assist {
       this.agents[id]?.onDisconnect?.()
       delete this.agents[id]
 
+      Object.values(this.calls).forEach(pc => pc.close())
+      this.calls.clear();
+     
       recordingState.stopAgentRecording(id)
-      endAgentCall(id)
+      endAgentCall({ socketId: id })
     })
+
     socket.on('NO_AGENT', () => {
       Object.values(this.agents).forEach(a => a.onDisconnect?.())
+      this.cleanCanvasConnections();
       this.agents = {}
       if (recordingState.isActive) recordingState.stopRecording()
     })
-    socket.on('call_end', (id) => {
+
-      if (!callingAgents.has(id)) {
+    socket.on('call_end', (socketId, { data: callId }) => {
-        app.debug.warn('Received call_end from unknown agent', id)
+      if (!callingAgents.has(socketId)) {
+        app.debug.warn('Received call_end from unknown agent', socketId)
         return
       }
-      endAgentCall(id)
+
+      endAgentCall({ socketId, callId })
     })
 
     socket.on('_agent_name', (id, info) => {
       if (app.getTabId() !== info.meta.tabId) return
       const name = info.data
       callingAgents.set(id, name)
-
-      if (!this.peer) {
-        setupPeer()
-      }
       updateCallerNames()
     })
+
+    socket.on('webrtc_canvas_answer', async (_, data: { answer, id }) => {
+      const pc = this.canvasPeers[data.id];
+      if (pc) {
+        try {
+          await pc.setRemoteDescription(new RTCSessionDescription(data.answer));
+        } catch (e) {
+          app.debug.error('Error adding ICE candidate', e);
+        }
+      }
+    })
+
+    socket.on('webrtc_canvas_ice_candidate', async (_, data: { candidate, id }) => {
+      const pc = this.canvasPeers[data.id];
+      if (pc) {
+        try {
+          await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
+        } catch (e) {
+          app.debug.error('Error adding ICE candidate', e);
+        }
+      }
+    })
+
+    // If a videofeed arrives, then we show the video in the ui
     socket.on('videofeed', (_, info) => {
       if (app.getTabId() !== info.meta.tabId) return
       const feedState = info.data
       callUI?.toggleVideoStream(feedState)
     })
+
     socket.on('request_recording', (id, info) => {
       if (app.getTabId() !== info.meta.tabId) return
       const agentData = info.data
@@ -448,29 +475,52 @@ export default class Assist {
       }
     })
 
+    socket.on('webrtc_call_offer', async (_, data: { from: string, offer: RTCSessionDescriptionInit }) => {
+      if (!this.calls.has(data.from)) {
+        await handleIncomingCallOffer(data.from, data.offer);
+      }
+    });
+
+    socket.on('webrtc_call_ice_candidate', async (data: { from: string, candidate: RTCIceCandidateInit }) => {
+      const pc = this.calls[data.from];
+      if (pc) {
+        try {
+          await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
+        } catch (e) {
+          app.debug.error('Error adding ICE candidate', e);
+        }
+      }
+    });
+
     const callingAgents: Map<string, string> = new Map() // !! uses socket.io ID
     // TODO: merge peerId & socket.io id  (simplest way - send peerId with the name)
-    const calls: Record<string, MediaConnection> = {} // !! uses peerJS ID
     const lStreams: Record<string, LocalStream> = {}
 
     function updateCallerNames() {
       callUI?.setAssistentName(callingAgents)
     }
-    function endAgentCall(id: string) {
+    function endAgentCall({ socketId, callId }: { socketId: string, callId?: string }) {
-      callingAgents.delete(id)
+      callingAgents.delete(socketId)
+
       if (callingAgents.size === 0) {
         handleCallEnd()
       } else {
         updateCallerNames()
-        //TODO: close() specific call and corresponding lStreams (after connecting peerId & socket.io id)
+        if (callId) {
+          handleCallEndWithAgent(callId)
         }
       }
-    const handleCallEnd = () => { // Complete stop and clear all calls
+    }
-      // Streams
+
-      Object.values(calls).forEach(call => call.close())
+    const handleCallEndWithAgent = (id: string) => {
-      Object.keys(calls).forEach(peerId => {
+      this.calls.get(id)?.close()
-        delete calls[peerId]
+      this.calls.delete(id)
-      })
+    }
+
+    // call end handling
+    const handleCallEnd = () => {
+      Object.values(this.calls).forEach(pc => pc.close())
+      this.calls.clear();
       Object.values(lStreams).forEach((stream) => { stream.stop() })
       Object.keys(lStreams).forEach((peerId: string) => { delete lStreams[peerId] })
       // UI
@@ -484,7 +534,7 @@ export default class Assist {
         callUI?.hideControls()
       }
 
-      this.emit('UPDATE_SESSION', { agentIds: [], isCallActive: false, })
+      this.emit('UPDATE_SESSION', { agentIds: [], isCallActive: false })
       this.setCallingState(CallingState.False)
       sessionStorage.removeItem(this.options.session_calling_peer_key)
 
@@ -498,175 +548,192 @@ export default class Assist {
       }
     }
 
-    // PeerJS call (todo: use native WebRTC)
+    const handleIncomingCallOffer = async (from: string, offer: RTCSessionDescriptionInit) => {
-    const peerOptions = {
+      app.debug.log('handleIncomingCallOffer', from)
-      host: this.getHost(),
-      path: this.getBasePrefixUrl()+'/assist',
-      port: location.protocol === 'http:' && this.noSecureMode ? 80 : 443,
-      debug: 2, //appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs.
-    }
-    const setupPeer = () => {
-      if (this.options.config) {
-        peerOptions['config'] = this.options.config
-      }
-
-      const peer = new safeCastedPeer(peerID, peerOptions) as Peer
-      this.peer = peer
-      let peerReconnectAttempts = 0
-      // @ts-ignore (peerjs typing)
-      peer.on('error', e => app.debug.warn('Peer error: ', e.type, e))
-      peer.on('disconnected', () => {
-        if (peerReconnectAttempts < 30) {
-          this.peerReconnectTimeout = setTimeout(() => {
-            if (this.app.active() && !peer.destroyed) {
-              peer.reconnect()
-            }
-          }, Math.min(peerReconnectAttempts, 8) * 2 * 1000)
-          peerReconnectAttempts += 1
-        }
-      })
-
-
-      const requestCallConfirm = () => {
-        if (callConfirmAnswer) { // Already asking
-          return callConfirmAnswer
-        }
-        callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || {
-          text: this.options.confirmText,
-          style: this.options.confirmStyle,
-        })) // TODO: reuse ?
-        return callConfirmAnswer = callConfirmWindow.mount().then(answer => {
-          closeCallConfirmWindow()
-          return answer
-        })
-      }
-
-      const initiateCallEnd = () => {
-        this.emit('call_end')
-        handleCallEnd()
-      }
-      const updateVideoFeed = ({ enabled, }) => this.emit('videofeed', { streamId: this.peer?.id, enabled, })
-
-      peer.on('call', (call) => {
-        app.debug.log('Incoming call from', call.peer)
       let confirmAnswer: Promise<boolean>
       const callingPeerIds = JSON.parse(sessionStorage.getItem(this.options.session_calling_peer_key) || '[]')
-        if (callingPeerIds.includes(call.peer) || this.callingState === CallingState.True) {
+      // if the caller is already in the list, then we immediately accept the call without ui
+      if (callingPeerIds.includes(from) || this.callingState === CallingState.True) {
         confirmAnswer = Promise.resolve(true)
       } else {
+        // set the state to wait for confirmation
         this.setCallingState(CallingState.Requesting)
+       // call the call confirmation window
         confirmAnswer = requestCallConfirm()
-          this.playNotificationSound() // For every new agent during confirmation here
+        // sound notification of a call
+        this.playNotificationSound()
 
-          // TODO: only one (latest) timeout
+        // after 30 seconds we drop the call
         setTimeout(() => {
           if (this.callingState !== CallingState.Requesting) { return }
           initiateCallEnd()
         }, 30000)
       }
 
-        confirmAnswer.then(async agreed => {
+      try {
+        // waiting for a decision on accepting the challenge
+        const agreed = await confirmAnswer
+        // if rejected, then terminate the call
         if (!agreed) {
           initiateCallEnd()
           this.options.onCallDeny?.()
           return
         }
-          // Request local stream for the new connection
-          try {
-            // lStreams are reusable so fare we don't delete them in the `endAgentCall`
-            if (!lStreams[call.peer]) {
-              app.debug.log('starting new stream for', call.peer)
-              lStreams[call.peer] = await RequestLocalStream()
-            }
-            calls[call.peer] = call
-          } catch (e) {
-            app.debug.error('Audio media device request error:', e)
-            initiateCallEnd()
-            return
-          }
-
         if (!callUI) {
           callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
-            callUI.setVideoToggleCallback(updateVideoFeed)
+          callUI.setVideoToggleCallback((args: { enabled: boolean }) =>
+            this.emit('videofeed', { streamId: from, enabled: args.enabled })
+          );
         }
+        // show buttons in the call window
         callUI.showControls(initiateCallEnd)
-
         if (!annot) {
           annot = new AnnotationCanvas()
           annot.mount()
         }
-          // have to be updated
-          callUI.setLocalStreams(Object.values(lStreams))
 
-          call.on('error', e => {
+        // callUI.setLocalStreams(Object.values(lStreams))
-            app.debug.warn('Call error:', e)
+        try {
-            initiateCallEnd()
+         // if there are no local streams in lStrems then we set
-          })
+          if (!lStreams[from]) {
-          call.on('stream', (rStream) => {
+            app.debug.log('starting new stream for', from)
-            callUI?.addRemoteStream(rStream, call.peer)
+            // request a local stream, and set it to lStreams
-            const onInteraction = () => { // do only if document.hidden ?
+            lStreams[from] = await RequestLocalStream()
-              callUI?.playRemote()
-              document.removeEventListener('click', onInteraction)
           }
-            document.addEventListener('click', onInteraction)
+         // we pass the received tracks to Call ui
-          })
+          callUI.setLocalStreams(Object.values(lStreams))
+        } catch (e) {
+          app.debug.error('Error requesting local stream', e);
+          // if something didn't work out, we terminate the call
+          initiateCallEnd();
+          return;
+        }
+        // create a new RTCPeerConnection with ice server config
+        const pc = new RTCPeerConnection({
+          iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
+        });
 
-          // remote video on/off/camera change
+        // get all local tracks and add them to RTCPeerConnection
-          lStreams[call.peer].onVideoTrack(vTrack => {
+        lStreams[from].stream.getTracks().forEach(track => {
-            const sender = call.peerConnection.getSenders().find(s => s.track?.kind === 'video')
+          pc.addTrack(track, lStreams[from].stream);
+        });
+
+        // When we receive local ice candidates, we emit them via socket
+        pc.onicecandidate = (event) => {
+          if (event.candidate) {
+            socket.emit('webrtc_call_ice_candidate', { from, candidate: event.candidate });
+          }
+        };
+
+        // when we get a remote stream, add it to call ui
+        pc.ontrack = (event) => {
+          const rStream = event.streams[0];
+          if (rStream && callUI) {
+            callUI.addRemoteStream(rStream, from);
+            const onInteraction = () => {
+              callUI?.playRemote();
+              document.removeEventListener('click', onInteraction);
+            };
+            document.addEventListener('click', onInteraction);
+          }
+        };
+
+        // Keep connection with the caller
+        this.calls.set(from, pc);
+
+        // set remote description on incoming request
+        await pc.setRemoteDescription(new RTCSessionDescription(offer));
+        // create a response to the incoming request
+        const answer = await pc.createAnswer();
+        // set answer as local description
+        await pc.setLocalDescription(answer);
+        // set the response as local
+        socket.emit('webrtc_call_answer', { from, answer });
+
+        // If the state changes to an error, we terminate the call
+        // pc.onconnectionstatechange = () => {
+        //   if (pc.connectionState === 'disconnected' || pc.connectionState === 'failed') {
+        //     initiateCallEnd();
+        //   }
+        // };
+
+        // Update track when local video changes
+        lStreams[from].onVideoTrack(vTrack => {
+          const sender = pc.getSenders().find(s => s.track?.kind === 'video');
           if (!sender) {
             app.debug.warn('No video sender found')
             return
           }
-            app.debug.log('sender found:', sender)
+          sender.replaceTrack(vTrack)
-            void sender.replaceTrack(vTrack)
         })
 
-          call.answer(lStreams[call.peer].stream)
+        // if the user closed the tab or switched, then we end the call
         document.addEventListener('visibilitychange', () => {
           initiateCallEnd()
         })
 
+        // when everything is set, we change the state to true
         this.setCallingState(CallingState.True)
         if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() }
-
+        const callingPeerIdsNow = Array.from(this.calls.keys())
-          const callingPeerIds = Object.keys(calls)
+        // in session storage we write down everyone with whom the call is established
-          sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIds))
+        sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIdsNow))
-          this.emit('UPDATE_SESSION', { agentIds: callingPeerIds, isCallActive: true, })
+        this.emit('UPDATE_SESSION', { agentIds: callingPeerIdsNow, isCallActive: true })
-        }).catch(reason => { // in case of Confirm.remove() without user answer (not an error)
+      } catch (reason) {
-          app.debug.log(reason)
+        app.debug.log(reason);
-        })
-      })
       }
+    };
 
+    // Functions for requesting confirmation, ending a call, notifying, etc.
+    const requestCallConfirm = () => {
+      if (callConfirmAnswer) { // If confirmation has already been requested
+        return callConfirmAnswer;
+      }
+      callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || {
+        text: this.options.confirmText,
+        style: this.options.confirmStyle,
+      }));
+      return callConfirmAnswer = callConfirmWindow.mount().then(answer => {
+        closeCallConfirmWindow();
+        return answer;
+      });
+    };
 
-    const startCanvasStream = (stream: MediaStream, id: number) => {
+    const initiateCallEnd = () => {
-      const canvasPID = `${app.getProjectKey()}-${sessionId}-${id}`
+      this.emit('call_end');
-      if (!this.canvasPeers[id]) {
+      handleCallEnd();
-        this.canvasPeers[id] = new safeCastedPeer(canvasPID, peerOptions) as Peer
+    };
-      }
+
-      this.canvasPeers[id]?.on('error', (e) => app.debug.error(e))
+    const startCanvasStream = async (stream: MediaStream, id: number) => {
+      for (const agent of Object.values(this.agents)) {
+        if (!agent.agentInfo) return;
+
+        const uniqueId = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas-${id}`;
+
+        if (!this.canvasPeers[uniqueId]) {
+          this.canvasPeers[uniqueId] = new RTCPeerConnection({
+            iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
+          });
+          this.setupPeerListeners(uniqueId);
+
+          stream.getTracks().forEach((track) => {
+            this.canvasPeers[uniqueId]?.addTrack(track, stream);
+          });
+
+          // Create SDP offer
+          const offer = await this.canvasPeers[uniqueId].createOffer();
+          await this.canvasPeers[uniqueId].setLocalDescription(offer);
+
+          // Send offer via signaling server
+          socket.emit('webrtc_canvas_offer', { offer, id: uniqueId });
 
-      Object.values(this.agents).forEach(agent => {
-        if (agent.agentInfo) {
-          const target = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas`
-          const connection = this.canvasPeers[id]?.connect(target)
-          connection?.on('open', () => {
-            if (agent.agentInfo) {
-              const call = this.canvasPeers[id]?.call(target, stream.clone())
-              call?.on('error', app.debug.error)
         }
-          })
-          connection?.on('error',  (e) => app.debug.error(e))
-        } else {
-          app.debug.error('Assist: cant establish canvas peer to agent, no agent info')
       }
-      })
     }
 
     app.nodes.attachNodeCallback((node) => {
       const id = app.nodes.getID(node)
-      if (id && hasTag(node, 'canvas')) {
+      if (id && hasTag(node, 'canvas') && !app.sanitizer.isHidden(id)) {
         app.debug.log(`Creating stream for canvas ${id}`)
         const canvasHandler = new Canvas(
           node as unknown as HTMLCanvasElement,
@@ -686,14 +753,30 @@ export default class Assist {
           if (!isPresent) {
             canvasHandler.stop()
             this.canvasMap.delete(id)
-            this.canvasPeers[id]?.destroy()
+            if (this.canvasPeers[id]) {
+              this.canvasPeers[id]?.close()
               this.canvasPeers[id] = null
+            }
             clearInterval(int)
           }
         }, 5000)
         this.canvasNodeCheckers.set(id, int)
       }
-    })
+    });
+  }
+
+  private setupPeerListeners(id: string) {
+    const peer = this.canvasPeers[id];
+    if (!peer) return;
+    // ICE candidates
+    peer.onicecandidate = (event) => {
+      if (event.candidate && this.socket) {
+        this.socket.emit('webrtc_canvas_ice_candidate', {
+          candidate: event.candidate,
+          id,
+        });
+      }
+    };
   }
 
   private playNotificationSound() {
@@ -706,26 +789,32 @@ export default class Assist {
     }
   }
 
+  // clear all data
   private clean() {
     // sometimes means new agent connected, so we keep id for control
-    this.remoteControl?.releaseControl(false, true)
+    this.remoteControl?.releaseControl(false, true);
     if (this.peerReconnectTimeout) {
       clearTimeout(this.peerReconnectTimeout)
       this.peerReconnectTimeout = null
     }
-    if (this.peer) {
+    this.cleanCanvasConnections();
-      this.peer.destroy()
+    Object.values(this.calls).forEach(pc => pc.close())
-      this.app.debug.log('Peer destroyed')
+    this.calls.clear();
-    }
     if (this.socket) {
       this.socket.disconnect()
       this.app.debug.log('Socket disconnected')
     }
     this.canvasMap.clear()
-    this.canvasPeers = []
+    this.canvasPeers = {}
     this.canvasNodeCheckers.forEach((int) => clearInterval(int))
     this.canvasNodeCheckers.clear()
   }
+
+  private cleanCanvasConnections() {
+    Object.values(this.canvasPeers).forEach(pc => pc?.close())
+    this.canvasPeers = {}
+    this.socket?.emit('webrtc_canvas_restart')
+  }
 }
 
 /** simple peers impl

tracker/tracker-assist/src/CallWindow.ts

@@ -12,6 +12,7 @@ export default class CallWindow {
 	private videoBtn: HTMLElement | null = null
 	private endCallBtn: HTMLElement | null = null
 	private agentNameElem: HTMLElement | null = null
+	private remoteStreamVideoContainerSample: HTMLElement | null = null
 	private videoContainer: HTMLElement | null = null
 	private vPlaceholder: HTMLElement | null = null
 	private remoteControlContainer: HTMLElement | null = null
@@ -62,7 +63,6 @@ export default class CallWindow {
 					this.adjustIframeSize()
 					iframe.onload = null
 				}
-
 				// ?
 				text = text.replace(/href="css/g, `href="${baseHref}/css`)
 				doc.open()
@@ -71,6 +71,7 @@ export default class CallWindow {
 
 				this.vLocal = doc.getElementById('video-local') as HTMLVideoElement | null
 				this.vRemote = doc.getElementById('video-remote') as HTMLVideoElement | null
+				
 				this.videoContainer = doc.getElementById('video-container')
 				
 				this.audioBtn = doc.getElementById('audio-btn')

tracker/tracker-redux/package.json

@@ -28,14 +28,16 @@
     "redux": "^4.0.0"
   },
   "devDependencies": {
+    "@babel/core": "^7.26.8",
     "@openreplay/tracker": "file:../tracker",
-    "prettier": "^1.18.2",
-    "replace-in-files-cli": "^1.0.0",
-    "typescript": "^4.6.0-dev.20211126",
     "@rollup/plugin-babel": "^6.0.4",
     "@rollup/plugin-node-resolve": "^15.2.3",
+    "prettier": "^1.18.2",
     "replace-in-files": "^3.0.0",
+    "replace-in-files-cli": "^1.0.0",
     "rollup": "^4.14.0",
-    "rollup-plugin-terser": "^7.0.2"
+    "rollup-plugin-terser": "^7.0.2",
-  }
+    "typescript": "^4.6.0-dev.20211126"
+  },
+  "packageManager": "yarn@4.6.0+sha512.5383cc12567a95f1d668fbe762dfe0075c595b4bfff433be478dbbe24e05251a8e8c3eb992a986667c1d53b6c3a9c85b8398c35a960587fbd9fa3a0915406728"
 }