diff --git a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx index bd31a3983..e595c9c6e 100644 --- a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx +++ b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx @@ -142,8 +142,11 @@ function AssistActions({ }); }; - const removeIncomeStream = () => { - setIncomeStream([]); + const removeIncomeStream = (stream: MediaStream) => { + setIncomeStream((prevState) => { + if (!prevState) return []; + return prevState.filter((existingStream) => existingStream.stream.id !== stream.id); + }); }; function call() { diff --git a/frontend/app/player/web/assist/AssistManager.ts b/frontend/app/player/web/assist/AssistManager.ts index b97a46512..ac87d25b6 100644 --- a/frontend/app/player/web/assist/AssistManager.ts +++ b/frontend/app/player/web/assist/AssistManager.ts @@ -335,7 +335,7 @@ export default class AssistManager { /** * Sends event ping to stats service * */ - public ping(event: StatsEvent, id: number) { + public ping(event: StatsEvent, id: string) { this.socket?.emit(event, id); } diff --git a/frontend/app/player/web/assist/Call.ts b/frontend/app/player/web/assist/Call.ts index 5880fa6f5..2d9fbdbe1 100644 --- a/frontend/app/player/web/assist/Call.ts +++ b/frontend/app/player/web/assist/Call.ts @@ -46,13 +46,11 @@ export default class Call { ) { socket.on('WEBRTC_AGENT_CALL', (data) => { - console.log("!WEBRTC AGENT CALL RECEIVED", data.type, Object.keys(this.connections)); switch (data.type) { case WEBRTC_CALL_AGENT_EVENT_TYPES.OFFER: this.handleOffer(data, true); break; case WEBRTC_CALL_AGENT_EVENT_TYPES.ICE_CANDIDATE: - console.log("#### RECEIVED ICE CANDIDATE", data); this.handleIceCandidate(data); break; case WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER: @@ -63,7 +61,6 @@ export default class Call { }) socket.on('UPDATE_SESSION', (data: { data: { agentIds: string[] }}) => { - console.log("UPDATE SESSION", data.data.agentIds); this.callAgentsInSession({ agentIds: data.data.agentIds }); }); @@ -104,12 +101,10 @@ export default class Call { }); socket.on('webrtc_call_offer', (data: { data: { from: string, offer: RTCSessionDescriptionInit } }) => { - console.log("RECEIVED OFFER", data); this.handleOffer(data.data); }); socket.on('webrtc_call_answer', (data: { data: { from: string, answer: RTCSessionDescriptionInit } }) => { - console.log("ПРИШЕЛ оБЫЧНЫЙ сОКЕТ ANSWER", data.data); this.handleAnswer(data.data); }); socket.on('webrtc_call_ice_candidate', (data: { data: { from: string, candidate: RTCIceCandidateInit } }) => { @@ -127,10 +122,6 @@ export default class Call { iceServers: [{ urls: "stun:stun.l.google.com:19302" }], }); - if (isAgent) { - console.log('!!! CREATED PC FOR AGENT', remotePeerId); - } - // If there is a local stream, add its tracks to the connection if (this.callArgs && this.callArgs.localStream && this.callArgs.localStream.stream) { this.callArgs.localStream.stream.getTracks().forEach((track) => { @@ -142,7 +133,6 @@ export default class Call { pc.onicecandidate = (event) => { if (event.candidate) { if (isAgent) { - console.log("!!! SEND ICE CANDIDATE for", getSocketIdByCallId(remotePeerId)); this.socket.emit('WEBRTC_AGENT_CALL', { from: localPeerId, candidate: event.candidate, toAgentId: getSocketIdByCallId(remotePeerId), type: WEBRTC_CALL_AGENT_EVENT_TYPES.ICE_CANDIDATE }); } else { this.socket.emit('webrtc_call_ice_candidate', { from: remotePeerId, candidate: event.candidate }); @@ -157,7 +147,6 @@ export default class Call { const stream = event.streams[0]; if (stream && !this.videoStreams[remotePeerId]) { const clonnedStream = stream.clone(); - console.log('SETTING VIDEOTRACKS TO', remotePeerId); this.videoStreams[remotePeerId] = clonnedStream.getVideoTracks()[0]; if (this.store.get().calling !== CallingState.OnCall) { this.store.update({ calling: CallingState.OnCall }); @@ -192,7 +181,6 @@ export default class Call { // ESTABLISHING A CONNECTION private async _peerConnection({ remotePeerId, isAgent, socketId, localPeerId }: { remotePeerId: string, isAgent?: boolean, socketId?: string, localPeerId?: string }) { - console.log("_ PEER CONNECTION", remotePeerId); try { // Create RTCPeerConnection with client const pc = await this.createPeerConnection({ remotePeerId, localPeerId, isAgent }); @@ -201,11 +189,9 @@ export default class Call { // Create an SDP offer const offer = await pc.createOffer(); await pc.setLocalDescription(offer); - console.log("1 !!! CREATED PC WITH OFFER FOR AN AGENT", pc, offer); // Sending offer if (isAgent) { - console.log("2 !!! SENDING OFFER TO AGENT", socketId); this.socket.emit('WEBRTC_AGENT_CALL', { from: localPeerId, offer, toAgentId: socketId, type: WEBRTC_CALL_AGENT_EVENT_TYPES.OFFER }); } else { this.socket.emit('webrtc_call_offer', { from: remotePeerId, offer }); @@ -232,14 +218,13 @@ export default class Call { // Process the received offer to answer private async handleOffer(data: { from: string, offer: RTCSessionDescriptionInit }, isAgent?: boolean) { // set to remotePeerId data.from + logger.log("RECEIVED OFFER", data); const fromCallId = data.from; let pc = this.connections[fromCallId]; - console.log("3 !!! HANDLE OFFER", isAgent, pc); if (!pc) { if (isAgent) { this.connections[fromCallId] = await this.createPeerConnection({ remotePeerId: fromCallId, isAgent, localPeerId: this.callID }); pc = this.connections[fromCallId]; - console.log("4 CREATED NEW PC FOR INCOMING OFFER", pc); } else { logger.error("No connection found for remote peer", fromCallId); return; @@ -252,10 +237,8 @@ export default class Call { const answer = await pc.createAnswer(); await pc.setLocalDescription(answer); if (isAgent) { - console.log("4 !!! SENDING ANSWER TO AGENT", getSocketIdByCallId(fromCallId)); this.socket.emit('WEBRTC_AGENT_CALL', { from: this.callID, answer, toAgentId: getSocketIdByCallId(fromCallId), type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER }); } else { - console.log("4.1 !!! SENDING PLAIN ANSWER SOCKET TO AGENT", getSocketIdByCallId(fromCallId)); this.socket.emit('webrtc_call_answer', { from: fromCallId, answer }); } } else { @@ -270,7 +253,7 @@ export default class Call { // Process the received answer to offer private async handleAnswer(data: { from: string, answer: RTCSessionDescriptionInit }, isAgent?: boolean) { // set to remotePeerId data.from - console.log("5 !!! HANDLE ANSWER", this.connections, data.from, this.connections[data.from]); + logger.log("RECEIVED ANSWER", data); if (this.agentInCallIds.includes(data.from) && !isAgent) { return; } @@ -295,7 +278,6 @@ export default class Call { // process the received iceCandidate private async handleIceCandidate(data: { from: string, candidate: RTCIceCandidateInit }) { - console.log("### GET ICE CANDIDATE", data); const callId = data.from; const pc = this.connections[callId]; if (!pc) return; @@ -325,6 +307,8 @@ export default class Call { // Clear connections this.connections = {}; this.callArgs = null; + this.videoStreams = {}; + this.callArgs = null; } // Call completion event handler by signal @@ -384,7 +368,6 @@ export default class Call { // Initiates a call call(): { end: () => void } { - console.log("INTIATE CALL", this.agentIds); this._callSessionPeer(); // this.callAgentsInSession({ agentIds: this.agentInCallIds }); return { @@ -397,12 +380,6 @@ export default class Call { this.emitData('videofeed', { streamId: this.callID, enabled }); } - // Connect with other agents - // addPeerCall(thirdPartyPeers: string[]) { - // console.log("ADD THRID PARTY CALL", thirdPartyPeers); - // thirdPartyPeers.forEach((peerId) => this._peerConnection(peerId)); - // } - // Calls the method to create a connection with a peer private _callSessionPeer() { if (![CallingState.NoCall, CallingState.Reconnecting].includes(this.store.get().calling)) { @@ -425,16 +402,12 @@ export default class Call { private callAgentsInSession({ agentIds }: { agentIds: string[] }) { if (agentIds) { const filteredAgentIds = agentIds.filter((id: string) => id.split('-')[3] !== this.agent.id.toString()); - console.log("!!! FILTERED AGENT IDS", filteredAgentIds, this.agentInCallIds); const newIds = filteredAgentIds.filter((id: string) => !this.agentInCallIds.includes(id)); - console.log("!!! NEW AGENT IDS", newIds); const removedIds = this.agentInCallIds.filter((id: string) => !filteredAgentIds.includes(id)); - console.log("!!! REMOVED AGENT IDS", removedIds); removedIds.forEach((id: string) => this.agentDisconnected(id)); if (this.store.get().calling === CallingState.OnCall) { newIds.forEach((id: string) => { const socketId = getSocketIdByCallId(id); - console.log("!!! INITIALISING CALL WITH AgENT", id); this._peerConnection({ remotePeerId: id, isAgent: true, socketId, localPeerId: this.callID }); }); } diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index e8616e44a..d66242184 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -502,8 +502,6 @@ export default class Assist { function endAgentCall({ socketId, callId }: { socketId: string, callId?: string }) { callingAgents.delete(socketId) - console.log("CALLING AGENTS", callingAgents) - if (callingAgents.size === 0) { handleCallEnd() } else { @@ -618,7 +616,6 @@ export default class Assist { // get all local tracks and add them to RTCPeerConnection lStreams[from].stream.getTracks().forEach(track => { - console.log('GETTING TRACKS FROM', from); pc.addTrack(track, lStreams[from].stream); }); @@ -633,10 +630,9 @@ export default class Assist { pc.ontrack = (event) => { const rStream = event.streams[0]; if (rStream && callUI) { - console.log('2 GETTING TRACKS FROM', from); callUI.addRemoteStream(rStream, from); const onInteraction = () => { - callUI?.playRemote(from); + callUI?.playRemote(); document.removeEventListener('click', onInteraction); }; document.addEventListener('click', onInteraction); diff --git a/tracker/tracker-assist/src/CallWindow.ts b/tracker/tracker-assist/src/CallWindow.ts index 60d2dbee2..f8bd31796 100644 --- a/tracker/tracker-assist/src/CallWindow.ts +++ b/tracker/tracker-assist/src/CallWindow.ts @@ -3,246 +3,10 @@ import attachDND from './dnd.js' const SS_START_TS_KEY = '__openreplay_assist_call_start_ts' -const text = ` - - - - - - - - OpenReplay | Assist - - - - - - - - - - -
-
-
Connecting...
-
-
- -
- - 00:00 -
-
-
-
- - -
- -
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
-
- End -
-
-
-
-
This tab has remote control access
- -
-
-
- - - -` - export default class CallWindow { + private remoteVideoId: string private readonly iframe: HTMLIFrameElement - private vRemote: Map = new Map() + private vRemote: HTMLVideoElement | null = null private vLocal: HTMLVideoElement | null = null private audioBtn: HTMLElement | null = null private videoBtn: HTMLElement | null = null @@ -256,8 +20,7 @@ export default class CallWindow { private controlsContainer: HTMLElement | null = null private onToggleVideo: (args: any) => void private tsInterval: ReturnType - private remoteVideos: Map = new Map() - private vContainer: HTMLDivElement | null = null + private remoteVideo: MediaStreamTrack private readonly load: Promise @@ -289,7 +52,7 @@ export default class CallWindow { // this.load = fetch(this.callUITemplate || baseHref + '/index2.html') this.load = fetch(this.callUITemplate || baseHref + '/index.html') .then((r) => r.text()) - .then(() => { + .then((text) => { iframe.onload = () => { const assistSection = doc.getElementById('or-assist') setTimeout(() => { @@ -301,15 +64,13 @@ export default class CallWindow { iframe.onload = null } // ? - const newText = text.replace(/href="css/g, `href="${baseHref}/css`) + text = text.replace(/href="css/g, `href="${baseHref}/css`) doc.open() - doc.write(newText) + doc.write(text) doc.close() this.vLocal = doc.getElementById('video-local') as HTMLVideoElement | null - // this.vRemote = doc.getElementById('video-remote') as HTMLVideoElement | null - this.remoteStreamVideoContainerSample = doc.querySelector('[data-attr="remote-stream"]'); - this.remoteStreamVideoContainerSample?.remove(); + this.vRemote = doc.getElementById('video-remote') as HTMLVideoElement | null this.videoContainer = doc.getElementById('video-container') @@ -378,52 +139,30 @@ export default class CallWindow { this.load .then(() => { // Video - console.log('VREMOTE', this.vRemote); - if (!this.vRemote.has(peerId)) { - if (this.remoteStreamVideoContainerSample && this.videoContainer) { - const newRemoteStreamVideoContainer = this.remoteStreamVideoContainerSample.cloneNode(true) as HTMLDivElement; - newRemoteStreamVideoContainer.setAttribute("data-peer-id", peerId); - - const videoElement = document.createElement("video"); - videoElement.autoplay = true; - newRemoteStreamVideoContainer.appendChild(videoElement); - const clonedStream = rStream.clone() - videoElement.srcObject = clonedStream - - const videoElementTest = document.createElement("video"); - videoElementTest.autoplay = true; - videoElementTest.srcObject = clonedStream; - videoElementTest.setAttribute("data-peer-id", peerId); - document.body.appendChild(videoElementTest); - - this.remoteVideos.set(peerId, clonedStream.getVideoTracks()[0]) - console.log("ADD REMOTE STREAM", clonedStream.getVideoTracks()[0]); - if (this.vPlaceholder) { - this.vPlaceholder.innerText = - 'Video has been paused. Click anywhere to resume.' + if (this.vRemote && !this.vRemote.srcObject) { + this.vRemote.srcObject = rStream + this.remoteVideo = rStream.getVideoTracks()[0] + this.remoteVideoId = peerId + if (this.vPlaceholder) { + this.vPlaceholder.innerText = + 'Video has been paused. Click anywhere to resume.' + } + // Hack to determine if the remote video is enabled + // TODO: pass this info through socket + if (this.checkRemoteVideoInterval) { + clearInterval(this.checkRemoteVideoInterval) + } // just in case + let enabled = false + this.checkRemoteVideoInterval = setInterval(() => { + const settings = this.remoteVideo?.getSettings() + const isDummyVideoTrack = !this.remoteVideo.enabled || (!!settings && (settings.width === 2 || settings.frameRate === 0)) + const shouldBeEnabled = !isDummyVideoTrack + if (enabled !== shouldBeEnabled) { + this.toggleRemoteVideoUI((enabled = shouldBeEnabled)) } - this.vRemote.set(peerId, videoElement) - this.videoContainer.appendChild(newRemoteStreamVideoContainer); - console.log('ДОБАВИЛИ ВИДЕО В ДОМ для', peerId); - } + }, 1000) } - // Hack to determine if the remote video is enabled - // TODO: pass this info through socket - if (this.checkRemoteVideoInterval) { - clearInterval(this.checkRemoteVideoInterval) - } // just in case - let enabled = false - this.checkRemoteVideoInterval = setInterval(() => { - const settings = this.remoteVideos.get(peerId)?.getSettings() - const isDummyVideoTrack = !this.remoteVideos.get(peerId)?.enabled || (!!settings && (settings.width === 2 || settings.frameRate === 0)) - const shouldBeEnabled = !isDummyVideoTrack - if (enabled !== shouldBeEnabled) { - this.toggleRemoteVideoUI((enabled = shouldBeEnabled)) - } - }, 1000) - - // Audio if (!this.audioContainer) { this.audioContainer = document.createElement('div') @@ -461,13 +200,8 @@ export default class CallWindow { this.localStreams = streams } - playRemote(peerId: string) { - if (this.vRemote.has(peerId)) { - const vRemote = this.vRemote.get(peerId) - if (vRemote) { - vRemote.play() - } - } + playRemote() { + this.vRemote && this.vRemote.play() } setAssistentName(callingAgents: Map) { @@ -596,16 +330,9 @@ export default class CallWindow { } toggleVideoStream({ streamId, enabled, }: { streamId: string, enabled: boolean }) { - if (this.remoteVideos.has(streamId)) { - console.log("TRYING TO ENABLE", this.remoteVideos.get[streamId]) - const track = this.remoteVideos.get(streamId) - if (track) { - console.log('ENABLING TRACK', track); - track.enabled = enabled - } - if (this.vRemote.size <= 1) { - this.toggleRemoteVideoUI(enabled) - } + if (this.remoteVideoId === streamId) { + this.remoteVideo.enabled = enabled + this.toggleRemoteVideoUI(enabled) } } -} +} \ No newline at end of file