diff --git a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx index b3c1ea884..a5bdc0b6d 100644 --- a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx +++ b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx @@ -82,7 +82,7 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) { { stream: MediaStream; isAgent: boolean }[] | null >([]); const [localStream, setLocalStream] = useState(null); - const [callObject, setCallObject] = useState<{ end: () => void } | null>( + const [callObject, setCallObject] = useState<{ end: () => void } | null | undefined>( null, ); @@ -135,6 +135,7 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) { }, [peerConnectionStatus]); const addIncomeStream = (stream: MediaStream, isAgent: boolean) => { + if (!stream.active) return; setIncomeStream((oldState) => { if (oldState === null) return [{ stream, isAgent }]; if ( @@ -149,13 +150,8 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) { }); }; - const removeIncomeStream = (stream: MediaStream) => { - setIncomeStream((prevState) => { - if (!prevState) return []; - return prevState.filter( - (existingStream) => existingStream.stream.id !== stream.id, - ); - }); + const removeIncomeStream = () => { + setIncomeStream([]); }; function onReject() { @@ -181,7 +177,12 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) { () => { player.assistManager.ping(AssistActionsPing.call.end, agentId); lStream.stop.apply(lStream); - removeIncomeStream(lStream.stream); + removeIncomeStream(); + }, + () => { + player.assistManager.ping(AssistActionsPing.call.end, agentId); + lStream.stop.apply(lStream); + removeIncomeStream(); }, onReject, onError, diff --git a/frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx b/frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx index 3899d1ea0..812d4b9b9 100644 --- a/frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx +++ b/frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx @@ -34,43 +34,40 @@ function VideoContainer({ } const iid = setInterval(() => { const track = stream.getVideoTracks()[0]; - const settings = track?.getSettings(); - const isDummyVideoTrack = settings - ? settings.width === 2 || - settings.frameRate === 0 || - (!settings.frameRate && !settings.width) - : true; - const shouldBeEnabled = track.enabled && !isDummyVideoTrack; - if (isEnabled !== shouldBeEnabled) { - setEnabled(shouldBeEnabled); - setRemoteEnabled?.(shouldBeEnabled); + if (track) { + if (!track.enabled) { + setEnabled(false); + setRemoteEnabled?.(false); + } else { + setEnabled(true); + setRemoteEnabled?.(true); + } + } else { + setEnabled(false); + setRemoteEnabled?.(false); } }, 500); return () => clearInterval(iid); - }, [stream, isEnabled]); + }, [stream]); return (
-
); } diff --git a/frontend/app/player/web/assist/Call.ts b/frontend/app/player/web/assist/Call.ts index 592945e95..f5116c089 100644 --- a/frontend/app/player/web/assist/Call.ts +++ b/frontend/app/player/web/assist/Call.ts @@ -185,8 +185,7 @@ export default class Call { pc.ontrack = (event) => { const stream = event.streams[0]; if (stream && !this.videoStreams[remotePeerId]) { - const clonnedStream = stream.clone(); - this.videoStreams[remotePeerId] = clonnedStream.getVideoTracks()[0]; + this.videoStreams[remotePeerId] = stream.getVideoTracks()[0]; if (this.store.get().calling !== CallingState.OnCall) { this.store.update({ calling: CallingState.OnCall }); } @@ -305,22 +304,18 @@ export default class Call { } try { // if the connection is not established yet, then set remoteDescription to peer - if (!pc.localDescription) { - await pc.setRemoteDescription(new RTCSessionDescription(data.offer)); - const answer = await pc.createAnswer(); - await pc.setLocalDescription(answer); - if (isAgent) { - this.socket.emit('WEBRTC_AGENT_CALL', { - from: this.callID, - answer, - toAgentId: getSocketIdByCallId(fromCallId), - type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER, - }); - } else { - this.socket.emit('webrtc_call_answer', { from: fromCallId, answer }); - } + await pc.setRemoteDescription(new RTCSessionDescription(data.offer)); + const answer = await pc.createAnswer(); + await pc.setLocalDescription(answer); + if (isAgent) { + this.socket.emit('WEBRTC_AGENT_CALL', { + from: this.callID, + answer, + toAgentId: getSocketIdByCallId(fromCallId), + type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER, + }); } else { - logger.warn('Skipping setRemoteDescription: Already in stable state'); + this.socket.emit('webrtc_call_answer', { from: fromCallId, answer }); } } catch (e) { logger.error('Error setting remote description from answer', e); @@ -388,13 +383,13 @@ export default class Call { private handleCallEnd() { // If the call is not completed, then call onCallEnd if (this.store.get().calling !== CallingState.NoCall) { - this.callArgs && this.callArgs.onCallEnd(); + this.callArgs && this.callArgs.onRemoteCallEnd(); } // change state to NoCall this.store.update({ calling: CallingState.NoCall }); // Close all created RTCPeerConnection Object.values(this.connections).forEach((pc) => pc.close()); - this.callArgs?.onCallEnd(); + this.callArgs?.onRemoteCallEnd(); // Clear connections this.connections = {}; this.callArgs = null; @@ -414,7 +409,7 @@ export default class Call { // Close all connections and reset callArgs Object.values(this.connections).forEach((pc) => pc.close()); this.connections = {}; - this.callArgs?.onCallEnd(); + this.callArgs?.onRemoteCallEnd(); this.store.update({ calling: CallingState.NoCall }); this.callArgs = null; } else { @@ -443,7 +438,8 @@ export default class Call { private callArgs: { localStream: LocalStream; onStream: (s: MediaStream, isAgent: boolean) => void; - onCallEnd: () => void; + onRemoteCallEnd: () => void; + onLocalCallEnd: () => void; onReject: () => void; onError?: (arg?: any) => void; } | null = null; @@ -451,14 +447,16 @@ export default class Call { setCallArgs( localStream: LocalStream, onStream: (s: MediaStream, isAgent: boolean) => void, - onCallEnd: () => void, + onRemoteCallEnd: () => void, + onLocalCallEnd: () => void, onReject: () => void, onError?: (e?: any) => void, ) { this.callArgs = { localStream, onStream, - onCallEnd, + onRemoteCallEnd, + onLocalCallEnd, onReject, onError, }; @@ -549,7 +547,7 @@ export default class Call { void this.initiateCallEnd(); Object.values(this.connections).forEach((pc) => pc.close()); this.connections = {}; - this.callArgs?.onCallEnd(); + this.callArgs?.onLocalCallEnd(); } } diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 41b3190e0..5a4fb9437 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -548,6 +548,16 @@ export default class Assist { } } + const renegotiateConnection = async ({ pc, from }: { pc: RTCPeerConnection, from: string }) => { + try { + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); + this.emit('webrtc_call_offer', { from, offer }); + } catch (error) { + app.debug.error("Error with renegotiation:", error); + } + }; + const handleIncomingCallOffer = async (from: string, offer: RTCSessionDescriptionInit) => { app.debug.log('handleIncomingCallOffer', from) let confirmAnswer: Promise @@ -572,56 +582,59 @@ export default class Assist { try { // waiting for a decision on accepting the challenge - const agreed = await confirmAnswer + const agreed = await confirmAnswer; // if rejected, then terminate the call if (!agreed) { - initiateCallEnd() - this.options.onCallDeny?.() - return - } - if (!callUI) { - callUI = new CallWindow(app.debug.error, this.options.callUITemplate) - callUI.setVideoToggleCallback((args: { enabled: boolean }) => - this.emit('videofeed', { streamId: from, enabled: args.enabled }) - ); - } - // show buttons in the call window - callUI.showControls(initiateCallEnd) - if (!annot) { - annot = new AnnotationCanvas() - annot.mount() - } - - // callUI.setLocalStreams(Object.values(lStreams)) - try { - // if there are no local streams in lStrems then we set - if (!lStreams[from]) { - app.debug.log('starting new stream for', from) - // request a local stream, and set it to lStreams - lStreams[from] = await RequestLocalStream() - } - // we pass the received tracks to Call ui - callUI.setLocalStreams(Object.values(lStreams)) - } catch (e) { - app.debug.error('Error requesting local stream', e); - // if something didn't work out, we terminate the call initiateCallEnd(); + this.options.onCallDeny?.(); return; } - // create a new RTCPeerConnection with ice server config + + // create a new RTCPeerConnection with ice server config const pc = new RTCPeerConnection({ iceServers: [{ urls: "stun:stun.l.google.com:19302" }], }); - // get all local tracks and add them to RTCPeerConnection - lStreams[from].stream.getTracks().forEach(track => { - pc.addTrack(track, lStreams[from].stream); - }); + if (!callUI) { + callUI = new CallWindow(app.debug.error, this.options.callUITemplate); + callUI.setVideoToggleCallback((args: { enabled: boolean }) => { + this.emit("videofeed", { streamId: from, enabled: args.enabled }) + }); + } + // show buttons in the call window + callUI.showControls(initiateCallEnd); + if (!annot) { + annot = new AnnotationCanvas(); + annot.mount(); + } + + + // callUI.setLocalStreams(Object.values(lStreams)) + try { + // if there are no local streams in lStrems then we set + if (!lStreams[from]) { + app.debug.log("starting new stream for", from); + // request a local stream, and set it to lStreams + lStreams[from] = await RequestLocalStream(pc, renegotiateConnection.bind(null, { pc, from })); + } + // we pass the received tracks to Call ui + callUI.setLocalStreams(Object.values(lStreams)); + } catch (e) { + app.debug.error("Error requesting local stream", e); + // if something didn't work out, we terminate the call + initiateCallEnd(); + return; + } + + // get all local tracks and add them to RTCPeerConnection // When we receive local ice candidates, we emit them via socket pc.onicecandidate = (event) => { if (event.candidate) { - socket.emit('webrtc_call_ice_candidate', { from, candidate: event.candidate }); + socket.emit("webrtc_call_ice_candidate", { + from, + candidate: event.candidate, + }); } }; @@ -632,9 +645,9 @@ export default class Assist { callUI.addRemoteStream(rStream, from); const onInteraction = () => { callUI?.playRemote(); - document.removeEventListener('click', onInteraction); + document.removeEventListener("click", onInteraction); }; - document.addEventListener('click', onInteraction); + document.addEventListener("click", onInteraction); } }; @@ -648,7 +661,7 @@ export default class Assist { // set answer as local description await pc.setLocalDescription(answer); // set the response as local - socket.emit('webrtc_call_answer', { from, answer }); + socket.emit("webrtc_call_answer", { from, answer }); // If the state changes to an error, we terminate the call // pc.onconnectionstatechange = () => { @@ -658,27 +671,35 @@ export default class Assist { // }; // Update track when local video changes - lStreams[from].onVideoTrack(vTrack => { - const sender = pc.getSenders().find(s => s.track?.kind === 'video'); + lStreams[from].onVideoTrack((vTrack) => { + const sender = pc.getSenders().find((s) => s.track?.kind === "video"); if (!sender) { - app.debug.warn('No video sender found') - return + app.debug.warn("No video sender found"); + return; } - sender.replaceTrack(vTrack) - }) + sender.replaceTrack(vTrack); + }); // if the user closed the tab or switched, then we end the call - document.addEventListener('visibilitychange', () => { - initiateCallEnd() - }) + document.addEventListener("visibilitychange", () => { + initiateCallEnd(); + }); // when everything is set, we change the state to true - this.setCallingState(CallingState.True) - if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() } - const callingPeerIdsNow = Array.from(this.calls.keys()) + this.setCallingState(CallingState.True); + if (!callEndCallback) { + callEndCallback = this.options.onCallStart?.(); + } + const callingPeerIdsNow = Array.from(this.calls.keys()); // in session storage we write down everyone with whom the call is established - sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIdsNow)) - this.emit('UPDATE_SESSION', { agentIds: callingPeerIdsNow, isCallActive: true }) + sessionStorage.setItem( + this.options.session_calling_peer_key, + JSON.stringify(callingPeerIdsNow) + ); + this.emit("UPDATE_SESSION", { + agentIds: callingPeerIdsNow, + isCallActive: true, + }); } catch (reason) { app.debug.log(reason); } diff --git a/tracker/tracker-assist/src/CallWindow.ts b/tracker/tracker-assist/src/CallWindow.ts index f8bd31796..fd9cfb4ee 100644 --- a/tracker/tracker-assist/src/CallWindow.ts +++ b/tracker/tracker-assist/src/CallWindow.ts @@ -48,7 +48,7 @@ export default class CallWindow { } // const baseHref = "https://static.openreplay.com/tracker-assist/test" - const baseHref = 'https://static.openreplay.com/tracker-assist/4.0.0' + const baseHref = 'https://static.openreplay.com/tracker-assist/widget' // this.load = fetch(this.callUITemplate || baseHref + '/index2.html') this.load = fetch(this.callUITemplate || baseHref + '/index.html') .then((r) => r.text()) @@ -60,7 +60,7 @@ export default class CallWindow { }, 0) //iframe.style.height = doc.body.scrollHeight + 'px'; //iframe.style.width = doc.body.scrollWidth + 'px'; - this.adjustIframeSize() + this.adjustIframeSize() iframe.onload = null } // ? @@ -152,15 +152,6 @@ export default class CallWindow { if (this.checkRemoteVideoInterval) { clearInterval(this.checkRemoteVideoInterval) } // just in case - let enabled = false - this.checkRemoteVideoInterval = setInterval(() => { - const settings = this.remoteVideo?.getSettings() - const isDummyVideoTrack = !this.remoteVideo.enabled || (!!settings && (settings.width === 2 || settings.frameRate === 0)) - const shouldBeEnabled = !isDummyVideoTrack - if (enabled !== shouldBeEnabled) { - this.toggleRemoteVideoUI((enabled = shouldBeEnabled)) - } - }, 1000) } // Audio diff --git a/tracker/tracker-assist/src/LocalStream.ts b/tracker/tracker-assist/src/LocalStream.ts index ef3a135e1..ffda37792 100644 --- a/tracker/tracker-assist/src/LocalStream.ts +++ b/tracker/tracker-assist/src/LocalStream.ts @@ -1,88 +1,86 @@ -declare global { - interface HTMLCanvasElement { - captureStream(frameRate?: number): MediaStream; - } -} - -function dummyTrack(): MediaStreamTrack { - const canvas = document.createElement('canvas')//, { width: 0, height: 0}) - canvas.setAttribute('data-openreplay-hidden', '1') - canvas.width=canvas.height=2 // Doesn't work when 1 (?!) - const ctx = canvas.getContext('2d') - ctx?.fillRect(0, 0, canvas.width, canvas.height) - requestAnimationFrame(function draw(){ - ctx?.fillRect(0,0, canvas.width, canvas.height) - requestAnimationFrame(draw) - }) - // Also works. Probably it should be done once connected. - //setTimeout(() => { ctx?.fillRect(0,0, canvas.width, canvas.height) }, 4000) - return canvas.captureStream(60).getTracks()[0] -} - -export default function RequestLocalStream(): Promise { - return navigator.mediaDevices.getUserMedia({ audio:true, }) - .then(aStream => { - const aTrack = aStream.getAudioTracks()[0] - - if (!aTrack) { throw new Error('No audio tracks provided') } - return new _LocalStream(aTrack) - }) +export default function RequestLocalStream( + pc: RTCPeerConnection, + toggleVideoCb?: () => void +): Promise { + return navigator.mediaDevices + .getUserMedia({ audio: true, video: false }) + .then((stream) => { + const aTrack = stream.getAudioTracks()[0]; + if (!aTrack) { + throw new Error("No audio tracks provided"); + } + stream.getTracks().forEach((track) => { + pc.addTrack(track, stream); + }); + return new _LocalStream(stream, pc, toggleVideoCb); + }); } class _LocalStream { - private mediaRequested = false - readonly stream: MediaStream - private readonly vdTrack: MediaStreamTrack - constructor(aTrack: MediaStreamTrack) { - this.vdTrack = dummyTrack() - this.stream = new MediaStream([ aTrack, this.vdTrack, ]) + private mediaRequested = false; + readonly stream: MediaStream; + readonly vTrack: MediaStreamTrack; + readonly pc: RTCPeerConnection; + readonly toggleVideoCb?: () => void; + constructor(stream: MediaStream, pc: RTCPeerConnection, toggleVideoCb?: () => void) { + this.stream = stream; + this.pc = pc; + this.toggleVideoCb = toggleVideoCb; } toggleVideo(): Promise { + const videoTracks = this.stream.getVideoTracks(); if (!this.mediaRequested) { - return navigator.mediaDevices.getUserMedia({video:true,}) - .then(vStream => { - const vTrack = vStream.getVideoTracks()[0] - if (!vTrack) { - throw new Error('No video track provided') - } - this.stream.addTrack(vTrack) - this.stream.removeTrack(this.vdTrack) - this.mediaRequested = true - if (this.onVideoTrackCb) { - this.onVideoTrackCb(vTrack) - } - return true - }) - .catch(e => { - // TODO: log - console.error(e) - return false - }) + return navigator.mediaDevices + .getUserMedia({ video: true }) + .then((vStream) => { + const vTrack = vStream.getVideoTracks()[0]; + if (!vTrack) { + throw new Error("No video track provided"); + } + + this.pc.addTrack(vTrack, this.stream); + this.stream.addTrack(vTrack); + + if (this.toggleVideoCb) { + this.toggleVideoCb(); + } + + this.mediaRequested = true; + + if (this.onVideoTrackCb) { + this.onVideoTrackCb(vTrack); + } + return true; + }) + .catch((e) => { + // TODO: log + return false; + }); + } else { + videoTracks.forEach((track) => { + track.enabled = !track.enabled; + }); } - let enabled = true - this.stream.getVideoTracks().forEach(track => { - track.enabled = enabled = enabled && !track.enabled - }) - return Promise.resolve(enabled) + return Promise.resolve(videoTracks[0].enabled); } toggleAudio(): boolean { - let enabled = true - this.stream.getAudioTracks().forEach(track => { - track.enabled = enabled = enabled && !track.enabled - }) - return enabled + let enabled = true; + this.stream.getAudioTracks().forEach((track) => { + track.enabled = enabled = enabled && !track.enabled; + }); + return enabled; } - private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null + private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null; onVideoTrack(cb: (t: MediaStreamTrack) => void) { - this.onVideoTrackCb = cb + this.onVideoTrackCb = cb; } stop() { - this.stream.getTracks().forEach(t => t.stop()) + this.stream.getTracks().forEach((t) => t.stop()); } } -export type LocalStream = InstanceType +export type LocalStream = InstanceType;