jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

updated widget link (#3158)

Browse source 
* updated widget link

* fix calls

* updated widget url
This commit is contained in:
Andrey Babushkin 2025-03-18 13:07:09 +03:00 • committed by GitHub
parent 4c967d4bc1
commit 7365d8639c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 195 additions and 189 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
frontend/app/components/Assist/components/AssistActions/AssistActions.tsx
View file

@ -82,7 +82,7 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
{ stream: MediaStream; isAgent: boolean }[] | null { stream: MediaStream; isAgent: boolean }[] | null
>([]); >([]);
const [localStream, setLocalStream] = useState<LocalStream | null>(null); const [localStream, setLocalStream] = useState<LocalStream | null>(null);
const [callObject, setCallObject] = useState<{ end: () => void } | null>( const [callObject, setCallObject] = useState<{ end: () => void } | null | undefined>(
null, null,
); );
@ -135,6 +135,7 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
}, [peerConnectionStatus]); }, [peerConnectionStatus]);
const addIncomeStream = (stream: MediaStream, isAgent: boolean) => { const addIncomeStream = (stream: MediaStream, isAgent: boolean) => {
if (!stream.active) return;
setIncomeStream((oldState) => { setIncomeStream((oldState) => {
if (oldState === null) return [{ stream, isAgent }]; if (oldState === null) return [{ stream, isAgent }];
if ( if (
@ -149,13 +150,8 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
}); });
}; };
const removeIncomeStream = (stream: MediaStream) => { const removeIncomeStream = () => {
setIncomeStream((prevState) => { setIncomeStream([]);
if (!prevState) return [];
return prevState.filter(
(existingStream) => existingStream.stream.id !== stream.id,
);
});
}; };
function onReject() { function onReject() {
@ -181,7 +177,12 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
() => { () => {
player.assistManager.ping(AssistActionsPing.call.end, agentId); player.assistManager.ping(AssistActionsPing.call.end, agentId);
lStream.stop.apply(lStream); lStream.stop.apply(lStream);
removeIncomeStream(lStream.stream); removeIncomeStream();
},
() => {
player.assistManager.ping(AssistActionsPing.call.end, agentId);
lStream.stop.apply(lStream);
removeIncomeStream();
}, },
onReject, onReject,
onError, onError,

43
frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx
View file

@ -34,43 +34,40 @@ function VideoContainer({
} }
const iid = setInterval(() => { const iid = setInterval(() => {
const track = stream.getVideoTracks()[0]; const track = stream.getVideoTracks()[0];
const settings = track?.getSettings();
const isDummyVideoTrack = settings
? settings.width === 2 ||
settings.frameRate === 0 ||
(!settings.frameRate && !settings.width)
: true;
const shouldBeEnabled = track.enabled && !isDummyVideoTrack;
if (isEnabled !== shouldBeEnabled) { if (track) {
setEnabled(shouldBeEnabled); if (!track.enabled) {
setRemoteEnabled?.(shouldBeEnabled); setEnabled(false);
setRemoteEnabled?.(false);
} else {
setEnabled(true);
setRemoteEnabled?.(true);
}
} else {
setEnabled(false);
setRemoteEnabled?.(false);
} }
}, 500); }, 500);
return () => clearInterval(iid); return () => clearInterval(iid);
}, [stream, isEnabled]); }, [stream]);
return ( return (
<div <div
className="flex-1" className="flex-1"
style={{ style={{
display: isEnabled ? undefined : 'none',
width: isEnabled ? undefined : '0px!important', width: isEnabled ? undefined : '0px!important',
height: isEnabled ? undefined : '0px!important', height: isEnabled ? undefined : '0px !important',
border: '1px solid grey', border: '1px solid grey',
transform: local ? 'scaleX(-1)' : undefined, transform: local ? 'scaleX(-1)' : undefined,
display: isEnabled ? 'block' : 'none',
}} }}
> >
<video autoPlay ref={ref} muted={muted} style={{ height }} /> <video
{isAgent ? ( autoPlay
<div ref={ref}
style={{ muted={muted}
position: 'absolute', style={{ height }}
}} />
>
{t('Agent')}
</div>
) : null}
</div> </div>
); );
} }

46
frontend/app/player/web/assist/Call.ts
View file

@ -185,8 +185,7 @@ export default class Call {
pc.ontrack = (event) => { pc.ontrack = (event) => {
const stream = event.streams[0]; const stream = event.streams[0];
if (stream && !this.videoStreams[remotePeerId]) { if (stream && !this.videoStreams[remotePeerId]) {
const clonnedStream = stream.clone(); this.videoStreams[remotePeerId] = stream.getVideoTracks()[0];
this.videoStreams[remotePeerId] = clonnedStream.getVideoTracks()[0];
if (this.store.get().calling !== CallingState.OnCall) { if (this.store.get().calling !== CallingState.OnCall) {
this.store.update({ calling: CallingState.OnCall }); this.store.update({ calling: CallingState.OnCall });
} }
@ -305,22 +304,18 @@ export default class Call {
} }
try { try {
// if the connection is not established yet, then set remoteDescription to peer // if the connection is not established yet, then set remoteDescription to peer
if (!pc.localDescription) { await pc.setRemoteDescription(new RTCSessionDescription(data.offer));
await pc.setRemoteDescription(new RTCSessionDescription(data.offer)); const answer = await pc.createAnswer();
const answer = await pc.createAnswer(); await pc.setLocalDescription(answer);
await pc.setLocalDescription(answer); if (isAgent) {
if (isAgent) { this.socket.emit('WEBRTC_AGENT_CALL', {
this.socket.emit('WEBRTC_AGENT_CALL', { from: this.callID,
from: this.callID, answer,
answer, toAgentId: getSocketIdByCallId(fromCallId),
toAgentId: getSocketIdByCallId(fromCallId), type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER,
type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER, });
});
} else {
this.socket.emit('webrtc_call_answer', { from: fromCallId, answer });
}
} else { } else {
logger.warn('Skipping setRemoteDescription: Already in stable state'); this.socket.emit('webrtc_call_answer', { from: fromCallId, answer });
} }
} catch (e) { } catch (e) {
logger.error('Error setting remote description from answer', e); logger.error('Error setting remote description from answer', e);
@ -388,13 +383,13 @@ export default class Call {
private handleCallEnd() { private handleCallEnd() {
// If the call is not completed, then call onCallEnd // If the call is not completed, then call onCallEnd
if (this.store.get().calling !== CallingState.NoCall) { if (this.store.get().calling !== CallingState.NoCall) {
this.callArgs && this.callArgs.onCallEnd(); this.callArgs && this.callArgs.onRemoteCallEnd();
} }
// change state to NoCall // change state to NoCall
this.store.update({ calling: CallingState.NoCall }); this.store.update({ calling: CallingState.NoCall });
// Close all created RTCPeerConnection // Close all created RTCPeerConnection
Object.values(this.connections).forEach((pc) => pc.close()); Object.values(this.connections).forEach((pc) => pc.close());
this.callArgs?.onCallEnd(); this.callArgs?.onRemoteCallEnd();
// Clear connections // Clear connections
this.connections = {}; this.connections = {};
this.callArgs = null; this.callArgs = null;
@ -414,7 +409,7 @@ export default class Call {
// Close all connections and reset callArgs // Close all connections and reset callArgs
Object.values(this.connections).forEach((pc) => pc.close()); Object.values(this.connections).forEach((pc) => pc.close());
this.connections = {}; this.connections = {};
this.callArgs?.onCallEnd(); this.callArgs?.onRemoteCallEnd();
this.store.update({ calling: CallingState.NoCall }); this.store.update({ calling: CallingState.NoCall });
this.callArgs = null; this.callArgs = null;
} else { } else {
@ -443,7 +438,8 @@ export default class Call {
private callArgs: { private callArgs: {
localStream: LocalStream; localStream: LocalStream;
onStream: (s: MediaStream, isAgent: boolean) => void; onStream: (s: MediaStream, isAgent: boolean) => void;
onCallEnd: () => void; onRemoteCallEnd: () => void;
onLocalCallEnd: () => void;
onReject: () => void; onReject: () => void;
onError?: (arg?: any) => void; onError?: (arg?: any) => void;
} | null = null; } | null = null;
@ -451,14 +447,16 @@ export default class Call {
setCallArgs( setCallArgs(
localStream: LocalStream, localStream: LocalStream,
onStream: (s: MediaStream, isAgent: boolean) => void, onStream: (s: MediaStream, isAgent: boolean) => void,
onCallEnd: () => void, onRemoteCallEnd: () => void,
onLocalCallEnd: () => void,
onReject: () => void, onReject: () => void,
onError?: (e?: any) => void, onError?: (e?: any) => void,
) { ) {
this.callArgs = { this.callArgs = {
localStream, localStream,
onStream, onStream,
onCallEnd, onRemoteCallEnd,
onLocalCallEnd,
onReject, onReject,
onError, onError,
}; };
@ -549,7 +547,7 @@ export default class Call {
void this.initiateCallEnd(); void this.initiateCallEnd();
Object.values(this.connections).forEach((pc) => pc.close()); Object.values(this.connections).forEach((pc) => pc.close());
this.connections = {}; this.connections = {};
this.callArgs?.onCallEnd(); this.callArgs?.onLocalCallEnd();
} }
} }

129
tracker/tracker-assist/src/Assist.ts
View file

@ -548,6 +548,16 @@ export default class Assist {
} }
} }
const renegotiateConnection = async ({ pc, from }: { pc: RTCPeerConnection, from: string }) => {
try {
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
this.emit('webrtc_call_offer', { from, offer });
} catch (error) {
app.debug.error("Error with renegotiation:", error);
}
};
const handleIncomingCallOffer = async (from: string, offer: RTCSessionDescriptionInit) => { const handleIncomingCallOffer = async (from: string, offer: RTCSessionDescriptionInit) => {
app.debug.log('handleIncomingCallOffer', from) app.debug.log('handleIncomingCallOffer', from)
let confirmAnswer: Promise<boolean> let confirmAnswer: Promise<boolean>
@ -572,56 +582,59 @@ export default class Assist {
try { try {
// waiting for a decision on accepting the challenge // waiting for a decision on accepting the challenge
const agreed = await confirmAnswer const agreed = await confirmAnswer;
// if rejected, then terminate the call // if rejected, then terminate the call
if (!agreed) { if (!agreed) {
initiateCallEnd()
this.options.onCallDeny?.()
return
}
if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
callUI.setVideoToggleCallback((args: { enabled: boolean }) =>
this.emit('videofeed', { streamId: from, enabled: args.enabled })
);
}
// show buttons in the call window
callUI.showControls(initiateCallEnd)
if (!annot) {
annot = new AnnotationCanvas()
annot.mount()
}
// callUI.setLocalStreams(Object.values(lStreams))
try {
// if there are no local streams in lStrems then we set
if (!lStreams[from]) {
app.debug.log('starting new stream for', from)
// request a local stream, and set it to lStreams
lStreams[from] = await RequestLocalStream()
}
// we pass the received tracks to Call ui
callUI.setLocalStreams(Object.values(lStreams))
} catch (e) {
app.debug.error('Error requesting local stream', e);
// if something didn't work out, we terminate the call
initiateCallEnd(); initiateCallEnd();
this.options.onCallDeny?.();
return; return;
} }
// create a new RTCPeerConnection with ice server config
// create a new RTCPeerConnection with ice server config
const pc = new RTCPeerConnection({ const pc = new RTCPeerConnection({
iceServers: [{ urls: "stun:stun.l.google.com:19302" }], iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
}); });
// get all local tracks and add them to RTCPeerConnection if (!callUI) {
lStreams[from].stream.getTracks().forEach(track => { callUI = new CallWindow(app.debug.error, this.options.callUITemplate);
pc.addTrack(track, lStreams[from].stream); callUI.setVideoToggleCallback((args: { enabled: boolean }) => {
}); this.emit("videofeed", { streamId: from, enabled: args.enabled })
});
}
// show buttons in the call window
callUI.showControls(initiateCallEnd);
if (!annot) {
annot = new AnnotationCanvas();
annot.mount();
}
// callUI.setLocalStreams(Object.values(lStreams))
try {
// if there are no local streams in lStrems then we set
if (!lStreams[from]) {
app.debug.log("starting new stream for", from);
// request a local stream, and set it to lStreams
lStreams[from] = await RequestLocalStream(pc, renegotiateConnection.bind(null, { pc, from }));
}
// we pass the received tracks to Call ui
callUI.setLocalStreams(Object.values(lStreams));
} catch (e) {
app.debug.error("Error requesting local stream", e);
// if something didn't work out, we terminate the call
initiateCallEnd();
return;
}
// get all local tracks and add them to RTCPeerConnection
// When we receive local ice candidates, we emit them via socket // When we receive local ice candidates, we emit them via socket
pc.onicecandidate = (event) => { pc.onicecandidate = (event) => {
if (event.candidate) { if (event.candidate) {
socket.emit('webrtc_call_ice_candidate', { from, candidate: event.candidate }); socket.emit("webrtc_call_ice_candidate", {
from,
candidate: event.candidate,
});
} }
}; };
@ -632,9 +645,9 @@ export default class Assist {
callUI.addRemoteStream(rStream, from); callUI.addRemoteStream(rStream, from);
const onInteraction = () => { const onInteraction = () => {
callUI?.playRemote(); callUI?.playRemote();
document.removeEventListener('click', onInteraction); document.removeEventListener("click", onInteraction);
}; };
document.addEventListener('click', onInteraction); document.addEventListener("click", onInteraction);
} }
}; };
@ -648,7 +661,7 @@ export default class Assist {
// set answer as local description // set answer as local description
await pc.setLocalDescription(answer); await pc.setLocalDescription(answer);
// set the response as local // set the response as local
socket.emit('webrtc_call_answer', { from, answer }); socket.emit("webrtc_call_answer", { from, answer });
// If the state changes to an error, we terminate the call // If the state changes to an error, we terminate the call
// pc.onconnectionstatechange = () => { // pc.onconnectionstatechange = () => {
@ -658,27 +671,35 @@ export default class Assist {
// }; // };
// Update track when local video changes // Update track when local video changes
lStreams[from].onVideoTrack(vTrack => { lStreams[from].onVideoTrack((vTrack) => {
const sender = pc.getSenders().find(s => s.track?.kind === 'video'); const sender = pc.getSenders().find((s) => s.track?.kind === "video");
if (!sender) { if (!sender) {
app.debug.warn('No video sender found') app.debug.warn("No video sender found");
return return;
} }
sender.replaceTrack(vTrack) sender.replaceTrack(vTrack);
}) });
// if the user closed the tab or switched, then we end the call // if the user closed the tab or switched, then we end the call
document.addEventListener('visibilitychange', () => { document.addEventListener("visibilitychange", () => {
initiateCallEnd() initiateCallEnd();
}) });
// when everything is set, we change the state to true // when everything is set, we change the state to true
this.setCallingState(CallingState.True) this.setCallingState(CallingState.True);
if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() } if (!callEndCallback) {
const callingPeerIdsNow = Array.from(this.calls.keys()) callEndCallback = this.options.onCallStart?.();
}
const callingPeerIdsNow = Array.from(this.calls.keys());
// in session storage we write down everyone with whom the call is established // in session storage we write down everyone with whom the call is established
sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIdsNow)) sessionStorage.setItem(
this.emit('UPDATE_SESSION', { agentIds: callingPeerIdsNow, isCallActive: true }) this.options.session_calling_peer_key,
JSON.stringify(callingPeerIdsNow)
);
this.emit("UPDATE_SESSION", {
agentIds: callingPeerIdsNow,
isCallActive: true,
});
} catch (reason) { } catch (reason) {
app.debug.log(reason); app.debug.log(reason);
} }

13
tracker/tracker-assist/src/CallWindow.ts
View file

@ -48,7 +48,7 @@ export default class CallWindow {
} }
// const baseHref = "https://static.openreplay.com/tracker-assist/test" // const baseHref = "https://static.openreplay.com/tracker-assist/test"
const baseHref = 'https://static.openreplay.com/tracker-assist/4.0.0' const baseHref = 'https://static.openreplay.com/tracker-assist/widget'
// this.load = fetch(this.callUITemplate || baseHref + '/index2.html') // this.load = fetch(this.callUITemplate || baseHref + '/index2.html')
this.load = fetch(this.callUITemplate || baseHref + '/index.html') this.load = fetch(this.callUITemplate || baseHref + '/index.html')
.then((r) => r.text()) .then((r) => r.text())
@ -60,7 +60,7 @@ export default class CallWindow {
}, 0) }, 0)
//iframe.style.height = doc.body.scrollHeight + 'px'; //iframe.style.height = doc.body.scrollHeight + 'px';
//iframe.style.width = doc.body.scrollWidth + 'px'; //iframe.style.width = doc.body.scrollWidth + 'px';
this.adjustIframeSize() this.adjustIframeSize()
iframe.onload = null iframe.onload = null
} }
// ? // ?
@ -152,15 +152,6 @@ export default class CallWindow {
if (this.checkRemoteVideoInterval) { if (this.checkRemoteVideoInterval) {
clearInterval(this.checkRemoteVideoInterval) clearInterval(this.checkRemoteVideoInterval)
} // just in case } // just in case
let enabled = false
this.checkRemoteVideoInterval = setInterval(() => {
const settings = this.remoteVideo?.getSettings()
const isDummyVideoTrack = !this.remoteVideo.enabled || (!!settings && (settings.width === 2 || settings.frameRate === 0))
const shouldBeEnabled = !isDummyVideoTrack
if (enabled !== shouldBeEnabled) {
this.toggleRemoteVideoUI((enabled = shouldBeEnabled))
}
}, 1000)
} }
// Audio // Audio

134
tracker/tracker-assist/src/LocalStream.ts
View file

@ -1,88 +1,86 @@
declare global { export default function RequestLocalStream(
interface HTMLCanvasElement { pc: RTCPeerConnection,
captureStream(frameRate?: number): MediaStream; toggleVideoCb?: () => void
} ): Promise<LocalStream> {
} return navigator.mediaDevices
.getUserMedia({ audio: true, video: false })
function dummyTrack(): MediaStreamTrack { .then((stream) => {
const canvas = document.createElement('canvas')//, { width: 0, height: 0}) const aTrack = stream.getAudioTracks()[0];
canvas.setAttribute('data-openreplay-hidden', '1') if (!aTrack) {
canvas.width=canvas.height=2 // Doesn't work when 1 (?!) throw new Error("No audio tracks provided");
const ctx = canvas.getContext('2d') }
ctx?.fillRect(0, 0, canvas.width, canvas.height) stream.getTracks().forEach((track) => {
requestAnimationFrame(function draw(){ pc.addTrack(track, stream);
ctx?.fillRect(0,0, canvas.width, canvas.height) });
requestAnimationFrame(draw) return new _LocalStream(stream, pc, toggleVideoCb);
}) });
// Also works. Probably it should be done once connected.
//setTimeout(() => { ctx?.fillRect(0,0, canvas.width, canvas.height) }, 4000)
return canvas.captureStream(60).getTracks()[0]
}
export default function RequestLocalStream(): Promise<LocalStream> {
return navigator.mediaDevices.getUserMedia({ audio:true, })
.then(aStream => {
const aTrack = aStream.getAudioTracks()[0]
if (!aTrack) { throw new Error('No audio tracks provided') }
return new _LocalStream(aTrack)
})
} }
class _LocalStream { class _LocalStream {
private mediaRequested = false private mediaRequested = false;
readonly stream: MediaStream readonly stream: MediaStream;
private readonly vdTrack: MediaStreamTrack readonly vTrack: MediaStreamTrack;
constructor(aTrack: MediaStreamTrack) { readonly pc: RTCPeerConnection;
this.vdTrack = dummyTrack() readonly toggleVideoCb?: () => void;
this.stream = new MediaStream([ aTrack, this.vdTrack, ]) constructor(stream: MediaStream, pc: RTCPeerConnection, toggleVideoCb?: () => void) {
this.stream = stream;
this.pc = pc;
this.toggleVideoCb = toggleVideoCb;
} }
toggleVideo(): Promise<boolean> { toggleVideo(): Promise<boolean> {
const videoTracks = this.stream.getVideoTracks();
if (!this.mediaRequested) { if (!this.mediaRequested) {
return navigator.mediaDevices.getUserMedia({video:true,}) return navigator.mediaDevices
.then(vStream => { .getUserMedia({ video: true })
const vTrack = vStream.getVideoTracks()[0] .then((vStream) => {
if (!vTrack) { const vTrack = vStream.getVideoTracks()[0];
throw new Error('No video track provided') if (!vTrack) {
} throw new Error("No video track provided");
this.stream.addTrack(vTrack) }
this.stream.removeTrack(this.vdTrack)
this.mediaRequested = true this.pc.addTrack(vTrack, this.stream);
if (this.onVideoTrackCb) { this.stream.addTrack(vTrack);
this.onVideoTrackCb(vTrack)
} if (this.toggleVideoCb) {
return true this.toggleVideoCb();
}) }
.catch(e => {
// TODO: log this.mediaRequested = true;
console.error(e)
return false if (this.onVideoTrackCb) {
}) this.onVideoTrackCb(vTrack);
}
return true;
})
.catch((e) => {
// TODO: log
return false;
});
} else {
videoTracks.forEach((track) => {
track.enabled = !track.enabled;
});
} }
let enabled = true return Promise.resolve(videoTracks[0].enabled);
this.stream.getVideoTracks().forEach(track => {
track.enabled = enabled = enabled && !track.enabled
})
return Promise.resolve(enabled)
} }
toggleAudio(): boolean { toggleAudio(): boolean {
let enabled = true let enabled = true;
this.stream.getAudioTracks().forEach(track => { this.stream.getAudioTracks().forEach((track) => {
track.enabled = enabled = enabled && !track.enabled track.enabled = enabled = enabled && !track.enabled;
}) });
return enabled return enabled;
} }
private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null;
onVideoTrack(cb: (t: MediaStreamTrack) => void) { onVideoTrack(cb: (t: MediaStreamTrack) => void) {
this.onVideoTrackCb = cb this.onVideoTrackCb = cb;
} }
stop() { stop() {
this.stream.getTracks().forEach(t => t.stop()) this.stream.getTracks().forEach((t) => t.stop());
} }
} }
export type LocalStream = InstanceType<typeof _LocalStream> export type LocalStream = InstanceType<typeof _LocalStream>;