jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

updated widget link (#3158)

Browse source 
* updated widget link

* fix calls

* updated widget url
This commit is contained in:
Andrey Babushkin 2025-03-18 13:07:09 +03:00 • committed by GitHub
parent 4c967d4bc1
commit 7365d8639c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 195 additions and 189 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
frontend/app/components/Assist/components/AssistActions/AssistActions.tsx
View file

@ -82,7 +82,7 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
{ stream: MediaStream; isAgent: boolean }[] | null
>([]);
const [localStream, setLocalStream] = useState<LocalStream | null>(null);
const [callObject, setCallObject] = useState<{ end: () => void } | null>(
const [callObject, setCallObject] = useState<{ end: () => void } | null | undefined>(
null,
);
@ -135,6 +135,7 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
}, [peerConnectionStatus]);
const addIncomeStream = (stream: MediaStream, isAgent: boolean) => {
if (!stream.active) return;
setIncomeStream((oldState) => {
if (oldState === null) return [{ stream, isAgent }];
if (
@ -149,13 +150,8 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
});
};
const removeIncomeStream = (stream: MediaStream) => {
setIncomeStream((prevState) => {
if (!prevState) return [];
return prevState.filter(
(existingStream) => existingStream.stream.id !== stream.id,
);
});
const removeIncomeStream = () => {
setIncomeStream([]);
};
function onReject() {
@ -181,7 +177,12 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
() => {
player.assistManager.ping(AssistActionsPing.call.end, agentId);
lStream.stop.apply(lStream);
removeIncomeStream(lStream.stream);
removeIncomeStream();
},
() => {
player.assistManager.ping(AssistActionsPing.call.end, agentId);
lStream.stop.apply(lStream);
removeIncomeStream();
},
onReject,
onError,

43
frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx
View file

@ -34,43 +34,40 @@ function VideoContainer({
}
const iid = setInterval(() => {
const track = stream.getVideoTracks()[0];
const settings = track?.getSettings();
const isDummyVideoTrack = settings
? settings.width === 2 ||
settings.frameRate === 0 ||
(!settings.frameRate && !settings.width)
: true;
const shouldBeEnabled = track.enabled && !isDummyVideoTrack;
if (isEnabled !== shouldBeEnabled) {
setEnabled(shouldBeEnabled);
setRemoteEnabled?.(shouldBeEnabled);
if (track) {
if (!track.enabled) {
setEnabled(false);
setRemoteEnabled?.(false);
} else {
setEnabled(true);
setRemoteEnabled?.(true);
}
} else {
setEnabled(false);
setRemoteEnabled?.(false);
}
}, 500);
return () => clearInterval(iid);
}, [stream, isEnabled]);
}, [stream]);
return (
<div
className="flex-1"
style={{
display: isEnabled ? undefined : 'none',
width: isEnabled ? undefined : '0px!important',
height: isEnabled ? undefined : '0px!important',
height: isEnabled ? undefined : '0px !important',
border: '1px solid grey',
transform: local ? 'scaleX(-1)' : undefined,
display: isEnabled ? 'block' : 'none',
}}
>
<video autoPlay ref={ref} muted={muted} style={{ height }} />
{isAgent ? (
<div
style={{
position: 'absolute',
}}
>
{t('Agent')}
</div>
) : null}
<video
autoPlay
ref={ref}
muted={muted}
style={{ height }}
/>
</div>
);
}

46
frontend/app/player/web/assist/Call.ts
View file

@ -185,8 +185,7 @@ export default class Call {
pc.ontrack = (event) => {
const stream = event.streams[0];
if (stream && !this.videoStreams[remotePeerId]) {
const clonnedStream = stream.clone();
this.videoStreams[remotePeerId] = clonnedStream.getVideoTracks()[0];
this.videoStreams[remotePeerId] = stream.getVideoTracks()[0];
if (this.store.get().calling !== CallingState.OnCall) {
this.store.update({ calling: CallingState.OnCall });
}
@ -305,22 +304,18 @@ export default class Call {
}
try {
// if the connection is not established yet, then set remoteDescription to peer
if (!pc.localDescription) {
await pc.setRemoteDescription(new RTCSessionDescription(data.offer));
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
if (isAgent) {
this.socket.emit('WEBRTC_AGENT_CALL', {
from: this.callID,
answer,
toAgentId: getSocketIdByCallId(fromCallId),
type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER,
});
} else {
this.socket.emit('webrtc_call_answer', { from: fromCallId, answer });
}
await pc.setRemoteDescription(new RTCSessionDescription(data.offer));
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
if (isAgent) {
this.socket.emit('WEBRTC_AGENT_CALL', {
from: this.callID,
answer,
toAgentId: getSocketIdByCallId(fromCallId),
type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER,
});
} else {
logger.warn('Skipping setRemoteDescription: Already in stable state');
this.socket.emit('webrtc_call_answer', { from: fromCallId, answer });
}
} catch (e) {
logger.error('Error setting remote description from answer', e);
@ -388,13 +383,13 @@ export default class Call {
private handleCallEnd() {
// If the call is not completed, then call onCallEnd
if (this.store.get().calling !== CallingState.NoCall) {
this.callArgs && this.callArgs.onCallEnd();
this.callArgs && this.callArgs.onRemoteCallEnd();
}
// change state to NoCall
this.store.update({ calling: CallingState.NoCall });
// Close all created RTCPeerConnection
Object.values(this.connections).forEach((pc) => pc.close());
this.callArgs?.onCallEnd();
this.callArgs?.onRemoteCallEnd();
// Clear connections
this.connections = {};
this.callArgs = null;
@ -414,7 +409,7 @@ export default class Call {
// Close all connections and reset callArgs
Object.values(this.connections).forEach((pc) => pc.close());
this.connections = {};
this.callArgs?.onCallEnd();
this.callArgs?.onRemoteCallEnd();
this.store.update({ calling: CallingState.NoCall });
this.callArgs = null;
} else {
@ -443,7 +438,8 @@ export default class Call {
private callArgs: {
localStream: LocalStream;
onStream: (s: MediaStream, isAgent: boolean) => void;
onCallEnd: () => void;
onRemoteCallEnd: () => void;
onLocalCallEnd: () => void;
onReject: () => void;
onError?: (arg?: any) => void;
} | null = null;
@ -451,14 +447,16 @@ export default class Call {
setCallArgs(
localStream: LocalStream,
onStream: (s: MediaStream, isAgent: boolean) => void,
onCallEnd: () => void,
onRemoteCallEnd: () => void,
onLocalCallEnd: () => void,
onReject: () => void,
onError?: (e?: any) => void,
) {
this.callArgs = {
localStream,
onStream,
onCallEnd,
onRemoteCallEnd,
onLocalCallEnd,
onReject,
onError,
};
@ -549,7 +547,7 @@ export default class Call {
void this.initiateCallEnd();
Object.values(this.connections).forEach((pc) => pc.close());
this.connections = {};
this.callArgs?.onCallEnd();
this.callArgs?.onLocalCallEnd();
}
}

129
tracker/tracker-assist/src/Assist.ts
View file

@ -548,6 +548,16 @@ export default class Assist {
}
}
const renegotiateConnection = async ({ pc, from }: { pc: RTCPeerConnection, from: string }) => {
try {
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
this.emit('webrtc_call_offer', { from, offer });
} catch (error) {
app.debug.error("Error with renegotiation:", error);
}
};
const handleIncomingCallOffer = async (from: string, offer: RTCSessionDescriptionInit) => {
app.debug.log('handleIncomingCallOffer', from)
let confirmAnswer: Promise<boolean>
@ -572,56 +582,59 @@ export default class Assist {
try {
// waiting for a decision on accepting the challenge
const agreed = await confirmAnswer
const agreed = await confirmAnswer;
// if rejected, then terminate the call
if (!agreed) {
initiateCallEnd()
this.options.onCallDeny?.()
return
}
if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
callUI.setVideoToggleCallback((args: { enabled: boolean }) =>
this.emit('videofeed', { streamId: from, enabled: args.enabled })
);
}
// show buttons in the call window
callUI.showControls(initiateCallEnd)
if (!annot) {
annot = new AnnotationCanvas()
annot.mount()
}
// callUI.setLocalStreams(Object.values(lStreams))
try {
// if there are no local streams in lStrems then we set
if (!lStreams[from]) {
app.debug.log('starting new stream for', from)
// request a local stream, and set it to lStreams
lStreams[from] = await RequestLocalStream()
}
// we pass the received tracks to Call ui
callUI.setLocalStreams(Object.values(lStreams))
} catch (e) {
app.debug.error('Error requesting local stream', e);
// if something didn't work out, we terminate the call
initiateCallEnd();
this.options.onCallDeny?.();
return;
}
// create a new RTCPeerConnection with ice server config
// create a new RTCPeerConnection with ice server config
const pc = new RTCPeerConnection({
iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
});
// get all local tracks and add them to RTCPeerConnection
lStreams[from].stream.getTracks().forEach(track => {
pc.addTrack(track, lStreams[from].stream);
});
if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate);
callUI.setVideoToggleCallback((args: { enabled: boolean }) => {
this.emit("videofeed", { streamId: from, enabled: args.enabled })
});
}
// show buttons in the call window
callUI.showControls(initiateCallEnd);
if (!annot) {
annot = new AnnotationCanvas();
annot.mount();
}
// callUI.setLocalStreams(Object.values(lStreams))
try {
// if there are no local streams in lStrems then we set
if (!lStreams[from]) {
app.debug.log("starting new stream for", from);
// request a local stream, and set it to lStreams
lStreams[from] = await RequestLocalStream(pc, renegotiateConnection.bind(null, { pc, from }));
}
// we pass the received tracks to Call ui
callUI.setLocalStreams(Object.values(lStreams));
} catch (e) {
app.debug.error("Error requesting local stream", e);
// if something didn't work out, we terminate the call
initiateCallEnd();
return;
}
// get all local tracks and add them to RTCPeerConnection
// When we receive local ice candidates, we emit them via socket
pc.onicecandidate = (event) => {
if (event.candidate) {
socket.emit('webrtc_call_ice_candidate', { from, candidate: event.candidate });
socket.emit("webrtc_call_ice_candidate", {
from,
candidate: event.candidate,
});
}
};
@ -632,9 +645,9 @@ export default class Assist {
callUI.addRemoteStream(rStream, from);
const onInteraction = () => {
callUI?.playRemote();
document.removeEventListener('click', onInteraction);
document.removeEventListener("click", onInteraction);
};
document.addEventListener('click', onInteraction);
document.addEventListener("click", onInteraction);
}
};
@ -648,7 +661,7 @@ export default class Assist {
// set answer as local description
await pc.setLocalDescription(answer);
// set the response as local
socket.emit('webrtc_call_answer', { from, answer });
socket.emit("webrtc_call_answer", { from, answer });
// If the state changes to an error, we terminate the call
// pc.onconnectionstatechange = () => {
@ -658,27 +671,35 @@ export default class Assist {
// };
// Update track when local video changes
lStreams[from].onVideoTrack(vTrack => {
const sender = pc.getSenders().find(s => s.track?.kind === 'video');
lStreams[from].onVideoTrack((vTrack) => {
const sender = pc.getSenders().find((s) => s.track?.kind === "video");
if (!sender) {
app.debug.warn('No video sender found')
return
app.debug.warn("No video sender found");
return;
}
sender.replaceTrack(vTrack)
})
sender.replaceTrack(vTrack);
});
// if the user closed the tab or switched, then we end the call
document.addEventListener('visibilitychange', () => {
initiateCallEnd()
})
document.addEventListener("visibilitychange", () => {
initiateCallEnd();
});
// when everything is set, we change the state to true
this.setCallingState(CallingState.True)
if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() }
const callingPeerIdsNow = Array.from(this.calls.keys())
this.setCallingState(CallingState.True);
if (!callEndCallback) {
callEndCallback = this.options.onCallStart?.();
}
const callingPeerIdsNow = Array.from(this.calls.keys());
// in session storage we write down everyone with whom the call is established
sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIdsNow))
this.emit('UPDATE_SESSION', { agentIds: callingPeerIdsNow, isCallActive: true })
sessionStorage.setItem(
this.options.session_calling_peer_key,
JSON.stringify(callingPeerIdsNow)
);
this.emit("UPDATE_SESSION", {
agentIds: callingPeerIdsNow,
isCallActive: true,
});
} catch (reason) {
app.debug.log(reason);
}

13
tracker/tracker-assist/src/CallWindow.ts
View file

@ -48,7 +48,7 @@ export default class CallWindow {
}
// const baseHref = "https://static.openreplay.com/tracker-assist/test"
const baseHref = 'https://static.openreplay.com/tracker-assist/4.0.0'
const baseHref = 'https://static.openreplay.com/tracker-assist/widget'
// this.load = fetch(this.callUITemplate || baseHref + '/index2.html')
this.load = fetch(this.callUITemplate || baseHref + '/index.html')
.then((r) => r.text())
@ -60,7 +60,7 @@ export default class CallWindow {
}, 0)
//iframe.style.height = doc.body.scrollHeight + 'px';
//iframe.style.width = doc.body.scrollWidth + 'px';
this.adjustIframeSize()
this.adjustIframeSize()
iframe.onload = null
}
// ?
@ -152,15 +152,6 @@ export default class CallWindow {
if (this.checkRemoteVideoInterval) {
clearInterval(this.checkRemoteVideoInterval)
} // just in case
let enabled = false
this.checkRemoteVideoInterval = setInterval(() => {
const settings = this.remoteVideo?.getSettings()
const isDummyVideoTrack = !this.remoteVideo.enabled || (!!settings && (settings.width === 2 || settings.frameRate === 0))
const shouldBeEnabled = !isDummyVideoTrack
if (enabled !== shouldBeEnabled) {
this.toggleRemoteVideoUI((enabled = shouldBeEnabled))
}
}, 1000)
}
// Audio

134
tracker/tracker-assist/src/LocalStream.ts
View file

@ -1,88 +1,86 @@
declare global {
interface HTMLCanvasElement {
captureStream(frameRate?: number): MediaStream;
}
}
function dummyTrack(): MediaStreamTrack {
const canvas = document.createElement('canvas')//, { width: 0, height: 0})
canvas.setAttribute('data-openreplay-hidden', '1')
canvas.width=canvas.height=2 // Doesn't work when 1 (?!)
const ctx = canvas.getContext('2d')
ctx?.fillRect(0, 0, canvas.width, canvas.height)
requestAnimationFrame(function draw(){
ctx?.fillRect(0,0, canvas.width, canvas.height)
requestAnimationFrame(draw)
})
// Also works. Probably it should be done once connected.
//setTimeout(() => { ctx?.fillRect(0,0, canvas.width, canvas.height) }, 4000)
return canvas.captureStream(60).getTracks()[0]
}
export default function RequestLocalStream(): Promise<LocalStream> {
return navigator.mediaDevices.getUserMedia({ audio:true, })
.then(aStream => {
const aTrack = aStream.getAudioTracks()[0]
if (!aTrack) { throw new Error('No audio tracks provided') }
return new _LocalStream(aTrack)
})
export default function RequestLocalStream(
pc: RTCPeerConnection,
toggleVideoCb?: () => void
): Promise<LocalStream> {
return navigator.mediaDevices
.getUserMedia({ audio: true, video: false })
.then((stream) => {
const aTrack = stream.getAudioTracks()[0];
if (!aTrack) {
throw new Error("No audio tracks provided");
}
stream.getTracks().forEach((track) => {
pc.addTrack(track, stream);
});
return new _LocalStream(stream, pc, toggleVideoCb);
});
}
class _LocalStream {
private mediaRequested = false
readonly stream: MediaStream
private readonly vdTrack: MediaStreamTrack
constructor(aTrack: MediaStreamTrack) {
this.vdTrack = dummyTrack()
this.stream = new MediaStream([ aTrack, this.vdTrack, ])
private mediaRequested = false;
readonly stream: MediaStream;
readonly vTrack: MediaStreamTrack;
readonly pc: RTCPeerConnection;
readonly toggleVideoCb?: () => void;
constructor(stream: MediaStream, pc: RTCPeerConnection, toggleVideoCb?: () => void) {
this.stream = stream;
this.pc = pc;
this.toggleVideoCb = toggleVideoCb;
}
toggleVideo(): Promise<boolean> {
const videoTracks = this.stream.getVideoTracks();
if (!this.mediaRequested) {
return navigator.mediaDevices.getUserMedia({video:true,})
.then(vStream => {
const vTrack = vStream.getVideoTracks()[0]
if (!vTrack) {
throw new Error('No video track provided')
}
this.stream.addTrack(vTrack)
this.stream.removeTrack(this.vdTrack)
this.mediaRequested = true
if (this.onVideoTrackCb) {
this.onVideoTrackCb(vTrack)
}
return true
})
.catch(e => {
// TODO: log
console.error(e)
return false
})
return navigator.mediaDevices
.getUserMedia({ video: true })
.then((vStream) => {
const vTrack = vStream.getVideoTracks()[0];
if (!vTrack) {
throw new Error("No video track provided");
}
this.pc.addTrack(vTrack, this.stream);
this.stream.addTrack(vTrack);
if (this.toggleVideoCb) {
this.toggleVideoCb();
}
this.mediaRequested = true;
if (this.onVideoTrackCb) {
this.onVideoTrackCb(vTrack);
}
return true;
})
.catch((e) => {
// TODO: log
return false;
});
} else {
videoTracks.forEach((track) => {
track.enabled = !track.enabled;
});
}
let enabled = true
this.stream.getVideoTracks().forEach(track => {
track.enabled = enabled = enabled && !track.enabled
})
return Promise.resolve(enabled)
return Promise.resolve(videoTracks[0].enabled);
}
toggleAudio(): boolean {
let enabled = true
this.stream.getAudioTracks().forEach(track => {
track.enabled = enabled = enabled && !track.enabled
})
return enabled
let enabled = true;
this.stream.getAudioTracks().forEach((track) => {
track.enabled = enabled = enabled && !track.enabled;
});
return enabled;
}
private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null
private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null;
onVideoTrack(cb: (t: MediaStreamTrack) => void) {
this.onVideoTrackCb = cb
this.onVideoTrackCb = cb;
}
stop() {
this.stream.getTracks().forEach(t => t.stop())
this.stream.getTracks().forEach((t) => t.stop());
}
}
export type LocalStream = InstanceType<typeof _LocalStream>
export type LocalStream = InstanceType<typeof _LocalStream>;