jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat tracker: open peer connection dynamically on call (#2197)

Browse source 
* feat tracker: open peer connection dynamically on call

* feat ui: move agent name trigger above peer init

* fix ui: adjust max amount of reconnections to wait for peer init
This commit is contained in:
Delirium 2024-05-24 17:43:22 +02:00 • committed by GitHub
parent 297be2bc9c
commit 5b48d391d5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 194 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
frontend/app/player/web/assist/Call.ts
View file

@ -38,7 +38,7 @@ export default class Call {
private peerID: string,
private getAssistVersion: () => number
) {
socket.on('call_end', (d) => {
socket.on('call_end', () => {
this.onRemoteCallEnd()
});
socket.on('videofeed', ({ streamId, enabled }) => {
@ -109,10 +109,10 @@ export default class Call {
const peer = (this._peer = new Peer(peerOpts));
peer.on('call', (call) => {
console.log('getting call from', call.peer);
call.answer(this.callArgs.localStream.stream);
call.answer(this.callArgs?.localStream.stream);
this.callConnection.push(call);
this.callArgs.localStream.onVideoTrack((vTrack) => {
this.callArgs?.localStream.onVideoTrack((vTrack) => {
const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video');
if (!sender) {
console.warn('No video sender found');
@ -125,7 +125,6 @@ export default class Call {
this.videoStreams[call.peer] = stream.getVideoTracks()[0];
this.callArgs && this.callArgs.onStream(stream);
});
// call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
call.on('close', this.onRemoteCallEnd);
call.on('error', (e) => {
@ -140,13 +139,6 @@ export default class Call {
} else if (e.type !== 'peer-unavailable') {
console.error(`PeerJS error (on peer). Type ${e.type}`, e);
}
// call-reconnection connected
// if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) {
// this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT
// ? ConnectionStatus.Connecting
// : ConnectionStatus.Disconnected);
// Reconnect...
});
return new Promise((resolve) => {
@ -199,7 +191,7 @@ export default class Call {
onStream: (s: MediaStream) => void;
onCallEnd: () => void;
onReject: () => void;
onError?: () => void;
onError?: (arg?: any) => void;
} | null = null;
setCallArgs(
@ -257,24 +249,45 @@ export default class Call {
? this.peerID
: `${this.peerID}-${tab || Object.keys(this.store.get().tabs)[0]}`;
void this._peerConnection(peerId);
this.emitData('_agent_name', appStore.getState().getIn(['user', 'account', 'name']));
void this._peerConnection(peerId);
}
connectAttempts = 0;
private async _peerConnection(remotePeerId: string) {
try {
const peer = await this.getPeer();
const call = peer.call(remotePeerId, this.callArgs.localStream.stream);
this.callConnection.push(call);
// let canCall = false
this.callArgs.localStream.onVideoTrack((vTrack) => {
const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video');
if (!sender) {
console.warn('No video sender found');
return;
const tryReconnect = async (e: any) => {
peer.off('error', tryReconnect)
console.log(e.type, this.connectAttempts);
if (e.type === 'peer-unavailable' && this.connectAttempts < 5) {
this.connectAttempts++;
console.log('reconnecting', this.connectAttempts);
await new Promise((resolve) => setTimeout(resolve, 250));
await this._peerConnection(remotePeerId);
} else {
console.log('error', this.connectAttempts);
this.callArgs?.onError?.('Could not establish a connection with the peer after 5 attempts');
}
sender.replaceTrack(vTrack);
});
}
const call = peer.call(remotePeerId, this.callArgs!.localStream.stream);
peer.on('error', tryReconnect);
peer.on('connection', () => {
this.callConnection.push(call);
this.connectAttempts = 0;
this.callArgs?.localStream.onVideoTrack((vTrack) => {
const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video');
if (!sender) {
console.warn('No video sender found');
return;
}
sender.replaceTrack(vTrack);
});
})
call.on('stream', (stream) => {
this.store.get().calling !== CallingState.OnCall &&
@ -284,7 +297,6 @@ export default class Call {
this.callArgs && this.callArgs.onStream(stream);
});
// call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
call.on('close', this.onRemoteCallEnd);
call.on('error', (e) => {
@ -301,7 +313,7 @@ export default class Call {
clean() {
this.cleaned = true; // sometimes cleaned before modules loaded
this.initiateCallEnd();
void this.initiateCallEnd();
if (this._peer) {
console.log('destroying peer...');
const peer = this._peer; // otherwise it calls reconnection on data chan close

2
tracker/tracker-assist/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@openreplay/tracker-assist",
"description": "Tracker plugin for screen assistance through the WebRTC",
"version": "9.0.0",
"version": "9.0.1-3",
"keywords": [
"WebRTC",
"assistance",

296
tracker/tracker-assist/src/Assist.ts
View file

@ -156,8 +156,18 @@ export default class Assist {
// @ts-ignore No need in statistics messages. TODO proper filter
if (batchSize === 2 && messages[0]._id === 0 && messages[1]._id === 49) { return }
if (batchSize > this.options.compressionMinBatchSize && this.options.compressionEnabled) {
while (messages.length > 0) {
const batch = messages.splice(0, this.options.compressionMinBatchSize)
const toSend: any[] = []
if (batchSize > 10000) {
const middle = Math.floor(batchSize / 2)
const firstHalf = messages.slice(0, middle)
const secondHalf = messages.slice(middle)
toSend.push(firstHalf)
toSend.push(secondHalf)
} else {
toSend.push(messages)
}
toSend.forEach(batch => {
const str = JSON.stringify(batch)
const byteArr = new TextEncoder().encode(str)
gzip(byteArr, { mtime: 0, }, (err, result) => {
@ -167,7 +177,7 @@ export default class Assist {
this.emit('messages_gz', result)
}
})
}
})
} else {
this.emit('messages', messages)
}
@ -401,6 +411,10 @@ export default class Assist {
if (app.getTabId() !== info.meta.tabId) return
const name = info.data
callingAgents.set(id, name)
if (!this.peer) {
setupPeer()
}
updateCallerNames()
})
socket.on('videofeed', (_, info) => {
@ -429,7 +443,10 @@ export default class Assist {
// TODO: merge peerId & socket.io id (simplest way - send peerId with the name)
const calls: Record<string, MediaConnection> = {} // !! uses peerJS ID
const lStreams: Record<string, LocalStream> = {}
// const callingPeers: Map<string, { call: MediaConnection, lStream: LocalStream }> = new Map() // Maybe
function updateCallerNames() {
callUI?.setAssistentName(callingAgents)
}
function endAgentCall(id: string) {
callingAgents.delete(id)
if (callingAgents.size === 0) {
@ -439,59 +456,6 @@ export default class Assist {
//TODO: close() specific call and corresponding lStreams (after connecting peerId & socket.io id)
}
}
// PeerJS call (todo: use native WebRTC)
const peerOptions = {
host: this.getHost(),
path: this.getBasePrefixUrl()+'/assist',
port: location.protocol === 'http:' && this.noSecureMode ? 80 : 443,
debug: 2, //appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs.
}
if (this.options.config) {
peerOptions['config'] = this.options.config
}
const peer = new safeCastedPeer(peerID, peerOptions) as Peer
this.peer = peer
let peerReconnectAttempts = 0
// @ts-ignore (peerjs typing)
peer.on('error', e => app.debug.warn('Peer error: ', e.type, e))
peer.on('disconnected', () => {
if (peerReconnectAttempts < 30) {
this.peerReconnectTimeout = setTimeout(() => {
if (this.app.active() && !peer.destroyed) {
peer.reconnect()
}
}, Math.min(peerReconnectAttempts, 8) * 2 * 1000)
peerReconnectAttempts += 1
}
})
function updateCallerNames() {
callUI?.setAssistentName(callingAgents)
}
const closeCallConfirmWindow = () => {
if (callConfirmWindow) {
callConfirmWindow.remove()
callConfirmWindow = null
callConfirmAnswer = null
}
}
const requestCallConfirm = () => {
if (callConfirmAnswer) { // Already asking
return callConfirmAnswer
}
callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || {
text: this.options.confirmText,
style: this.options.confirmStyle,
})) // TODO: reuse ?
return callConfirmAnswer = callConfirmWindow.mount().then(answer => {
closeCallConfirmWindow()
return answer
})
}
const handleCallEnd = () => { // Complete stop and clear all calls
// Streams
Object.values(calls).forEach(call => call.close())
@ -517,102 +481,154 @@ export default class Assist {
callEndCallback?.()
}
const initiateCallEnd = () => {
this.emit('call_end')
handleCallEnd()
const closeCallConfirmWindow = () => {
if (callConfirmWindow) {
callConfirmWindow.remove()
callConfirmWindow = null
callConfirmAnswer = null
}
}
const updateVideoFeed = ({ enabled, }) => this.emit('videofeed', { streamId: this.peer?.id, enabled, })
peer.on('call', (call) => {
app.debug.log('Incoming call from', call.peer)
let confirmAnswer: Promise<boolean>
const callingPeerIds = JSON.parse(sessionStorage.getItem(this.options.session_calling_peer_key) || '[]')
if (callingPeerIds.includes(call.peer) || this.callingState === CallingState.True) {
confirmAnswer = Promise.resolve(true)
} else {
this.setCallingState(CallingState.Requesting)
confirmAnswer = requestCallConfirm()
this.playNotificationSound() // For every new agent during confirmation here
// TODO: only one (latest) timeout
setTimeout(() => {
if (this.callingState !== CallingState.Requesting) { return }
initiateCallEnd()
}, 30000)
// PeerJS call (todo: use native WebRTC)
const peerOptions = {
host: this.getHost(),
path: this.getBasePrefixUrl()+'/assist',
port: location.protocol === 'http:' && this.noSecureMode ? 80 : 443,
debug: 2, //appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs.
}
const setupPeer = () => {
if (this.options.config) {
peerOptions['config'] = this.options.config
}
confirmAnswer.then(async agreed => {
if (!agreed) {
initiateCallEnd()
this.options.onCallDeny?.()
return
}
// Request local stream for the new connection
try {
// lStreams are reusable so fare we don't delete them in the `endAgentCall`
if (!lStreams[call.peer]) {
app.debug.log('starting new stream for', call.peer)
lStreams[call.peer] = await RequestLocalStream()
}
calls[call.peer] = call
} catch (e) {
app.debug.error('Audio media device request error:', e)
initiateCallEnd()
return
const peer = new safeCastedPeer(peerID, peerOptions) as Peer
this.peer = peer
let peerReconnectAttempts = 0
// @ts-ignore (peerjs typing)
peer.on('error', e => app.debug.warn('Peer error: ', e.type, e))
peer.on('disconnected', () => {
if (peerReconnectAttempts < 30) {
this.peerReconnectTimeout = setTimeout(() => {
if (this.app.active() && !peer.destroyed) {
peer.reconnect()
}
}, Math.min(peerReconnectAttempts, 8) * 2 * 1000)
peerReconnectAttempts += 1
}
})
if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
callUI.setVideoToggleCallback(updateVideoFeed)
}
callUI.showControls(initiateCallEnd)
if (!annot) {
annot = new AnnotationCanvas()
annot.mount()
const requestCallConfirm = () => {
if (callConfirmAnswer) { // Already asking
return callConfirmAnswer
}
// have to be updated
callUI.setLocalStreams(Object.values(lStreams))
call.on('error', e => {
app.debug.warn('Call error:', e)
initiateCallEnd()
})
call.on('stream', (rStream) => {
callUI?.addRemoteStream(rStream, call.peer)
const onInteraction = () => { // do only if document.hidden ?
callUI?.playRemote()
document.removeEventListener('click', onInteraction)
}
document.addEventListener('click', onInteraction)
callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || {
text: this.options.confirmText,
style: this.options.confirmStyle,
})) // TODO: reuse ?
return callConfirmAnswer = callConfirmWindow.mount().then(answer => {
closeCallConfirmWindow()
return answer
})
}
// remote video on/off/camera change
lStreams[call.peer].onVideoTrack(vTrack => {
const sender = call.peerConnection.getSenders().find(s => s.track?.kind === 'video')
if (!sender) {
app.debug.warn('No video sender found')
const initiateCallEnd = () => {
this.emit('call_end')
handleCallEnd()
}
const updateVideoFeed = ({ enabled, }) => this.emit('videofeed', { streamId: this.peer?.id, enabled, })
peer.on('call', (call) => {
app.debug.log('Incoming call from', call.peer)
let confirmAnswer: Promise<boolean>
const callingPeerIds = JSON.parse(sessionStorage.getItem(this.options.session_calling_peer_key) || '[]')
if (callingPeerIds.includes(call.peer) || this.callingState === CallingState.True) {
confirmAnswer = Promise.resolve(true)
} else {
this.setCallingState(CallingState.Requesting)
confirmAnswer = requestCallConfirm()
this.playNotificationSound() // For every new agent during confirmation here
// TODO: only one (latest) timeout
setTimeout(() => {
if (this.callingState !== CallingState.Requesting) { return }
initiateCallEnd()
}, 30000)
}
confirmAnswer.then(async agreed => {
if (!agreed) {
initiateCallEnd()
this.options.onCallDeny?.()
return
}
app.debug.log('sender found:', sender)
void sender.replaceTrack(vTrack)
// Request local stream for the new connection
try {
// lStreams are reusable so fare we don't delete them in the `endAgentCall`
if (!lStreams[call.peer]) {
app.debug.log('starting new stream for', call.peer)
lStreams[call.peer] = await RequestLocalStream()
}
calls[call.peer] = call
} catch (e) {
app.debug.error('Audio media device request error:', e)
initiateCallEnd()
return
}
if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
callUI.setVideoToggleCallback(updateVideoFeed)
}
callUI.showControls(initiateCallEnd)
if (!annot) {
annot = new AnnotationCanvas()
annot.mount()
}
// have to be updated
callUI.setLocalStreams(Object.values(lStreams))
call.on('error', e => {
app.debug.warn('Call error:', e)
initiateCallEnd()
})
call.on('stream', (rStream) => {
callUI?.addRemoteStream(rStream, call.peer)
const onInteraction = () => { // do only if document.hidden ?
callUI?.playRemote()
document.removeEventListener('click', onInteraction)
}
document.addEventListener('click', onInteraction)
})
// remote video on/off/camera change
lStreams[call.peer].onVideoTrack(vTrack => {
const sender = call.peerConnection.getSenders().find(s => s.track?.kind === 'video')
if (!sender) {
app.debug.warn('No video sender found')
return
}
app.debug.log('sender found:', sender)
void sender.replaceTrack(vTrack)
})
call.answer(lStreams[call.peer].stream)
document.addEventListener('visibilitychange', () => {
initiateCallEnd()
})
this.setCallingState(CallingState.True)
if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() }
const callingPeerIds = Object.keys(calls)
sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIds))
this.emit('UPDATE_SESSION', { agentIds: callingPeerIds, isCallActive: true, })
}).catch(reason => { // in case of Confirm.remove() without user answer (not an error)
app.debug.log(reason)
})
call.answer(lStreams[call.peer].stream)
document.addEventListener('visibilitychange', () => {
initiateCallEnd()
})
this.setCallingState(CallingState.True)
if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() }
const callingPeerIds = Object.keys(calls)
sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIds))
this.emit('UPDATE_SESSION', { agentIds: callingPeerIds, isCallActive: true, })
}).catch(reason => { // in case of Confirm.remove() without user answer (not a error)
app.debug.log(reason)
})
})
}
const startCanvasStream = (stream: MediaStream, id: number) => {
@ -668,7 +684,7 @@ export default class Assist {
}
private clean() {
// sometimes means new agent connected so we keep id for control
// sometimes means new agent connected, so we keep id for control
this.remoteControl?.releaseControl(false, true)
if (this.peerReconnectTimeout) {
clearTimeout(this.peerReconnectTimeout)

2
tracker/tracker-assist/src/version.ts
View file

@ -1 +1 @@
export const pkgVersion = '8.0.5-4'
export const pkgVersion = '9.0.1-3'