diff --git a/frontend/app/player/web/assist/Call.ts b/frontend/app/player/web/assist/Call.ts index 6c71c3ed8..7dd24e668 100644 --- a/frontend/app/player/web/assist/Call.ts +++ b/frontend/app/player/web/assist/Call.ts @@ -38,7 +38,7 @@ export default class Call { private peerID: string, private getAssistVersion: () => number ) { - socket.on('call_end', (d) => { + socket.on('call_end', () => { this.onRemoteCallEnd() }); socket.on('videofeed', ({ streamId, enabled }) => { @@ -109,10 +109,10 @@ export default class Call { const peer = (this._peer = new Peer(peerOpts)); peer.on('call', (call) => { console.log('getting call from', call.peer); - call.answer(this.callArgs.localStream.stream); + call.answer(this.callArgs?.localStream.stream); this.callConnection.push(call); - this.callArgs.localStream.onVideoTrack((vTrack) => { + this.callArgs?.localStream.onVideoTrack((vTrack) => { const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video'); if (!sender) { console.warn('No video sender found'); @@ -125,7 +125,6 @@ export default class Call { this.videoStreams[call.peer] = stream.getVideoTracks()[0]; this.callArgs && this.callArgs.onStream(stream); }); - // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) call.on('close', this.onRemoteCallEnd); call.on('error', (e) => { @@ -140,13 +139,6 @@ export default class Call { } else if (e.type !== 'peer-unavailable') { console.error(`PeerJS error (on peer). Type ${e.type}`, e); } - - // call-reconnection connected - // if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) { - // this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT - // ? ConnectionStatus.Connecting - // : ConnectionStatus.Disconnected); - // Reconnect... }); return new Promise((resolve) => { @@ -199,7 +191,7 @@ export default class Call { onStream: (s: MediaStream) => void; onCallEnd: () => void; onReject: () => void; - onError?: () => void; + onError?: (arg?: any) => void; } | null = null; setCallArgs( @@ -257,24 +249,45 @@ export default class Call { ? this.peerID : `${this.peerID}-${tab || Object.keys(this.store.get().tabs)[0]}`; - void this._peerConnection(peerId); this.emitData('_agent_name', appStore.getState().getIn(['user', 'account', 'name'])); + void this._peerConnection(peerId); } + connectAttempts = 0; private async _peerConnection(remotePeerId: string) { try { const peer = await this.getPeer(); - const call = peer.call(remotePeerId, this.callArgs.localStream.stream); - this.callConnection.push(call); + // let canCall = false - this.callArgs.localStream.onVideoTrack((vTrack) => { - const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video'); - if (!sender) { - console.warn('No video sender found'); - return; + const tryReconnect = async (e: any) => { + peer.off('error', tryReconnect) + console.log(e.type, this.connectAttempts); + if (e.type === 'peer-unavailable' && this.connectAttempts < 5) { + this.connectAttempts++; + console.log('reconnecting', this.connectAttempts); + await new Promise((resolve) => setTimeout(resolve, 250)); + await this._peerConnection(remotePeerId); + } else { + console.log('error', this.connectAttempts); + this.callArgs?.onError?.('Could not establish a connection with the peer after 5 attempts'); } - sender.replaceTrack(vTrack); - }); + } + const call = peer.call(remotePeerId, this.callArgs!.localStream.stream); + peer.on('error', tryReconnect); + + peer.on('connection', () => { + this.callConnection.push(call); + this.connectAttempts = 0; + + this.callArgs?.localStream.onVideoTrack((vTrack) => { + const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video'); + if (!sender) { + console.warn('No video sender found'); + return; + } + sender.replaceTrack(vTrack); + }); + }) call.on('stream', (stream) => { this.store.get().calling !== CallingState.OnCall && @@ -284,7 +297,6 @@ export default class Call { this.callArgs && this.callArgs.onStream(stream); }); - // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) call.on('close', this.onRemoteCallEnd); call.on('error', (e) => { @@ -301,7 +313,7 @@ export default class Call { clean() { this.cleaned = true; // sometimes cleaned before modules loaded - this.initiateCallEnd(); + void this.initiateCallEnd(); if (this._peer) { console.log('destroying peer...'); const peer = this._peer; // otherwise it calls reconnection on data chan close diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 7c847efce..e7364f1b7 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "9.0.0", + "version": "9.0.1-3", "keywords": [ "WebRTC", "assistance", diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index fe50eed58..0fe2116d2 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -156,8 +156,18 @@ export default class Assist { // @ts-ignore No need in statistics messages. TODO proper filter if (batchSize === 2 && messages[0]._id === 0 && messages[1]._id === 49) { return } if (batchSize > this.options.compressionMinBatchSize && this.options.compressionEnabled) { - while (messages.length > 0) { - const batch = messages.splice(0, this.options.compressionMinBatchSize) + const toSend: any[] = [] + if (batchSize > 10000) { + const middle = Math.floor(batchSize / 2) + const firstHalf = messages.slice(0, middle) + const secondHalf = messages.slice(middle) + + toSend.push(firstHalf) + toSend.push(secondHalf) + } else { + toSend.push(messages) + } + toSend.forEach(batch => { const str = JSON.stringify(batch) const byteArr = new TextEncoder().encode(str) gzip(byteArr, { mtime: 0, }, (err, result) => { @@ -167,7 +177,7 @@ export default class Assist { this.emit('messages_gz', result) } }) - } + }) } else { this.emit('messages', messages) } @@ -401,6 +411,10 @@ export default class Assist { if (app.getTabId() !== info.meta.tabId) return const name = info.data callingAgents.set(id, name) + + if (!this.peer) { + setupPeer() + } updateCallerNames() }) socket.on('videofeed', (_, info) => { @@ -429,7 +443,10 @@ export default class Assist { // TODO: merge peerId & socket.io id (simplest way - send peerId with the name) const calls: Record = {} // !! uses peerJS ID const lStreams: Record = {} - // const callingPeers: Map = new Map() // Maybe + + function updateCallerNames() { + callUI?.setAssistentName(callingAgents) + } function endAgentCall(id: string) { callingAgents.delete(id) if (callingAgents.size === 0) { @@ -439,59 +456,6 @@ export default class Assist { //TODO: close() specific call and corresponding lStreams (after connecting peerId & socket.io id) } } - - // PeerJS call (todo: use native WebRTC) - const peerOptions = { - host: this.getHost(), - path: this.getBasePrefixUrl()+'/assist', - port: location.protocol === 'http:' && this.noSecureMode ? 80 : 443, - debug: 2, //appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs. - } - if (this.options.config) { - peerOptions['config'] = this.options.config - } - - const peer = new safeCastedPeer(peerID, peerOptions) as Peer - this.peer = peer - let peerReconnectAttempts = 0 - // @ts-ignore (peerjs typing) - peer.on('error', e => app.debug.warn('Peer error: ', e.type, e)) - peer.on('disconnected', () => { - if (peerReconnectAttempts < 30) { - this.peerReconnectTimeout = setTimeout(() => { - if (this.app.active() && !peer.destroyed) { - peer.reconnect() - } - }, Math.min(peerReconnectAttempts, 8) * 2 * 1000) - peerReconnectAttempts += 1 - } - }) - - function updateCallerNames() { - callUI?.setAssistentName(callingAgents) - } - - const closeCallConfirmWindow = () => { - if (callConfirmWindow) { - callConfirmWindow.remove() - callConfirmWindow = null - callConfirmAnswer = null - } - } - const requestCallConfirm = () => { - if (callConfirmAnswer) { // Already asking - return callConfirmAnswer - } - callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || { - text: this.options.confirmText, - style: this.options.confirmStyle, - })) // TODO: reuse ? - return callConfirmAnswer = callConfirmWindow.mount().then(answer => { - closeCallConfirmWindow() - return answer - }) - } - const handleCallEnd = () => { // Complete stop and clear all calls // Streams Object.values(calls).forEach(call => call.close()) @@ -517,102 +481,154 @@ export default class Assist { callEndCallback?.() } - const initiateCallEnd = () => { - this.emit('call_end') - handleCallEnd() + const closeCallConfirmWindow = () => { + if (callConfirmWindow) { + callConfirmWindow.remove() + callConfirmWindow = null + callConfirmAnswer = null + } } - const updateVideoFeed = ({ enabled, }) => this.emit('videofeed', { streamId: this.peer?.id, enabled, }) - peer.on('call', (call) => { - app.debug.log('Incoming call from', call.peer) - let confirmAnswer: Promise - const callingPeerIds = JSON.parse(sessionStorage.getItem(this.options.session_calling_peer_key) || '[]') - if (callingPeerIds.includes(call.peer) || this.callingState === CallingState.True) { - confirmAnswer = Promise.resolve(true) - } else { - this.setCallingState(CallingState.Requesting) - confirmAnswer = requestCallConfirm() - this.playNotificationSound() // For every new agent during confirmation here - - // TODO: only one (latest) timeout - setTimeout(() => { - if (this.callingState !== CallingState.Requesting) { return } - initiateCallEnd() - }, 30000) + // PeerJS call (todo: use native WebRTC) + const peerOptions = { + host: this.getHost(), + path: this.getBasePrefixUrl()+'/assist', + port: location.protocol === 'http:' && this.noSecureMode ? 80 : 443, + debug: 2, //appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs. + } + const setupPeer = () => { + if (this.options.config) { + peerOptions['config'] = this.options.config } - confirmAnswer.then(async agreed => { - if (!agreed) { - initiateCallEnd() - this.options.onCallDeny?.() - return - } - // Request local stream for the new connection - try { - // lStreams are reusable so fare we don't delete them in the `endAgentCall` - if (!lStreams[call.peer]) { - app.debug.log('starting new stream for', call.peer) - lStreams[call.peer] = await RequestLocalStream() - } - calls[call.peer] = call - } catch (e) { - app.debug.error('Audio media device request error:', e) - initiateCallEnd() - return + const peer = new safeCastedPeer(peerID, peerOptions) as Peer + this.peer = peer + let peerReconnectAttempts = 0 + // @ts-ignore (peerjs typing) + peer.on('error', e => app.debug.warn('Peer error: ', e.type, e)) + peer.on('disconnected', () => { + if (peerReconnectAttempts < 30) { + this.peerReconnectTimeout = setTimeout(() => { + if (this.app.active() && !peer.destroyed) { + peer.reconnect() + } + }, Math.min(peerReconnectAttempts, 8) * 2 * 1000) + peerReconnectAttempts += 1 } + }) - if (!callUI) { - callUI = new CallWindow(app.debug.error, this.options.callUITemplate) - callUI.setVideoToggleCallback(updateVideoFeed) - } - callUI.showControls(initiateCallEnd) - if (!annot) { - annot = new AnnotationCanvas() - annot.mount() + const requestCallConfirm = () => { + if (callConfirmAnswer) { // Already asking + return callConfirmAnswer } - // have to be updated - callUI.setLocalStreams(Object.values(lStreams)) - - call.on('error', e => { - app.debug.warn('Call error:', e) - initiateCallEnd() - }) - call.on('stream', (rStream) => { - callUI?.addRemoteStream(rStream, call.peer) - const onInteraction = () => { // do only if document.hidden ? - callUI?.playRemote() - document.removeEventListener('click', onInteraction) - } - document.addEventListener('click', onInteraction) + callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || { + text: this.options.confirmText, + style: this.options.confirmStyle, + })) // TODO: reuse ? + return callConfirmAnswer = callConfirmWindow.mount().then(answer => { + closeCallConfirmWindow() + return answer }) + } - // remote video on/off/camera change - lStreams[call.peer].onVideoTrack(vTrack => { - const sender = call.peerConnection.getSenders().find(s => s.track?.kind === 'video') - if (!sender) { - app.debug.warn('No video sender found') + const initiateCallEnd = () => { + this.emit('call_end') + handleCallEnd() + } + const updateVideoFeed = ({ enabled, }) => this.emit('videofeed', { streamId: this.peer?.id, enabled, }) + + peer.on('call', (call) => { + app.debug.log('Incoming call from', call.peer) + let confirmAnswer: Promise + const callingPeerIds = JSON.parse(sessionStorage.getItem(this.options.session_calling_peer_key) || '[]') + if (callingPeerIds.includes(call.peer) || this.callingState === CallingState.True) { + confirmAnswer = Promise.resolve(true) + } else { + this.setCallingState(CallingState.Requesting) + confirmAnswer = requestCallConfirm() + this.playNotificationSound() // For every new agent during confirmation here + + // TODO: only one (latest) timeout + setTimeout(() => { + if (this.callingState !== CallingState.Requesting) { return } + initiateCallEnd() + }, 30000) + } + + confirmAnswer.then(async agreed => { + if (!agreed) { + initiateCallEnd() + this.options.onCallDeny?.() return } - app.debug.log('sender found:', sender) - void sender.replaceTrack(vTrack) + // Request local stream for the new connection + try { + // lStreams are reusable so fare we don't delete them in the `endAgentCall` + if (!lStreams[call.peer]) { + app.debug.log('starting new stream for', call.peer) + lStreams[call.peer] = await RequestLocalStream() + } + calls[call.peer] = call + } catch (e) { + app.debug.error('Audio media device request error:', e) + initiateCallEnd() + return + } + + if (!callUI) { + callUI = new CallWindow(app.debug.error, this.options.callUITemplate) + callUI.setVideoToggleCallback(updateVideoFeed) + } + callUI.showControls(initiateCallEnd) + + if (!annot) { + annot = new AnnotationCanvas() + annot.mount() + } + // have to be updated + callUI.setLocalStreams(Object.values(lStreams)) + + call.on('error', e => { + app.debug.warn('Call error:', e) + initiateCallEnd() + }) + call.on('stream', (rStream) => { + callUI?.addRemoteStream(rStream, call.peer) + const onInteraction = () => { // do only if document.hidden ? + callUI?.playRemote() + document.removeEventListener('click', onInteraction) + } + document.addEventListener('click', onInteraction) + }) + + // remote video on/off/camera change + lStreams[call.peer].onVideoTrack(vTrack => { + const sender = call.peerConnection.getSenders().find(s => s.track?.kind === 'video') + if (!sender) { + app.debug.warn('No video sender found') + return + } + app.debug.log('sender found:', sender) + void sender.replaceTrack(vTrack) + }) + + call.answer(lStreams[call.peer].stream) + document.addEventListener('visibilitychange', () => { + initiateCallEnd() + }) + + this.setCallingState(CallingState.True) + if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() } + + const callingPeerIds = Object.keys(calls) + sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIds)) + this.emit('UPDATE_SESSION', { agentIds: callingPeerIds, isCallActive: true, }) + }).catch(reason => { // in case of Confirm.remove() without user answer (not an error) + app.debug.log(reason) }) - - call.answer(lStreams[call.peer].stream) - document.addEventListener('visibilitychange', () => { - initiateCallEnd() - }) - - this.setCallingState(CallingState.True) - if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() } - - const callingPeerIds = Object.keys(calls) - sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIds)) - this.emit('UPDATE_SESSION', { agentIds: callingPeerIds, isCallActive: true, }) - }).catch(reason => { // in case of Confirm.remove() without user answer (not a error) - app.debug.log(reason) }) - }) + } const startCanvasStream = (stream: MediaStream, id: number) => { @@ -668,7 +684,7 @@ export default class Assist { } private clean() { - // sometimes means new agent connected so we keep id for control + // sometimes means new agent connected, so we keep id for control this.remoteControl?.releaseControl(false, true) if (this.peerReconnectTimeout) { clearTimeout(this.peerReconnectTimeout) diff --git a/tracker/tracker-assist/src/version.ts b/tracker/tracker-assist/src/version.ts index ccebb6b43..e83b5c612 100644 --- a/tracker/tracker-assist/src/version.ts +++ b/tracker/tracker-assist/src/version.ts @@ -1 +1 @@ -export const pkgVersion = '8.0.5-4' +export const pkgVersion = '9.0.1-3'