From 6b35df7125cd89cab62c0d79e5fa12f558fa096c Mon Sep 17 00:00:00 2001 From: Andrey Babushkin <55714097+reyand43@users.noreply.github.com> Date: Mon, 31 Mar 2025 18:13:51 +0200 Subject: [PATCH] pulled updates (#3254) --- assist/utils/assistHelper.js | 3 +- assist/utils/socketHandlers.js | 13 +- .../app/player/web/assist/AssistManager.ts | 2 +- frontend/app/player/web/assist/Call.ts | 5 +- tracker/tracker-assist/src/Assist.ts | 892 ++++++++++-------- 5 files changed, 523 insertions(+), 392 deletions(-) diff --git a/assist/utils/assistHelper.js b/assist/utils/assistHelper.js index ce90aa2a8..8e0e6589c 100644 --- a/assist/utils/assistHelper.js +++ b/assist/utils/assistHelper.js @@ -26,7 +26,8 @@ EVENTS_DEFINITION.emit = { NO_SESSIONS: "SESSION_DISCONNECTED", SESSION_ALREADY_CONNECTED: "SESSION_ALREADY_CONNECTED", SESSION_RECONNECTED: "SESSION_RECONNECTED", - UPDATE_EVENT: EVENTS_DEFINITION.listen.UPDATE_EVENT + UPDATE_EVENT: EVENTS_DEFINITION.listen.UPDATE_EVENT, + WEBRTC_CONFIG: "WEBRTC_CONFIG", }; const BASE_sessionInfo = { diff --git a/assist/utils/socketHandlers.js b/assist/utils/socketHandlers.js index 7921b4f9e..019408d64 100644 --- a/assist/utils/socketHandlers.js +++ b/assist/utils/socketHandlers.js @@ -42,7 +42,7 @@ const findSessionSocketId = async (io, roomId, tabId) => { }; async function getRoomData(io, roomID) { - let tabsCount = 0, agentsCount = 0, tabIDs = [], agentIDs = []; + let tabsCount = 0, agentsCount = 0, tabIDs = [], agentIDs = [], config = null; const connected_sockets = await io.in(roomID).fetchSockets(); if (connected_sockets.length > 0) { for (let socket of connected_sockets) { @@ -52,13 +52,16 @@ async function getRoomData(io, roomID) { } else { agentsCount++; agentIDs.push(socket.id); + if (socket.handshake.query.config !== undefined) { + config = socket.handshake.query.config; + } } } } else { tabsCount = -1; agentsCount = -1; } - return {tabsCount, agentsCount, tabIDs, agentIDs}; + return {tabsCount, agentsCount, tabIDs, agentIDs, config}; } function processNewSocket(socket) { @@ -78,7 +81,7 @@ async function onConnect(socket) { IncreaseOnlineConnections(socket.handshake.query.identity); const io = getServer(); - const {tabsCount, agentsCount, tabIDs, agentIDs} = await getRoomData(io, socket.handshake.query.roomId); + const {tabsCount, agentsCount, tabIDs, agentIDs, config} = await getRoomData(io, socket.handshake.query.roomId); if (socket.handshake.query.identity === IDENTITIES.session) { // Check if session with the same tabID already connected, if so, refuse new connexion @@ -100,6 +103,7 @@ async function onConnect(socket) { // Inform all connected agents about reconnected session if (agentsCount > 0) { logger.debug(`notifying new session about agent-existence`); + io.to(socket.id).emit(EVENTS_DEFINITION.emit.WEBRTC_CONFIG, config); io.to(socket.id).emit(EVENTS_DEFINITION.emit.AGENTS_CONNECTED, agentIDs); socket.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.SESSION_RECONNECTED, socket.id); } @@ -118,7 +122,8 @@ async function onConnect(socket) { // Stats startAssist(socket, socket.handshake.query.agentID); } - socket.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.NEW_AGENT, socket.id, { ...socket.handshake.query.agentInfo, config: socket.handshake.query.config }); + io.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.WEBRTC_CONFIG, socket.handshake.query.config); + socket.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.NEW_AGENT, socket.id, { ...socket.handshake.query.agentInfo }); } // Set disconnect handler diff --git a/frontend/app/player/web/assist/AssistManager.ts b/frontend/app/player/web/assist/AssistManager.ts index 1b107cf17..499e32dea 100644 --- a/frontend/app/player/web/assist/AssistManager.ts +++ b/frontend/app/player/web/assist/AssistManager.ts @@ -372,7 +372,7 @@ export default class AssistManager { 'stun:stun1.l.google.com:19302', 'stun:stun2.l.google.com:19302', 'stun:stun3.l.google.com:19302', - 'stun:stun4.l.google.com:19302' + 'stun:stun4.l.google.com:19302', ], }, ] as RTCIceServer[]; diff --git a/frontend/app/player/web/assist/Call.ts b/frontend/app/player/web/assist/Call.ts index 972649814..c2145c07b 100644 --- a/frontend/app/player/web/assist/Call.ts +++ b/frontend/app/player/web/assist/Call.ts @@ -365,10 +365,7 @@ export default class Call { const pc = this.connections[callId]; if (!pc) return; // if there are ice candidates then add candidate to peer - if ( - data.candidate && - (data.candidate.sdpMid || data.candidate.sdpMLineIndex !== null) - ) { + if (data.candidate) { try { await pc.addIceCandidate(new RTCIceCandidate(data.candidate)); } catch (e) { diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 85ec6fa4f..70da5d717 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -1,31 +1,31 @@ /* eslint-disable @typescript-eslint/no-empty-function */ -import type { Socket } from 'socket.io-client' -import { connect } from 'socket.io-client' -import type { Properties } from 'csstype' -import { App } from '@openreplay/tracker' +import type { Socket } from "socket.io-client"; +import { connect } from "socket.io-client"; +import type { Properties } from "csstype"; +import { App } from "@openreplay/tracker"; -import RequestLocalStream, { LocalStream } from './LocalStream.js' -import { hasTag } from './guards.js' -import RemoteControl, { RCStatus } from './RemoteControl.js' -import CallWindow from './CallWindow.js' -import AnnotationCanvas from './AnnotationCanvas.js' -import ConfirmWindow from './ConfirmWindow/ConfirmWindow.js' -import { callConfirmDefault } from './ConfirmWindow/defaults.js' -import type { Options as ConfirmOptions } from './ConfirmWindow/defaults.js' -import ScreenRecordingState from './ScreenRecordingState.js' -import { pkgVersion } from './version.js' -import Canvas from './Canvas.js' -import { gzip } from 'fflate' +import RequestLocalStream, { LocalStream } from "./LocalStream.js"; +import { hasTag } from "./guards.js"; +import RemoteControl, { RCStatus } from "./RemoteControl.js"; +import CallWindow from "./CallWindow.js"; +import AnnotationCanvas from "./AnnotationCanvas.js"; +import ConfirmWindow from "./ConfirmWindow/ConfirmWindow.js"; +import { callConfirmDefault } from "./ConfirmWindow/defaults.js"; +import type { Options as ConfirmOptions } from "./ConfirmWindow/defaults.js"; +import ScreenRecordingState from "./ScreenRecordingState.js"; +import { pkgVersion } from "./version.js"; +import Canvas from "./Canvas.js"; +import { gzip } from "fflate"; -type StartEndCallback = (agentInfo?: Record) => ((() => any) | void) +type StartEndCallback = (agentInfo?: Record) => (() => any) | void; interface AgentInfo { config: string; email: string; - id: number - name: string - peerId: string - query: string + id: number; + name: string; + peerId: string; + query: string; } export interface Options { @@ -48,7 +48,6 @@ export interface Options { // @deprecated confirmStyle?: Properties; - config: RTCConfiguration; serverURL: string; callUITemplate?: string; compressionEnabled: boolean; @@ -65,175 +64,205 @@ enum CallingState { False, } -type OptionalCallback = (() => Record) | void +type OptionalCallback = (() => Record) | void; type Agent = { - onDisconnect?: OptionalCallback, - onControlReleased?: OptionalCallback, - agentInfo: AgentInfo | undefined + onDisconnect?: OptionalCallback; + onControlReleased?: OptionalCallback; + agentInfo: AgentInfo | undefined; // -} - +}; export default class Assist { - readonly version = pkgVersion + readonly version = pkgVersion; - private socket: Socket | null = null + private socket: Socket | null = null; private calls: Map = new Map(); - private canvasPeers: { [id: number]: RTCPeerConnection | null } = {} - private canvasNodeCheckers: Map = new Map() - private assistDemandedRestart = false - private callingState: CallingState = CallingState.False + private canvasPeers: { [id: number]: RTCPeerConnection | null } = {}; + private canvasNodeCheckers: Map = new Map(); + private assistDemandedRestart = false; + private callingState: CallingState = CallingState.False; private remoteControl: RemoteControl | null = null; - private peerReconnectTimeout: ReturnType | null = null - private agents: Record = {} - private config: RTCIceServer[] | undefined - private readonly options: Options - private readonly canvasMap: Map = new Map() + private peerReconnectTimeout: ReturnType | null = null; + private agents: Record = {}; + private config: RTCIceServer[] | undefined; + private readonly options: Options; + private readonly canvasMap: Map = new Map(); + private iceCandidatesBuffer: Map = new Map(); constructor( private readonly app: App, options?: Partial, - private readonly noSecureMode: boolean = false, + private readonly noSecureMode: boolean = false ) { // @ts-ignore - window.__OR_ASSIST_VERSION = this.version - this.options = Object.assign({ - session_calling_peer_key: '__openreplay_calling_peer', - session_control_peer_key: '__openreplay_control_peer', - config: null, - serverURL: null, - onCallStart: () => { }, - onAgentConnect: () => { }, - onRemoteControlStart: () => { }, - callConfirm: {}, - controlConfirm: {}, // TODO: clear options passing/merging/overwriting - recordingConfirm: {}, - socketHost: '', - compressionEnabled: false, - compressionMinBatchSize: 5000, - }, - options, - ) + window.__OR_ASSIST_VERSION = this.version; + this.options = Object.assign( + { + session_calling_peer_key: "__openreplay_calling_peer", + session_control_peer_key: "__openreplay_control_peer", + config: null, + serverURL: null, + onCallStart: () => {}, + onAgentConnect: () => {}, + onRemoteControlStart: () => {}, + callConfirm: {}, + controlConfirm: {}, // TODO: clear options passing/merging/overwriting + recordingConfirm: {}, + socketHost: "", + compressionEnabled: false, + compressionMinBatchSize: 5000, + }, + options + ); if (this.app.options.assistSocketHost) { - this.options.socketHost = this.app.options.assistSocketHost + this.options.socketHost = this.app.options.assistSocketHost; } if (document.hidden !== undefined) { - const sendActivityState = (): void => this.emit('UPDATE_SESSION', { active: !document.hidden, }) + const sendActivityState = (): void => + this.emit("UPDATE_SESSION", { active: !document.hidden }); app.attachEventListener( document, - 'visibilitychange', + "visibilitychange", sendActivityState, false, - false, - ) + false + ); } - const titleNode = document.querySelector('title') - const observer = titleNode && new MutationObserver(() => { - this.emit('UPDATE_SESSION', { pageTitle: document.title, }) - }) + const titleNode = document.querySelector("title"); + const observer = + titleNode && + new MutationObserver(() => { + this.emit("UPDATE_SESSION", { pageTitle: document.title }); + }); app.addOnUxtCb((uxtId: number) => { - this.emit('UPDATE_SESSION', { uxtId, }) - }) + this.emit("UPDATE_SESSION", { uxtId }); + }); app.attachStartCallback(() => { - if (this.assistDemandedRestart) { return } - this.onStart() - observer && observer.observe(titleNode, { subtree: true, characterData: true, childList: true, }) - }) + if (this.assistDemandedRestart) { + return; + } + this.onStart(); + observer && + observer.observe(titleNode, { + subtree: true, + characterData: true, + childList: true, + }); + }); app.attachStopCallback(() => { - if (this.assistDemandedRestart) { return } - this.clean() - observer && observer.disconnect() - }) + if (this.assistDemandedRestart) { + return; + } + this.clean(); + observer && observer.disconnect(); + }); app.attachCommitCallback((messages) => { if (this.agentsConnected) { - const batchSize = messages.length + const batchSize = messages.length; // @ts-ignore No need in statistics messages. TODO proper filter - if (batchSize === 2 && messages[0]._id === 0 && messages[1]._id === 49) { return } - if (batchSize > this.options.compressionMinBatchSize && this.options.compressionEnabled) { - const toSend: any[] = [] + if ( + batchSize === 2 && + // @ts-ignore No need in statistics messages. TODO proper filter + messages[0]._id === 0 && + // @ts-ignore No need in statistics messages. TODO proper filter + messages[1]._id === 49 + ) { + return; + } + if ( + batchSize > this.options.compressionMinBatchSize && + this.options.compressionEnabled + ) { + const toSend: any[] = []; if (batchSize > 10000) { - const middle = Math.floor(batchSize / 2) - const firstHalf = messages.slice(0, middle) - const secondHalf = messages.slice(middle) + const middle = Math.floor(batchSize / 2); + const firstHalf = messages.slice(0, middle); + const secondHalf = messages.slice(middle); - toSend.push(firstHalf) - toSend.push(secondHalf) + toSend.push(firstHalf); + toSend.push(secondHalf); } else { - toSend.push(messages) + toSend.push(messages); } - toSend.forEach(batch => { - const str = JSON.stringify(batch) - const byteArr = new TextEncoder().encode(str) - gzip(byteArr, { mtime: 0, }, (err, result) => { + toSend.forEach((batch) => { + const str = JSON.stringify(batch); + const byteArr = new TextEncoder().encode(str); + gzip(byteArr, { mtime: 0 }, (err, result) => { if (err) { - this.emit('messages', batch) + this.emit("messages", batch); } else { - this.emit('messages_gz', result) + this.emit("messages_gz", result); } - }) - }) + }); + }); } else { - this.emit('messages', messages) + this.emit("messages", messages); } } - }) - app.session.attachUpdateCallback(sessInfo => this.emit('UPDATE_SESSION', sessInfo)) + }); + app.session.attachUpdateCallback((sessInfo) => + this.emit("UPDATE_SESSION", sessInfo) + ); } private emit(ev: string, args?: any): void { - this.socket && this.socket.emit(ev, { meta: { tabId: this.app.getTabId(), }, data: args, }) + this.socket && + this.socket.emit(ev, { + meta: { tabId: this.app.getTabId() }, + data: args, + }); } private get agentsConnected(): boolean { - return Object.keys(this.agents).length > 0 + return Object.keys(this.agents).length > 0; } private readonly setCallingState = (newState: CallingState): void => { - this.callingState = newState - } + this.callingState = newState; + }; private getHost(): string { if (this.options.socketHost) { - return this.options.socketHost + return this.options.socketHost; } if (this.options.serverURL) { - return new URL(this.options.serverURL).host + return new URL(this.options.serverURL).host; } - return this.app.getHost() + return this.app.getHost(); } private getBasePrefixUrl(): string { if (this.options.serverURL) { - return new URL(this.options.serverURL).pathname + return new URL(this.options.serverURL).pathname; } - return '' + return ""; } private onStart() { - const app = this.app - const sessionId = app.getSessionID() + const app = this.app; + const sessionId = app.getSessionID(); // Common for all incoming call requests - let callUI: CallWindow | null = null - let annot: AnnotationCanvas | null = null + let callUI: CallWindow | null = null; + let annot: AnnotationCanvas | null = null; // TODO: encapsulate - let callConfirmWindow: ConfirmWindow | null = null - let callConfirmAnswer: Promise | null = null - let callEndCallback: ReturnType | null = null + let callConfirmWindow: ConfirmWindow | null = null; + let callConfirmAnswer: Promise | null = null; + let callEndCallback: ReturnType | null = null; if (!sessionId) { - return app.debug.error('No session ID') + return app.debug.error("No session ID"); } - const peerID = `${app.getProjectKey()}-${sessionId}-${this.app.getTabId()}` + const peerID = `${app.getProjectKey()}-${sessionId}-${this.app.getTabId()}`; // SocketIO - const socket = this.socket = connect(this.getHost(), { - path: this.getBasePrefixUrl() + '/ws-assist/socket', + const socket = (this.socket = connect(this.getHost(), { + path: this.getBasePrefixUrl() + "/ws-assist/socket", query: { - 'peerId': peerID, - 'identity': 'session', - 'tabId': this.app.getTabId(), - 'sessionInfo': JSON.stringify({ - 'uxtId': this.app.getUxtId() ?? undefined, + peerId: peerID, + identity: "session", + tabId: this.app.getTabId(), + sessionInfo: JSON.stringify({ + uxtId: this.app.getUxtId() ?? undefined, pageTitle: document.title, active: true, assistOnly: this.app.socketMode, @@ -247,342 +276,424 @@ export default class Assist { reconnectionDelay: 1000, reconnectionDelayMax: 25000, randomizationFactor: 0.5, - }) + })); socket.onAny((...args) => { - if (args[0] === 'messages' || args[0] === 'UPDATE_SESSION') { - return + if (args[0] === "messages" || args[0] === "UPDATE_SESSION") { + return; } - if (args[0] !== 'webrtc_call_ice_candidate') { + if (args[0] !== "webrtc_call_ice_candidate") { app.debug.log("Socket:", ...args); - }; - socket.on('close', (e) => { - app.debug.warn('Socket closed:', e); - }) - }) + } + socket.on("close", (e) => { + app.debug.warn("Socket closed:", e); + }); + }); const onGrand = (id: string) => { if (!callUI) { - callUI = new CallWindow(app.debug.error, this.options.callUITemplate) + callUI = new CallWindow(app.debug.error, this.options.callUITemplate); } if (this.remoteControl) { - callUI?.showRemoteControl(this.remoteControl.releaseControl) + callUI?.showRemoteControl(this.remoteControl.releaseControl); } - this.agents[id] = { ...this.agents[id], onControlReleased: this.options.onRemoteControlStart(this.agents[id]?.agentInfo), } - this.emit('control_granted', id) - annot = new AnnotationCanvas() - annot.mount() - return callingAgents.get(id) - } + this.agents[id] = { + ...this.agents[id], + onControlReleased: this.options.onRemoteControlStart( + this.agents[id]?.agentInfo + ), + }; + this.emit("control_granted", id); + annot = new AnnotationCanvas(); + annot.mount(); + return callingAgents.get(id); + }; const onRelease = (id?: string | null, isDenied?: boolean) => { if (id) { - const cb = this.agents[id].onControlReleased - delete this.agents[id].onControlReleased - typeof cb === 'function' && cb() - this.emit('control_rejected', id) + const cb = this.agents[id].onControlReleased; + delete this.agents[id].onControlReleased; + typeof cb === "function" && cb(); + this.emit("control_rejected", id); } if (annot != null) { - annot.remove() - annot = null + annot.remove(); + annot = null; } - callUI?.hideRemoteControl() + callUI?.hideRemoteControl(); if (this.callingState !== CallingState.True) { - callUI?.remove() - callUI = null + callUI?.remove(); + callUI = null; } if (isDenied) { - const info = id ? this.agents[id]?.agentInfo : {} - this.options.onRemoteControlDeny?.(info || {}) + const info = id ? this.agents[id]?.agentInfo : {}; + this.options.onRemoteControlDeny?.(info || {}); } - } + }; this.remoteControl = new RemoteControl( this.options, onGrand, (id, isDenied) => onRelease(id, isDenied), - (id) => this.emit('control_busy', id), - ) + (id) => this.emit("control_busy", id) + ); const onAcceptRecording = () => { - socket.emit('recording_accepted') - } + socket.emit("recording_accepted"); + }; const onRejectRecording = (agentData: AgentInfo) => { - socket.emit('recording_rejected') + socket.emit("recording_rejected"); - this.options.onRecordingDeny?.(agentData || {}) - } - const recordingState = new ScreenRecordingState(this.options.recordingConfirm) + this.options.onRecordingDeny?.(agentData || {}); + }; + const recordingState = new ScreenRecordingState( + this.options.recordingConfirm + ); - function processEvent(agentId: string, event: { meta: { tabId: string }, data?: any }, callback?: (id: string, data: any) => void) { + function processEvent( + agentId: string, + event: { meta: { tabId: string }; data?: any }, + callback?: (id: string, data: any) => void + ) { if (app.getTabId() === event.meta.tabId) { - return callback?.(agentId, event.data) + return callback?.(agentId, event.data); } } if (this.remoteControl !== null) { - socket.on('request_control', (agentId, dataObj) => { - processEvent(agentId, dataObj, this.remoteControl?.requestControl) - }) - socket.on('release_control', (agentId, dataObj) => { + socket.on("request_control", (agentId, dataObj) => { + processEvent(agentId, dataObj, this.remoteControl?.requestControl); + }); + socket.on("release_control", (agentId, dataObj) => { processEvent(agentId, dataObj, (_, data) => this.remoteControl?.releaseControl(data) - ) - }) - socket.on('scroll', (id, event) => processEvent(id, event, this.remoteControl?.scroll)) - socket.on('click', (id, event) => processEvent(id, event, this.remoteControl?.click)) - socket.on('move', (id, event) => processEvent(id, event, this.remoteControl?.move)) - socket.on('focus', (id, event) => processEvent(id, event, (clientID, nodeID) => { - const el = app.nodes.getNode(nodeID) - if (el instanceof HTMLElement && this.remoteControl) { - this.remoteControl.focus(clientID, el) - } - })) - socket.on('input', (id, event) => processEvent(id, event, this.remoteControl?.input)) + ); + }); + socket.on("scroll", (id, event) => + processEvent(id, event, this.remoteControl?.scroll) + ); + socket.on("click", (id, event) => + processEvent(id, event, this.remoteControl?.click) + ); + socket.on("move", (id, event) => + processEvent(id, event, this.remoteControl?.move) + ); + socket.on("focus", (id, event) => + processEvent(id, event, (clientID, nodeID) => { + const el = app.nodes.getNode(nodeID); + if (el instanceof HTMLElement && this.remoteControl) { + this.remoteControl.focus(clientID, el); + } + }) + ); + socket.on("input", (id, event) => + processEvent(id, event, this.remoteControl?.input) + ); } - // TODO: restrict by id - socket.on('moveAnnotation', (id, event) => processEvent(id, event, (_, d) => annot && annot.move(d))) - socket.on('startAnnotation', (id, event) => processEvent(id, event, (_, d) => annot?.start(d))) - socket.on('stopAnnotation', (id, event) => processEvent(id, event, annot?.stop)) + socket.on("moveAnnotation", (id, event) => + processEvent(id, event, (_, d) => annot && annot.move(d)) + ); + socket.on("startAnnotation", (id, event) => + processEvent(id, event, (_, d) => annot?.start(d)) + ); + socket.on("stopAnnotation", (id, event) => + processEvent(id, event, annot?.stop) + ); - socket.on('NEW_AGENT', (id: string, info: AgentInfo) => { + socket.on( + "WEBRTC_CONFIG", + (config: string) => { + if (config) { + this.config = JSON.parse(config) + } + } + ); + + socket.on("NEW_AGENT", (id: string, info: AgentInfo) => { this.cleanCanvasConnections(); this.agents[id] = { onDisconnect: this.options.onAgentConnect?.(info), agentInfo: info, // TODO ? - } + }; if (this.app.active()) { - this.assistDemandedRestart = true - this.app.stop() - this.app.clearBuffers() - this.app.waitStatus(0) - .then(() => { - this.config = JSON.parse(info.config); - }) - .then(() => { - this.app.allowAppStart() - setTimeout(() => { - this.app.start().then(() => { this.assistDemandedRestart = false }) - .then(() => { - this.remoteControl?.reconnect([id,]) - }) - .catch(e => app.debug.error(e)) - // TODO: check if it's needed; basically allowing some time for the app to finish everything before starting again - }, 100) - }) + this.assistDemandedRestart = true; + this.app.stop(); + this.app.clearBuffers(); + this.app.waitStatus(0).then(() => { + this.app.allowAppStart(); + setTimeout(() => { + this.app + .start() + .then(() => { + this.assistDemandedRestart = false; + }) + .then(() => { + this.remoteControl?.reconnect([id]); + }) + .catch((e) => app.debug.error(e)); + // TODO: check if it's needed; basically allowing some time for the app to finish everything before starting again + }, 100); + }); } - }) + }); - socket.on('AGENTS_CONNECTED', (ids: string[]) => { + socket.on("AGENTS_CONNECTED", (ids: string[]) => { this.cleanCanvasConnections(); - ids.forEach(id => { - const agentInfo = this.agents[id]?.agentInfo + ids.forEach((id) => { + const agentInfo = this.agents[id]?.agentInfo; this.agents[id] = { agentInfo, onDisconnect: this.options.onAgentConnect?.(agentInfo), - } - }) + }; + }); if (this.app.active()) { - this.assistDemandedRestart = true - this.app.stop() - this.app.waitStatus(0) - .then(() => { - this.app.allowAppStart() - setTimeout(() => { - this.app.start().then(() => { this.assistDemandedRestart = false }) - .then(() => { - this.remoteControl?.reconnect(ids) - }) - .catch(e => app.debug.error(e)) - }, 100) - }) + this.assistDemandedRestart = true; + this.app.stop(); + this.app.waitStatus(0).then(() => { + this.app.allowAppStart(); + setTimeout(() => { + this.app + .start() + .then(() => { + this.assistDemandedRestart = false; + }) + .then(() => { + this.remoteControl?.reconnect(ids); + }) + .catch((e) => app.debug.error(e)); + }, 100); + }); } - }) + }); - socket.on('AGENT_DISCONNECTED', (id) => { - this.remoteControl?.releaseControl() + socket.on("AGENT_DISCONNECTED", (id) => { + this.remoteControl?.releaseControl(); - this.agents[id]?.onDisconnect?.() - delete this.agents[id] + this.agents[id]?.onDisconnect?.(); + delete this.agents[id]; - Object.values(this.calls).forEach(pc => pc.close()) + Object.values(this.calls).forEach((pc) => pc.close()); this.calls.clear(); - recordingState.stopAgentRecording(id) - endAgentCall({ socketId: id }) - }) + recordingState.stopAgentRecording(id); + endAgentCall({ socketId: id }); + }); - socket.on('NO_AGENT', () => { - Object.values(this.agents).forEach(a => a.onDisconnect?.()) + socket.on("NO_AGENT", () => { + Object.values(this.agents).forEach((a) => a.onDisconnect?.()); this.cleanCanvasConnections(); - this.agents = {} - if (recordingState.isActive) recordingState.stopRecording() - }) + this.agents = {}; + if (recordingState.isActive) recordingState.stopRecording(); + }); - socket.on('call_end', (socketId, { data: callId }) => { + socket.on("call_end", (socketId, { data: callId }) => { if (!callingAgents.has(socketId)) { - app.debug.warn('Received call_end from unknown agent', socketId) - return + app.debug.warn("Received call_end from unknown agent", socketId); + return; } - endAgentCall({ socketId, callId }) - }) + endAgentCall({ socketId, callId }); + }); - socket.on('_agent_name', (id, info) => { - if (app.getTabId() !== info.meta.tabId) return - const name = info.data - callingAgents.set(id, name) - updateCallerNames() - }) + socket.on("_agent_name", (id, info) => { + if (app.getTabId() !== info.meta.tabId) return; + const name = info.data; + callingAgents.set(id, name); + updateCallerNames(); + }); - socket.on('webrtc_canvas_answer', async (_, data: { answer, id }) => { + socket.on("webrtc_canvas_answer", async (_, data: { answer; id }) => { const pc = this.canvasPeers[data.id]; if (pc) { try { await pc.setRemoteDescription(new RTCSessionDescription(data.answer)); } catch (e) { - app.debug.error('Error adding ICE candidate', e); + app.debug.error("Error adding ICE candidate", e); } } - }) + }); - socket.on('webrtc_canvas_ice_candidate', async (_, data: { candidate, id }) => { - const pc = this.canvasPeers[data.id]; - if (pc) { - try { - await pc.addIceCandidate(new RTCIceCandidate(data.candidate)); - } catch (e) { - app.debug.error('Error adding ICE candidate', e); + socket.on( + "webrtc_canvas_ice_candidate", + async (_, data: { candidate; id }) => { + const pc = this.canvasPeers[data.id]; + if (pc) { + try { + await pc.addIceCandidate(new RTCIceCandidate(data.candidate)); + } catch (e) { + app.debug.error("Error adding ICE candidate", e); + } } } - }) + ); // If a videofeed arrives, then we show the video in the ui - socket.on('videofeed', (_, info) => { - if (app.getTabId() !== info.meta.tabId) return - const feedState = info.data - callUI?.toggleVideoStream(feedState) - }) + socket.on("videofeed", (_, info) => { + if (app.getTabId() !== info.meta.tabId) return; + const feedState = info.data; + callUI?.toggleVideoStream(feedState); + }); - socket.on('request_recording', (id, info) => { - if (app.getTabId() !== info.meta.tabId) return - const agentData = info.data + socket.on("request_recording", (id, info) => { + if (app.getTabId() !== info.meta.tabId) return; + const agentData = info.data; if (!recordingState.isActive) { - this.options.onRecordingRequest?.(JSON.parse(agentData)) - recordingState.requestRecording(id, onAcceptRecording, () => onRejectRecording(agentData)) + this.options.onRecordingRequest?.(JSON.parse(agentData)); + recordingState.requestRecording(id, onAcceptRecording, () => + onRejectRecording(agentData) + ); } else { - this.emit('recording_busy') + this.emit("recording_busy"); } - }) - socket.on('stop_recording', (id, info) => { - if (app.getTabId() !== info.meta.tabId) return + }); + socket.on("stop_recording", (id, info) => { + if (app.getTabId() !== info.meta.tabId) return; if (recordingState.isActive) { - recordingState.stopAgentRecording(id) - } - }) - - socket.on('webrtc_call_offer', async (_, data: { from: string, offer: RTCSessionDescriptionInit }) => { - if (!this.calls.has(data.from)) { - await handleIncomingCallOffer(data.from, data.offer); + recordingState.stopAgentRecording(id); } }); - socket.on('webrtc_call_ice_candidate', async (data: { from: string, candidate: RTCIceCandidateInit }) => { - const pc = this.calls[data.from]; - if (pc) { - try { - await pc.addIceCandidate(new RTCIceCandidate(data.candidate)); - } catch (e) { - app.debug.error('Error adding ICE candidate', e); + socket.on( + "webrtc_call_offer", + async (_, data: { from: string; offer: RTCSessionDescriptionInit }) => { + if (!this.calls.has(data.from)) { + await handleIncomingCallOffer(data.from, data.offer); } } - }); + ); - const callingAgents: Map = new Map() // !! uses socket.io ID + socket.on( + "webrtc_call_ice_candidate", + async (_, data: { from: string; candidate: RTCIceCandidateInit }) => { + const pc = this.calls[data.from]; + if (pc) { + try { + await pc.addIceCandidate(new RTCIceCandidate(data.candidate)); + } catch (e) { + app.debug.error("Error adding ICE candidate", e); + } + } else { + this.iceCandidatesBuffer.set( + data.from, + this.iceCandidatesBuffer + .get(data.from) + ?.concat([data.candidate]) || [data.candidate] + ); + } + } + ); + + const callingAgents: Map = new Map(); // !! uses socket.io ID // TODO: merge peerId & socket.io id (simplest way - send peerId with the name) - const lStreams: Record = {} + const lStreams: Record = {}; function updateCallerNames() { - callUI?.setAssistentName(callingAgents) + callUI?.setAssistentName(callingAgents); } - function endAgentCall({ socketId, callId }: { socketId: string, callId?: string }) { - callingAgents.delete(socketId) + function endAgentCall({ + socketId, + callId, + }: { + socketId: string; + callId?: string; + }) { + callingAgents.delete(socketId); if (callingAgents.size === 0) { - handleCallEnd() + handleCallEnd(); } else { - updateCallerNames() + updateCallerNames(); if (callId) { - handleCallEndWithAgent(callId) + handleCallEndWithAgent(callId); } } } const handleCallEndWithAgent = (id: string) => { - this.calls.get(id)?.close() - this.calls.delete(id) - } + this.calls.get(id)?.close(); + this.calls.delete(id); + }; // call end handling const handleCallEnd = () => { - Object.values(this.calls).forEach(pc => pc.close()) + Object.values(this.calls).forEach((pc) => pc.close()); this.calls.clear(); - Object.values(lStreams).forEach((stream) => { stream.stop() }) - Object.keys(lStreams).forEach((peerId: string) => { delete lStreams[peerId] }) + Object.values(lStreams).forEach((stream) => { + stream.stop(); + }); + Object.keys(lStreams).forEach((peerId: string) => { + delete lStreams[peerId]; + }); // UI - closeCallConfirmWindow() + closeCallConfirmWindow(); if (this.remoteControl?.status === RCStatus.Disabled) { - callUI?.remove() - annot?.remove() - callUI = null - annot = null + callUI?.remove(); + annot?.remove(); + callUI = null; + annot = null; } else { - callUI?.hideControls() + callUI?.hideControls(); } - this.emit('UPDATE_SESSION', { agentIds: [], isCallActive: false }) - this.setCallingState(CallingState.False) - sessionStorage.removeItem(this.options.session_calling_peer_key) + this.emit("UPDATE_SESSION", { agentIds: [], isCallActive: false }); + this.setCallingState(CallingState.False); + sessionStorage.removeItem(this.options.session_calling_peer_key); - callEndCallback?.() - } + callEndCallback?.(); + }; const closeCallConfirmWindow = () => { if (callConfirmWindow) { - callConfirmWindow.remove() - callConfirmWindow = null - callConfirmAnswer = null + callConfirmWindow.remove(); + callConfirmWindow = null; + callConfirmAnswer = null; } - } + }; - const renegotiateConnection = async ({ pc, from }: { pc: RTCPeerConnection, from: string }) => { + const renegotiateConnection = async ({ + pc, + from, + }: { + pc: RTCPeerConnection; + from: string; + }) => { try { const offer = await pc.createOffer(); await pc.setLocalDescription(offer); - this.emit('webrtc_call_offer', { from, offer }); + this.emit("webrtc_call_offer", { from, offer }); } catch (error) { app.debug.error("Error with renegotiation:", error); } }; - const handleIncomingCallOffer = async (from: string, offer: RTCSessionDescriptionInit) => { - app.debug.log('handleIncomingCallOffer', from) - let confirmAnswer: Promise - const callingPeerIds = JSON.parse(sessionStorage.getItem(this.options.session_calling_peer_key) || '[]') + const handleIncomingCallOffer = async ( + from: string, + offer: RTCSessionDescriptionInit + ) => { + app.debug.log("handleIncomingCallOffer", from); + let confirmAnswer: Promise; + const callingPeerIds = JSON.parse( + sessionStorage.getItem(this.options.session_calling_peer_key) || "[]" + ); // if the caller is already in the list, then we immediately accept the call without ui - if (callingPeerIds.includes(from) || this.callingState === CallingState.True) { - confirmAnswer = Promise.resolve(true) + if ( + callingPeerIds.includes(from) || + this.callingState === CallingState.True + ) { + confirmAnswer = Promise.resolve(true); } else { // set the state to wait for confirmation - this.setCallingState(CallingState.Requesting) - // call the call confirmation window - confirmAnswer = requestCallConfirm() + this.setCallingState(CallingState.Requesting); + // call the call confirmation window + confirmAnswer = requestCallConfirm(); // sound notification of a call - this.playNotificationSound() + this.playNotificationSound(); // after 30 seconds we drop the call setTimeout(() => { - if (this.callingState !== CallingState.Requesting) { return } - initiateCallEnd() - }, 30000) + if (this.callingState !== CallingState.Requesting) { + return; + } + initiateCallEnd(); + }, 30000); } try { @@ -595,10 +706,11 @@ export default class Assist { return } - // create a new RTCPeerConnection with ice server config + // create a new RTCPeerConnection with ice server config const pc = new RTCPeerConnection({ iceServers: this.config, }); + this.calls.set(from, pc); if (!callUI) { callUI = new CallWindow(app.debug.error, this.options.callUITemplate) @@ -634,7 +746,7 @@ export default class Assist { if (!callUI) { callUI = new CallWindow(app.debug.error, this.options.callUITemplate); callUI.setVideoToggleCallback((args: { enabled: boolean }) => { - this.emit("videofeed", { streamId: from, enabled: args.enabled }) + this.emit("videofeed", { streamId: from, enabled: args.enabled }); }); } // show buttons in the call window @@ -644,15 +756,16 @@ export default class Assist { annot.mount(); } - - // callUI.setLocalStreams(Object.values(lStreams)) try { // if there are no local streams in lStrems then we set if (!lStreams[from]) { app.debug.log("starting new stream for", from); // request a local stream, and set it to lStreams - lStreams[from] = await RequestLocalStream(pc, renegotiateConnection.bind(null, { pc, from })); + lStreams[from] = await RequestLocalStream( + pc, + renegotiateConnection.bind(null, { pc, from }) + ); } // we pass the received tracks to Call ui callUI.setLocalStreams(Object.values(lStreams)); @@ -687,9 +800,6 @@ export default class Assist { } }; - // Keep connection with the caller - this.calls.set(from, pc); - // set remote description on incoming request await pc.setRemoteDescription(new RTCSessionDescription(offer)); // create a response to the incoming request @@ -699,6 +809,8 @@ export default class Assist { // set the response as local socket.emit("webrtc_call_answer", { from, answer }); + this.applyBufferedIceCandidates(from); + // If the state changes to an error, we terminate the call // pc.onconnectionstatechange = () => { // if (pc.connectionState === 'disconnected' || pc.connectionState === 'failed') { @@ -743,21 +855,26 @@ export default class Assist { // Functions for requesting confirmation, ending a call, notifying, etc. const requestCallConfirm = () => { - if (callConfirmAnswer) { // If confirmation has already been requested + if (callConfirmAnswer) { + // If confirmation has already been requested return callConfirmAnswer; } - callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || { - text: this.options.confirmText, - style: this.options.confirmStyle, - })); - return callConfirmAnswer = callConfirmWindow.mount().then(answer => { + callConfirmWindow = new ConfirmWindow( + callConfirmDefault( + this.options.callConfirm || { + text: this.options.confirmText, + style: this.options.confirmStyle, + } + ) + ); + return (callConfirmAnswer = callConfirmWindow.mount().then((answer) => { closeCallConfirmWindow(); return answer; - }); + })); }; const initiateCallEnd = () => { - this.emit('call_end'); + this.emit("call_end"); handleCallEnd(); }; @@ -782,41 +899,41 @@ export default class Assist { await this.canvasPeers[uniqueId].setLocalDescription(offer); // Send offer via signaling server - socket.emit('webrtc_canvas_offer', { offer, id: uniqueId }); + socket.emit("webrtc_canvas_offer", { offer, id: uniqueId }); } } - } + }; app.nodes.attachNodeCallback((node) => { - const id = app.nodes.getID(node) - if (id && hasTag(node, 'canvas') && !app.sanitizer.isHidden(id)) { - app.debug.log(`Creating stream for canvas ${id}`) + const id = app.nodes.getID(node); + if (id && hasTag(node, "canvas") && !app.sanitizer.isHidden(id)) { + app.debug.log(`Creating stream for canvas ${id}`); const canvasHandler = new Canvas( node as unknown as HTMLCanvasElement, id, 30, (stream: MediaStream) => { - startCanvasStream(stream, id) + startCanvasStream(stream, id); }, - app.debug.error, - ) - this.canvasMap.set(id, canvasHandler) + app.debug.error + ); + this.canvasMap.set(id, canvasHandler); if (this.canvasNodeCheckers.has(id)) { - clearInterval(this.canvasNodeCheckers.get(id)) + clearInterval(this.canvasNodeCheckers.get(id)); } const int = setInterval(() => { - const isPresent = node.ownerDocument.defaultView && node.isConnected + const isPresent = node.ownerDocument.defaultView && node.isConnected; if (!isPresent) { - canvasHandler.stop() - this.canvasMap.delete(id) + canvasHandler.stop(); + this.canvasMap.delete(id); if (this.canvasPeers[id]) { - this.canvasPeers[id]?.close() - this.canvasPeers[id] = null + this.canvasPeers[id]?.close(); + this.canvasPeers[id] = null; } - clearInterval(int) + clearInterval(int); } - }, 5000) - this.canvasNodeCheckers.set(id, int) + }, 5000); + this.canvasNodeCheckers.set(id, int); } }); } @@ -827,7 +944,7 @@ export default class Assist { // ICE candidates peer.onicecandidate = (event) => { if (event.candidate && this.socket) { - this.socket.emit('webrtc_canvas_ice_candidate', { + this.socket.emit("webrtc_canvas_ice_candidate", { candidate: event.candidate, id, }); @@ -836,12 +953,12 @@ export default class Assist { } private playNotificationSound() { - if ('Audio' in window) { - new Audio('https://static.openreplay.com/tracker-assist/notification.mp3') + if ("Audio" in window) { + new Audio("https://static.openreplay.com/tracker-assist/notification.mp3") .play() - .catch(e => { - this.app.debug.warn(e) - }) + .catch((e) => { + this.app.debug.warn(e); + }); } } @@ -850,26 +967,37 @@ export default class Assist { // sometimes means new agent connected, so we keep id for control this.remoteControl?.releaseControl(false, true); if (this.peerReconnectTimeout) { - clearTimeout(this.peerReconnectTimeout) - this.peerReconnectTimeout = null + clearTimeout(this.peerReconnectTimeout); + this.peerReconnectTimeout = null; } this.cleanCanvasConnections(); - Object.values(this.calls).forEach(pc => pc.close()) + Object.values(this.calls).forEach((pc) => pc.close()); this.calls.clear(); if (this.socket) { - this.socket.disconnect() - this.app.debug.log('Socket disconnected') + this.socket.disconnect(); + this.app.debug.log("Socket disconnected"); } - this.canvasMap.clear() - this.canvasPeers = {} - this.canvasNodeCheckers.forEach((int) => clearInterval(int)) - this.canvasNodeCheckers.clear() + this.canvasMap.clear(); + this.canvasPeers = {}; + this.canvasNodeCheckers.forEach((int) => clearInterval(int)); + this.canvasNodeCheckers.clear(); + this.iceCandidatesBuffer.clear(); } private cleanCanvasConnections() { - Object.values(this.canvasPeers).forEach(pc => pc?.close()) - this.canvasPeers = {} - this.socket?.emit('webrtc_canvas_restart') + Object.values(this.canvasPeers).forEach((pc) => pc?.close()); + this.canvasPeers = {}; + this.socket?.emit("webrtc_canvas_restart"); + } + + private applyBufferedIceCandidates(from) { + const buffer = this.iceCandidatesBuffer.get(from); + if (buffer) { + buffer.forEach((candidate) => { + this.calls.get(from)?.addIceCandidate(new RTCIceCandidate(candidate)); + }); + this.iceCandidatesBuffer.delete(from); + } } }