import type { Socket } from 'socket.io-client'; import type Peer from 'peerjs'; import type { MediaConnection } from 'peerjs'; import type MessageDistributor from '../MessageDistributor'; import store from 'App/store'; import type { LocalStream } from './LocalStream'; import { update, getState } from '../../store'; // import { iceServerConfigFromString } from 'App/utils' import AnnotationCanvas from './AnnotationCanvas'; import MStreamReader from '../messages/MStreamReader'; import JSONRawMessageReader from '../messages/JSONRawMessageReader' export enum CallingState { NoCall, Connecting, Requesting, Reconnecting, OnCall, } export enum ConnectionStatus { Connecting, WaitingMessages, Connected, Inactive, Disconnected, Error, Closed, } export enum RemoteControlStatus { Disabled = 0, Requesting, Enabled, } export function getStatusText(status: ConnectionStatus): string { switch(status) { case ConnectionStatus.Closed: return 'Closed...'; case ConnectionStatus.Connecting: return "Connecting..."; case ConnectionStatus.Connected: return ""; case ConnectionStatus.Inactive: return "Client tab is inactive"; case ConnectionStatus.Disconnected: return "Disconnected"; case ConnectionStatus.Error: return "Something went wrong. Try to reload the page."; case ConnectionStatus.WaitingMessages: return "Connected. Waiting for the data... (The tab might be inactive)" } } export interface State { calling: CallingState; peerConnectionStatus: ConnectionStatus; remoteControl: RemoteControlStatus; annotating: boolean; assistStart: number; } export const INITIAL_STATE: State = { calling: CallingState.NoCall, peerConnectionStatus: ConnectionStatus.Connecting, remoteControl: RemoteControlStatus.Disabled, annotating: false, assistStart: 0, } const MAX_RECONNECTION_COUNT = 4; export default class AssistManager { private timeTravelJump = false; private jumped = false; constructor(private session: any, private md: MessageDistributor, private config: any) {} private setStatus(status: ConnectionStatus) { if (getState().peerConnectionStatus === ConnectionStatus.Disconnected && status !== ConnectionStatus.Connected) { return } if (status === ConnectionStatus.Connecting) { this.md.setMessagesLoading(true); } else { this.md.setMessagesLoading(false); } if (status === ConnectionStatus.Connected) { this.md.display(true); } else { this.md.display(false); } update({ peerConnectionStatus: status }); } private get peerID(): string { return `${this.session.projectKey}-${this.session.sessionId}` } private socketCloseTimeout: ReturnType | undefined private onVisChange = () => { this.socketCloseTimeout && clearTimeout(this.socketCloseTimeout) if (document.hidden) { this.socketCloseTimeout = setTimeout(() => { const state = getState() if (document.hidden && (state.calling === CallingState.NoCall && state.remoteControl === RemoteControlStatus.Enabled)) { this.socket?.close() } }, 30000) } else { this.socket?.open() } } private socket: Socket | null = null connect() { const jmr = new JSONRawMessageReader() const reader = new MStreamReader(jmr) let waitingForMessages = true let disconnectTimeout: ReturnType | undefined let inactiveTimeout: ReturnType | undefined function clearDisconnectTimeout() { disconnectTimeout && clearTimeout(disconnectTimeout) disconnectTimeout = undefined } function clearInactiveTimeout() { inactiveTimeout && clearTimeout(inactiveTimeout) inactiveTimeout = undefined } const now = +new Date() update({ assistStart: now }) import('socket.io-client').then(({ default: io }) => { if (this.cleaned) { return } if (this.socket) { this.socket.close() } // TODO: single socket connection // @ts-ignore const urlObject = new URL(window.env.API_EDP || window.location.origin) // does it handle ssl automatically? // @ts-ignore WTF, socket.io ??? const socket: Socket = this.socket = io(urlObject.origin, { path: '/ws-assist/socket', query: { peerId: this.peerID, identity: "agent", //agentInfo: JSON.stringify({}) } }) socket.on("connect", () => { waitingForMessages = true this.setStatus(ConnectionStatus.WaitingMessages) // TODO: happens frequently on bad network }) socket.on("disconnect", () => { this.toggleRemoteControl(false) update({ calling: CallingState.NoCall }) }) socket.on('messages', messages => { !this.timeTravelJump && jmr.append(messages) // as RawMessage[] if (waitingForMessages) { waitingForMessages = false // TODO: more explicit this.setStatus(ConnectionStatus.Connected) // Call State if (getState().calling === CallingState.Reconnecting) { this._callSessionPeer() // reconnecting call (todo improve code separation) } } if (this.timeTravelJump) { return; } for (let msg = reader.readNext();msg !== null;msg = reader.readNext()) { //@ts-ignore if (this.jumped) { // @ts-ignore msg.time = this.md.getLastRecordedMessageTime() + msg.time } // @ts-ignore TODO: fix msg types in generator this.md.distributeMessage(msg, msg._index) } }) socket.on("control_granted", id => { this.toggleRemoteControl(id === socket.id) }) socket.on("control_rejected", id => { id === socket.id && this.toggleRemoteControl(false) }) socket.on('SESSION_RECONNECTED', () => { clearDisconnectTimeout() clearInactiveTimeout() this.setStatus(ConnectionStatus.Connected) }) socket.on('UPDATE_SESSION', ({ active }) => { clearDisconnectTimeout() !inactiveTimeout && this.setStatus(ConnectionStatus.Connected) if (typeof active === "boolean") { clearInactiveTimeout() if (active) { this.setStatus(ConnectionStatus.Connected) } else { inactiveTimeout = setTimeout(() => this.setStatus(ConnectionStatus.Inactive), 5000) } } }) socket.on('SESSION_DISCONNECTED', e => { waitingForMessages = true clearDisconnectTimeout() disconnectTimeout = setTimeout(() => { if (this.cleaned) { return } this.setStatus(ConnectionStatus.Disconnected) }, 30000) if (getState().remoteControl === RemoteControlStatus.Requesting) { this.toggleRemoteControl(false) // else its remaining } // Call State if (getState().calling === CallingState.OnCall) { update({ calling: CallingState.Reconnecting }) } else if (getState().calling === CallingState.Requesting){ update({ calling: CallingState.NoCall }) } }) socket.on('error', e => { console.warn("Socket error: ", e ) this.setStatus(ConnectionStatus.Error); this.toggleRemoteControl(false) }) socket.on('call_end', this.onRemoteCallEnd) document.addEventListener('visibilitychange', this.onVisChange) }) } /* ==== Remote Control ==== */ private onMouseMove = (e: MouseEvent): void => { if (!this.socket) { return } const data = this.md.getInternalCoordinates(e) this.socket.emit("move", [ data.x, data.y ]) } private onWheel = (e: WheelEvent): void => { e.preventDefault() if (!this.socket) { return } //throttling makes movements less smooth, so it is omitted //this.onMouseMove(e) this.socket.emit("scroll", [ e.deltaX, e.deltaY ]) } private onMouseClick = (e: MouseEvent): void => { if (!this.socket) { return; } if (getState().annotating) { return; } // ignore clicks while annotating const data = this.md.getInternalViewportCoordinates(e) // const el = this.md.getElementFromPoint(e); // requires requestiong node_id from domManager const el = this.md.getElementFromInternalPoint(data) if (el instanceof HTMLElement) { el.focus() el.oninput = e => { if (el instanceof HTMLTextAreaElement || el instanceof HTMLInputElement ) { this.socket && this.socket.emit("input", el.value) } else if (el.isContentEditable) { this.socket && this.socket.emit("input", el.innerText) } } // TODO: send "focus" event to assist with the nodeID el.onkeydown = e => { if (e.key == "Tab") { e.preventDefault() } } el.onblur = () => { el.oninput = null el.onblur = null } } this.socket.emit("click", [ data.x, data.y ]); } private toggleRemoteControl(enable: boolean){ if (enable) { this.md.overlay.addEventListener("mousemove", this.onMouseMove) this.md.overlay.addEventListener("click", this.onMouseClick) this.md.overlay.addEventListener("wheel", this.onWheel) update({ remoteControl: RemoteControlStatus.Enabled }) } else { this.md.overlay.removeEventListener("mousemove", this.onMouseMove) this.md.overlay.removeEventListener("click", this.onMouseClick) this.md.overlay.removeEventListener("wheel", this.onWheel) update({ remoteControl: RemoteControlStatus.Disabled }) this.toggleAnnotation(false) } } requestReleaseRemoteControl = () => { if (!this.socket) { return } const remoteControl = getState().remoteControl if (remoteControl === RemoteControlStatus.Requesting) { return } if (remoteControl === RemoteControlStatus.Disabled) { update({ remoteControl: RemoteControlStatus.Requesting }) this.socket.emit("request_control") // setTimeout(() => { // if (getState().remoteControl !== RemoteControlStatus.Requesting) { return } // this.socket?.emit("release_control") // update({ remoteControl: RemoteControlStatus.Disabled }) // }, 8000) } else { this.socket.emit("release_control") this.toggleRemoteControl(false) } } /* ==== PeerJS Call ==== */ private _peer: Peer | null = null private connectionAttempts: number = 0 private callConnection: MediaConnection[] = [] private getPeer(): Promise { if (this._peer && !this._peer.disconnected) { return Promise.resolve(this._peer) } // @ts-ignore const urlObject = new URL(window.env.API_EDP || window.location.origin) return import('peerjs').then(({ default: Peer }) => { if (this.cleaned) {return Promise.reject("Already cleaned")} const peerOpts: any = { host: urlObject.hostname, path: '/assist', port: urlObject.port === "" ? (location.protocol === 'https:' ? 443 : 80 ): parseInt(urlObject.port), } if (this.config) { peerOpts['config'] = { iceServers: this.config, sdpSemantics: 'unified-plan', iceTransportPolicy: 'relay', }; } const peer = this._peer = new Peer(peerOpts) peer.on('call', call => { console.log('getting call from', call.peer) call.answer(this.callArgs.localStream.stream) this.callConnection.push(call) this.callArgs.localStream.onVideoTrack(vTrack => { const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video") if (!sender) { console.warn("No video sender found") return } sender.replaceTrack(vTrack) }) call.on('stream', stream => { this.callArgs && this.callArgs.onStream(stream) }); // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) call.on("close", this.onRemoteCallEnd) call.on("error", (e) => { console.error("PeerJS error (on call):", e) this.initiateCallEnd(); this.callArgs && this.callArgs.onError && this.callArgs.onError(); }); }) peer.on('error', e => { if (e.type === 'disconnected') { return peer.reconnect() } else if (e.type !== 'peer-unavailable') { console.error(`PeerJS error (on peer). Type ${e.type}`, e); } //call-reconnection connected // if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) { // this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT // ? ConnectionStatus.Connecting // : ConnectionStatus.Disconnected); // Reconnect... }) return new Promise(resolve => { peer.on("open", () => resolve(peer)) }) }); } private handleCallEnd() { this.callArgs && this.callArgs.onCallEnd() this.callConnection[0] && this.callConnection[0].close() update({ calling: CallingState.NoCall }) this.callArgs = null this.toggleAnnotation(false) } private initiateCallEnd = async () => { this.socket?.emit("call_end", store.getState().getIn([ 'user', 'account', 'name'])) this.handleCallEnd() } private onRemoteCallEnd = () => { if (getState().calling === CallingState.Requesting) { this.callArgs && this.callArgs.onReject() this.callConnection[0] && this.callConnection[0].close() update({ calling: CallingState.NoCall }) this.callArgs = null this.toggleAnnotation(false) } else { this.handleCallEnd() } } private callArgs: { localStream: LocalStream, onStream: (s: MediaStream)=>void, onCallEnd: () => void, onReject: () => void, onError?: ()=> void, } | null = null public setCallArgs( localStream: LocalStream, onStream: (s: MediaStream)=>void, onCallEnd: () => void, onReject: () => void, onError?: ()=> void, ) { this.callArgs = { localStream, onStream, onCallEnd, onReject, onError, } } public call(thirdPartyPeers?: string[]): { end: Function } { if (thirdPartyPeers && thirdPartyPeers.length > 0) { this.addPeerCall(thirdPartyPeers) } else { this._callSessionPeer() } return { end: this.initiateCallEnd, } } /** Connecting to the other agents that are already * in the call with the user */ public addPeerCall(thirdPartyPeers: string[]) { thirdPartyPeers.forEach(peer => this._peerConnection(peer)) } /** Connecting to the app user */ private _callSessionPeer() { if (![CallingState.NoCall, CallingState.Reconnecting].includes(getState().calling)) { return } update({ calling: CallingState.Connecting }) this._peerConnection(this.peerID); this.socket && this.socket.emit("_agent_name", store.getState().getIn([ 'user', 'account', 'name'])) } private async _peerConnection(remotePeerId: string) { try { const peer = await this.getPeer(); const call = peer.call(remotePeerId, this.callArgs.localStream.stream) this.callConnection.push(call) this.callArgs.localStream.onVideoTrack(vTrack => { const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video") if (!sender) { console.warn("No video sender found") return } sender.replaceTrack(vTrack) }) call.on('stream', stream => { getState().calling !== CallingState.OnCall && update({ calling: CallingState.OnCall }) this.callArgs && this.callArgs.onStream(stream) }); // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) call.on("close", this.onRemoteCallEnd) call.on("error", (e) => { console.error("PeerJS error (on call):", e) this.initiateCallEnd(); this.callArgs && this.callArgs.onError && this.callArgs.onError(); }); } catch (e) { console.error(e) } } toggleAnnotation(enable?: boolean) { // if (getState().calling !== CallingState.OnCall) { return } if (typeof enable !== "boolean") { enable = !!getState().annotating } if (enable && !this.annot) { const annot = this.annot = new AnnotationCanvas() annot.mount(this.md.overlay) annot.canvas.addEventListener("mousedown", e => { if (!this.socket) { return } const data = this.md.getInternalViewportCoordinates(e) annot.start([ data.x, data.y ]) this.socket.emit("startAnnotation", [ data.x, data.y ]) }) annot.canvas.addEventListener("mouseleave", () => { if (!this.socket) { return } annot.stop() this.socket.emit("stopAnnotation") }) annot.canvas.addEventListener("mouseup", () => { if (!this.socket) { return } annot.stop() this.socket.emit("stopAnnotation") }) annot.canvas.addEventListener("mousemove", e => { if (!this.socket || !annot.isPainting()) { return } const data = this.md.getInternalViewportCoordinates(e) annot.move([ data.x, data.y ]) this.socket.emit("moveAnnotation", [ data.x, data.y ]) }) update({ annotating: true }) } else if (!enable && !!this.annot) { this.annot.remove() this.annot = null update({ annotating: false }) } } private annot: AnnotationCanvas | null = null toggleTimeTravelJump() { this.jumped = true; this.timeTravelJump = !this.timeTravelJump; } /* ==== Cleaning ==== */ private cleaned: boolean = false clear() { this.cleaned = true // sometimes cleaned before modules loaded this.initiateCallEnd(); if (this._peer) { console.log("destroying peer...") const peer = this._peer; // otherwise it calls reconnection on data chan close this._peer = null; peer.disconnect(); peer.destroy(); } if (this.socket) { this.socket.close() document.removeEventListener('visibilitychange', this.onVisChange) } if (this.annot) { this.annot.remove() this.annot = null } } }