From e2a10c075170002d92740d8ce3f401b39aa9cda5 Mon Sep 17 00:00:00 2001 From: sylenien Date: Wed, 13 Jul 2022 15:46:32 +0200 Subject: [PATCH] feat(ui/tracker/player): handle multiple callers in one assist session feat(ui/tracker/player): small fixes feat(ui/tracker/player): fix incoming streams feat(tracker): some logs feat(tracker): fix types, fix stream binding feat(tracker): more stuff for multicall... feat(tracker): more stuff for multicall... feat(tracker): more stuff for multicall... feat(tracker): more stuff for multicall... feat(tracker): more stuff for multicall... feat(tracker): more stuff for multicall... feat(tracker): more stuff for multicall... feat(tracker): rm async feat(tracker): rewrite stuff feat(tracker): rewrite stuff feat(tracker): rewrite stuff feat(tracker): rewrite stuff feat(tracker): rewrite stuff feat(tracker): rewrite lstream feat(tracker): rewrite lstream feat(tracker): rewrite stuff feat(tracker): rewrite stuff feat(tracker): fix group call feat(tracker): fix group call feat(tracker): fix group call feat(tracker): fix group call feat(tracker): add text to ui feat(tracker): destroy calls obj on call end feat(tracker): rm unused prop fix(tracker-assist):simplify addRemoteStream logic fixup! fix(tracker-assist):simplify addRemoteStream logic refactor(tracker-assist): make multi-agents call logic more explicite => fixed few bugs --- .../Assist/ChatControls/ChatControls.tsx | 12 +- .../Assist/ChatWindow/ChatWindow.tsx | 27 +- .../AssistActions/AssistActions.tsx | 66 +++-- .../components/Session_/PlayerBlockHeader.js | 4 +- .../shared/SessionItem/SessionItem.tsx | 11 + .../managers/AssistManager.ts | 147 +++++---- .../managers/LocalStream.ts | 1 + frontend/app/player/singletone.js | 5 +- frontend/app/types/session/session.ts | 2 + frontend/package.json | 1 + tracker/tracker-assist/.eslintrc.cjs | 6 +- tracker/tracker-assist/package.json | 2 +- tracker/tracker-assist/src/Assist.ts | 279 ++++++++++-------- tracker/tracker-assist/src/CallWindow.ts | 118 ++++---- .../src/ConfirmWindow/ConfirmWindow.ts | 8 +- tracker/tracker-assist/src/LocalStream.ts | 2 + tracker/tracker-assist/src/Mouse.ts | 8 +- tracker/tracker-assist/src/RemoteControl.ts | 10 +- 18 files changed, 429 insertions(+), 280 deletions(-) diff --git a/frontend/app/components/Assist/ChatControls/ChatControls.tsx b/frontend/app/components/Assist/ChatControls/ChatControls.tsx index 42866ed4b..0b85dab09 100644 --- a/frontend/app/components/Assist/ChatControls/ChatControls.tsx +++ b/frontend/app/components/Assist/ChatControls/ChatControls.tsx @@ -9,9 +9,10 @@ interface Props { stream: LocalStream | null, endCall: () => void, videoEnabled: boolean, - setVideoEnabled: (boolean) => void + isPrestart?: boolean, + setVideoEnabled: (isEnabled: boolean) => void } -function ChatControls({ stream, endCall, videoEnabled, setVideoEnabled } : Props) { +function ChatControls({ stream, endCall, videoEnabled, setVideoEnabled, isPrestart } : Props) { const [audioEnabled, setAudioEnabled] = useState(true) const toggleAudio = () => { @@ -25,6 +26,13 @@ function ChatControls({ stream, endCall, videoEnabled, setVideoEnabled } : Props .then(setVideoEnabled) } + /** muting user if he is auto connected to the call */ + React.useEffect(() => { + if (isPrestart) { + audioEnabled && toggleAudio(); + } + }, []) + return (
diff --git a/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx b/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx index 8eb2a3620..5a6608fcf 100644 --- a/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx +++ b/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx @@ -1,5 +1,4 @@ -//@ts-nocheck -import React, { useState, FC, useEffect } from 'react' +import React, { useState, useEffect } from 'react' import VideoContainer from '../components/VideoContainer' import cn from 'classnames' import Counter from 'App/components/shared/SessionItem/Counter' @@ -8,23 +7,23 @@ import ChatControls from '../ChatControls/ChatControls' import Draggable from 'react-draggable'; import type { LocalStream } from 'Player/MessageDistributor/managers/LocalStream'; - export interface Props { - incomeStream: MediaStream | null, + incomeStream: MediaStream[] | null, localStream: LocalStream | null, - userId: String, + userId: string, + isPrestart?: boolean; endCall: () => void } -const ChatWindow: FC = function ChatWindow({ userId, incomeStream, localStream, endCall }) { +function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }: Props) { const [localVideoEnabled, setLocalVideoEnabled] = useState(false) const [remoteVideoEnabled, setRemoteVideoEnabled] = useState(false) useEffect(() => { - if (!incomeStream) { return } + if (!incomeStream || incomeStream.length === 0) { return } const iid = setInterval(() => { - const settings = incomeStream.getVideoTracks()[0]?.getSettings() - const isDummyVideoTrack = !!settings ? (settings.width === 2 || settings.frameRate === 0) : true + const settings = incomeStream.map(stream => stream.getVideoTracks()[0]?.getSettings()).filter(Boolean) + const isDummyVideoTrack = settings.length > 0 ? (settings.every(s => s.width === 2 || s.frameRate === 0 || s.frameRate === undefined)) : true const shouldBeEnabled = !isDummyVideoTrack if (shouldBeEnabled !== localVideoEnabled) { setRemoteVideoEnabled(shouldBeEnabled) @@ -42,16 +41,20 @@ const ChatWindow: FC = function ChatWindow({ userId, incomeStream, localS style={{ width: '280px' }} >
-
Talking to {userId ? userId : 'Anonymous User'}
+
+ Talking to {userId ? userId : 'Anonymous User'} + {incomeStream && incomeStream.length > 2 ? ' (+ other agents in the call)' : ''} +
- + {!incomeStream &&
Error obtaining incoming streams
} + {incomeStream && incomeStream.map(stream => )}
- +
) diff --git a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx index ba1a7cf0b..757beca54 100644 --- a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx +++ b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx @@ -1,11 +1,12 @@ import React, { useState, useEffect } from 'react'; import { Popup, Icon, Button, IconButton } from 'UI'; +import logger from 'App/logger'; import { connect } from 'react-redux'; import cn from 'classnames'; import { toggleChatWindow } from 'Duck/sessions'; import { connectPlayer } from 'Player/store'; import ChatWindow from '../../ChatWindow'; -import { callPeer, requestReleaseRemoteControl, toggleAnnotation } from 'Player'; +import { callPeer, setCallArgs, requestReleaseRemoteControl, toggleAnnotation } from 'Player'; import { CallingState, ConnectionStatus, RemoteControlStatus } from 'Player/MessageDistributor/managers/AssistManager'; import RequestLocalStream from 'Player/MessageDistributor/managers/LocalStream'; import type { LocalStream } from 'Player/MessageDistributor/managers/LocalStream'; @@ -14,15 +15,12 @@ import { toast } from 'react-toastify'; import { confirm } from 'UI'; import stl from './AassistActions.module.css'; -function onClose(stream) { - stream.getTracks().forEach((t) => t.stop()); -} - function onReject() { toast.info(`Call was rejected.`); } function onError(e) { + console.log(e) toast.error(typeof e === 'string' ? e : e.message); } @@ -35,6 +33,8 @@ interface Props { remoteControlStatus: RemoteControlStatus; hasPermission: boolean; isEnterprise: boolean; + isCallActive: boolean; + agentIds: string[]; } function AssistActions({ @@ -46,14 +46,21 @@ function AssistActions({ remoteControlStatus, hasPermission, isEnterprise, + isCallActive, + agentIds }: Props) { - const [incomeStream, setIncomeStream] = useState(null); + const [isPrestart, setPrestart] = useState(false); + const [incomeStream, setIncomeStream] = useState([]); const [localStream, setLocalStream] = useState(null); const [callObject, setCallObject] = useState<{ end: () => void } | null>(null); + const onCall = calling === CallingState.OnCall || calling === CallingState.Reconnecting; + const cannotCall = peerConnectionStatus !== ConnectionStatus.Connected || (isEnterprise && !hasPermission); + const remoteActive = remoteControlStatus === RemoteControlStatus.Enabled; + useEffect(() => { - return callObject?.end(); - }, []); + return callObject?.end() + }, []) useEffect(() => { if (peerConnectionStatus == ConnectionStatus.Disconnected) { @@ -61,15 +68,36 @@ function AssistActions({ } }, [peerConnectionStatus]); - function call() { - RequestLocalStream() - .then((lStream) => { - setLocalStream(lStream); - setCallObject(callPeer(lStream, setIncomeStream, lStream.stop.bind(lStream), onReject, onError)); - }) - .catch(onError); + const addIncomeStream = (stream: MediaStream) => { + console.log('new stream in component') + setIncomeStream(oldState => [...oldState, stream]); } + function call(agentIds?: string[]) { + RequestLocalStream().then(lStream => { + setLocalStream(lStream); + setCallArgs( + lStream, + addIncomeStream, + lStream.stop.bind(lStream), + onReject, + onError + ) + setCallObject(callPeer()); + if (agentIds) { + callPeer(agentIds) + } + }).catch(onError) + } + + React.useEffect(() => { + if (!onCall && isCallActive && agentIds) { + logger.log('joinig the party', agentIds) + setPrestart(true); + call(agentIds) + } + }, [agentIds, isCallActive]) + const confirmCall = async () => { if ( await confirm({ @@ -82,10 +110,6 @@ function AssistActions({ } }; - const onCall = calling === CallingState.OnCall || calling === CallingState.Reconnecting; - const cannotCall = peerConnectionStatus !== ConnectionStatus.Connected || (isEnterprise && !hasPermission); - const remoteActive = remoteControlStatus === RemoteControlStatus.Enabled; - return (
{(onCall || remoteActive) && ( @@ -123,7 +147,7 @@ function AssistActions({
- +
{onCall && callObject && ( - + )}
diff --git a/frontend/app/components/Session_/PlayerBlockHeader.js b/frontend/app/components/Session_/PlayerBlockHeader.js index f0576e419..f7afb2bdb 100644 --- a/frontend/app/components/Session_/PlayerBlockHeader.js +++ b/frontend/app/components/Session_/PlayerBlockHeader.js @@ -105,7 +105,7 @@ export default class PlayerBlockHeader extends React.PureComponent { const { hideBack } = this.state; - const { sessionId, userId, userNumericHash, live, metadata } = session; + const { sessionId, userId, userNumericHash, live, metadata, isCallActive, agentIds } = session; let _metaList = Object.keys(metadata) .filter((i) => metaList.includes(i)) .map((key) => { @@ -142,7 +142,7 @@ export default class PlayerBlockHeader extends React.PureComponent {
)} - {isAssist && } + {isAssist && } {!isAssist && ( diff --git a/frontend/app/components/shared/SessionItem/SessionItem.tsx b/frontend/app/components/shared/SessionItem/SessionItem.tsx index fa886ab8b..83d2e6bb3 100644 --- a/frontend/app/components/shared/SessionItem/SessionItem.tsx +++ b/frontend/app/components/shared/SessionItem/SessionItem.tsx @@ -41,6 +41,8 @@ interface Props { userSessionsCount: number; issueTypes: []; active: boolean; + isCallActive?: boolean; + agentIds?: string[]; }; onUserClick?: (userId: string, userAnonymousId: string) => void; hasUserFilter?: boolean; @@ -168,6 +170,15 @@ function SessionItem(props: RouteComponentProps & Props) {
+ {live && session.isCallActive && session.agentIds.length > 0 ? ( +
+ +
+ ) : null} {isSessions && (
{isLastPlayed && ( diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager.ts b/frontend/app/player/MessageDistributor/managers/AssistManager.ts index 2ba90311e..11d5e948c 100644 --- a/frontend/app/player/MessageDistributor/managers/AssistManager.ts +++ b/frontend/app/player/MessageDistributor/managers/AssistManager.ts @@ -136,7 +136,6 @@ export default class AssistManager { //agentInfo: JSON.stringify({}) } }) - //socket.onAny((...args) => console.log(...args)) socket.on("connect", () => { waitingForMessages = true this.setStatus(ConnectionStatus.WaitingMessages) // TODO: happens frequently on bad network @@ -146,7 +145,6 @@ export default class AssistManager { update({ calling: CallingState.NoCall }) }) socket.on('messages', messages => { - //console.log(messages.filter(m => m._id === 41 || m._id === 44)) jmr.append(messages) // as RawMessage[] if (waitingForMessages) { @@ -176,11 +174,11 @@ export default class AssistManager { this.setStatus(ConnectionStatus.Connected) }) - socket.on('UPDATE_SESSION', ({ active }) => { + socket.on('UPDATE_SESSION', (data) => { showDisconnectTimeout && clearTimeout(showDisconnectTimeout) !inactiveTimeout && this.setStatus(ConnectionStatus.Connected) - if (typeof active === "boolean") { - if (active) { + if (typeof data.active === "boolean") { + if (data.active) { inactiveTimeout && clearTimeout(inactiveTimeout) this.setStatus(ConnectionStatus.Connected) } else { @@ -305,7 +303,7 @@ export default class AssistManager { private _peer: Peer | null = null private connectionAttempts: number = 0 - private callConnection: MediaConnection | null = null + private callConnection: MediaConnection[] = [] private getPeer(): Promise { if (this._peer && !this._peer.disconnected) { return Promise.resolve(this._peer) } @@ -326,6 +324,32 @@ export default class AssistManager { }; } const peer = this._peer = new Peer(peerOpts) + peer.on('call', call => { + console.log('getting call from', call.peer) + call.answer(this.callArgs.localStream.stream) + this.callConnection.push(call) + + this.callArgs.localStream.onVideoTrack(vTrack => { + const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video") + if (!sender) { + console.warn("No video sender found") + return + } + sender.replaceTrack(vTrack) + }) + + call.on('stream', stream => { + this.callArgs && this.callArgs.onStream(stream) + }); + call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) + + call.on("close", this.onRemoteCallEnd) + call.on("error", (e) => { + console.error("PeerJS error (on call):", e) + this.initiateCallEnd(); + this.callArgs && this.callArgs.onError && this.callArgs.onError(); + }); + }) peer.on('error', e => { if (e.type === 'disconnected') { return peer.reconnect() @@ -351,21 +375,21 @@ export default class AssistManager { private handleCallEnd() { this.callArgs && this.callArgs.onCallEnd() - this.callConnection && this.callConnection.close() + this.callConnection[0] && this.callConnection[0].close() update({ calling: CallingState.NoCall }) this.callArgs = null this.toggleAnnotation(false) } - private initiateCallEnd = () => { - this.socket?.emit("call_end") + private initiateCallEnd = async () => { + this.socket?.emit("call_end", store.getState().getIn([ 'user', 'account', 'name'])) this.handleCallEnd() } private onRemoteCallEnd = () => { if (getState().calling === CallingState.Requesting) { this.callArgs && this.callArgs.onReject() - this.callConnection && this.callConnection.close() + this.callConnection[0] && this.callConnection[0].close() update({ calling: CallingState.NoCall }) this.callArgs = null this.toggleAnnotation(false) @@ -379,15 +403,16 @@ export default class AssistManager { onStream: (s: MediaStream)=>void, onCallEnd: () => void, onReject: () => void, - onError?: ()=> void + onError?: ()=> void, } | null = null - call( + public setCallArgs( localStream: LocalStream, onStream: (s: MediaStream)=>void, onCallEnd: () => void, onReject: () => void, - onError?: ()=> void): { end: Function } { + onError?: ()=> void, + ) { this.callArgs = { localStream, onStream, @@ -395,12 +420,66 @@ export default class AssistManager { onReject, onError, } - this._call() + } + + public call(thirdPartyPeers?: string[]): { end: Function } { + if (thirdPartyPeers && thirdPartyPeers.length > 0) { + this.addPeerCall(thirdPartyPeers) + } else { + this._callSessionPeer() + } return { end: this.initiateCallEnd, } } + /** Connecting to the other agents that are already + * in the call with the user + */ + public addPeerCall(thirdPartyPeers: string[]) { + thirdPartyPeers.forEach(peer => this._peerConnection(peer)) + } + + /** Connecting to the app user */ + private _callSessionPeer() { + if (![CallingState.NoCall, CallingState.Reconnecting].includes(getState().calling)) { return } + update({ calling: CallingState.Connecting }) + this._peerConnection(this.peerID); + this.socket && this.socket.emit("_agent_name", store.getState().getIn([ 'user', 'account', 'name'])) + } + + private async _peerConnection(remotePeerId: string) { + try { + const peer = await this.getPeer(); + const call = peer.call(remotePeerId, this.callArgs.localStream.stream) + this.callConnection.push(call) + + this.callArgs.localStream.onVideoTrack(vTrack => { + const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video") + if (!sender) { + console.warn("No video sender found") + return + } + sender.replaceTrack(vTrack) + }) + + call.on('stream', stream => { + getState().calling !== CallingState.OnCall && update({ calling: CallingState.OnCall }) + this.callArgs && this.callArgs.onStream(stream) + }); + call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) + + call.on("close", this.onRemoteCallEnd) + call.on("error", (e) => { + console.error("PeerJS error (on call):", e) + this.initiateCallEnd(); + this.callArgs && this.callArgs.onError && this.callArgs.onError(); + }); + } catch (e) { + console.error(e) + } + } + toggleAnnotation(enable?: boolean) { // if (getState().calling !== CallingState.OnCall) { return } if (typeof enable !== "boolean") { @@ -442,44 +521,6 @@ export default class AssistManager { private annot: AnnotationCanvas | null = null - private _call() { - if (![CallingState.NoCall, CallingState.Reconnecting].includes(getState().calling)) { return } - update({ calling: CallingState.Connecting }) - this.getPeer().then(peer => { - if (!this.callArgs) { return console.log("No call Args. Must not happen.") } - update({ calling: CallingState.Requesting }) - - // TODO: in a proper way - this.socket && this.socket.emit("_agent_name", store.getState().getIn([ 'user', 'account', 'name'])) - - const call = this.callConnection = peer.call(this.peerID, this.callArgs.localStream.stream) - this.callArgs.localStream.onVideoTrack(vTrack => { - const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video") - if (!sender) { - console.warn("No video sender found") - return - } - //logger.log("sender found:", sender) - sender.replaceTrack(vTrack) - }) - - call.on('stream', stream => { - update({ calling: CallingState.OnCall }) - this.callArgs && this.callArgs.onStream(stream) - }); - //call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) - - call.on("close", this.onRemoteCallEnd) - call.on("error", (e) => { - console.error("PeerJS error (on call):", e) - this.initiateCallEnd(); - this.callArgs && this.callArgs.onError && this.callArgs.onError(); - }); - - }) - } - - /* ==== Cleaning ==== */ private cleaned: boolean = false clear() { @@ -502,5 +543,3 @@ export default class AssistManager { } } } - - diff --git a/frontend/app/player/MessageDistributor/managers/LocalStream.ts b/frontend/app/player/MessageDistributor/managers/LocalStream.ts index 63f01ad58..360033c7f 100644 --- a/frontend/app/player/MessageDistributor/managers/LocalStream.ts +++ b/frontend/app/player/MessageDistributor/managers/LocalStream.ts @@ -54,6 +54,7 @@ class _LocalStream { }) .catch(e => { // TODO: log + console.error(e) return false }) } diff --git a/frontend/app/player/singletone.js b/frontend/app/player/singletone.js index a0fe6ff26..808605793 100644 --- a/frontend/app/player/singletone.js +++ b/frontend/app/player/singletone.js @@ -2,7 +2,7 @@ import Player from './Player'; import { update, clean as cleanStore, getState } from './store'; import { clean as cleanLists } from './lists'; - +/** @type {Player} */ let instance = null; const initCheck = method => (...args) => { @@ -69,7 +69,10 @@ export const attach = initCheck((...args) => instance.attach(...args)); export const markElement = initCheck((...args) => instance.marker && instance.marker.mark(...args)); export const scale = initCheck(() => instance.scale()); export const toggleInspectorMode = initCheck((...args) => instance.toggleInspectorMode(...args)); +/** @type {Player.assistManager.call} */ export const callPeer = initCheck((...args) => instance.assistManager.call(...args)) +/** @type {Player.assistManager.setCallArgs} */ +export const setCallArgs = initCheck((...args) => instance.assistManager.setCallArgs(...args)) export const requestReleaseRemoteControl = initCheck((...args) => instance.assistManager.requestReleaseRemoteControl(...args)) export const markTargets = initCheck((...args) => instance.markTargets(...args)) export const activeTarget = initCheck((...args) => instance.activeTarget(...args)) diff --git a/frontend/app/types/session/session.ts b/frontend/app/types/session/session.ts index a4ed48fe6..5eadadf4b 100644 --- a/frontend/app/types/session/session.ts +++ b/frontend/app/types/session/session.ts @@ -79,6 +79,8 @@ export default Record({ isIOS: false, revId: '', userSessionsCount: 0, + agentIds: [], + isCallActive: false }, { fromJS:({ startTs=0, diff --git a/frontend/package.json b/frontend/package.json index bce5abbad..957c65c3a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -83,6 +83,7 @@ "@openreplay/sourcemap-uploader": "^3.0.0", "@types/react": "^18.0.9", "@types/react-dom": "^18.0.4", + "@types/react-redux": "^7.1.24", "@types/react-router-dom": "^5.3.3", "@typescript-eslint/eslint-plugin": "^5.24.0", "@typescript-eslint/parser": "^5.24.0", diff --git a/tracker/tracker-assist/.eslintrc.cjs b/tracker/tracker-assist/.eslintrc.cjs index 01d5c5bc0..4480aa99f 100644 --- a/tracker/tracker-assist/.eslintrc.cjs +++ b/tracker/tracker-assist/.eslintrc.cjs @@ -24,12 +24,14 @@ module.exports = { '@typescript-eslint/camelcase': 'off', '@typescript-eslint/no-explicit-any': 'off', '@typescript-eslint/unbound-method': 'off', - '@typescript-eslint/explicit-function-return-type': 'warn', '@typescript-eslint/prefer-readonly': 'warn', '@typescript-eslint/ban-ts-comment': 'off', '@typescript-eslint/no-unsafe-assignment': 'off', '@typescript-eslint/no-unsafe-member-access': 'off', + '@typescript-eslint/no-unused-expressions': 'off', '@typescript-eslint/no-unsafe-call': 'off', + '@typescript-eslint/no-unsafe-argument': 'off', + '@typescript-eslint/explicit-function-return-type': 'off', '@typescript-eslint/restrict-plus-operands': 'warn', '@typescript-eslint/no-unsafe-return': 'warn', 'no-useless-escape': 'warn', @@ -38,9 +40,7 @@ module.exports = { '@typescript-eslint/no-useless-constructor': 'warn', '@typescript-eslint/no-this-alias': 'off', '@typescript-eslint/no-floating-promises': 'warn', - '@typescript-eslint/no-unsafe-argument': 'warn', 'no-unused-expressions': 'off', - '@typescript-eslint/no-unused-expressions': 'warn', '@typescript-eslint/no-useless-constructor': 'warn', 'semi': ["error", "never"], 'quotes': ["error", "single"], diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 4efae850e..7849eb6ff 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -34,6 +34,7 @@ "@openreplay/tracker": "^3.5.3" }, "devDependencies": { + "@openreplay/tracker": "file:../tracker", "@typescript-eslint/eslint-plugin": "^5.30.0", "@typescript-eslint/parser": "^5.30.0", "eslint": "^7.8.0", @@ -42,7 +43,6 @@ "husky": "^8.0.1", "lint-staged": "^13.0.3", "prettier": "^2.7.1", - "@openreplay/tracker": "file:../tracker", "replace-in-files-cli": "^1.0.0", "typescript": "^4.6.0-dev.20211126" }, diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 80023beaa..fa4b071d1 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -1,11 +1,11 @@ /* eslint-disable @typescript-eslint/no-empty-function */ import type { Socket, } from 'socket.io-client' import { connect, } from 'socket.io-client' -import Peer from 'peerjs' +import Peer, { MediaConnection, } from 'peerjs' import type { Properties, } from 'csstype' import { App, } from '@openreplay/tracker' -import RequestLocalStream from './LocalStream.js' +import RequestLocalStream, { LocalStream, } from './LocalStream.js' import RemoteControl from './RemoteControl.js' import CallWindow from './CallWindow.js' import AnnotationCanvas from './AnnotationCanvas.js' @@ -13,7 +13,7 @@ import ConfirmWindow from './ConfirmWindow/ConfirmWindow.js' import { callConfirmDefault, } from './ConfirmWindow/defaults.js' import type { Options as ConfirmOptions, } from './ConfirmWindow/defaults.js' -// TODO: fully specified strict check (everywhere) +// TODO: fully specified strict check with no-any (everywhere) type StartEndCallback = () => ((()=>Record) | void) @@ -45,7 +45,7 @@ type OptionalCallback = (()=>Record) | void type Agent = { onDisconnect?: OptionalCallback, onControlReleased?: OptionalCallback, - name?: string + //name?: string // } @@ -78,7 +78,7 @@ export default class Assist { ) if (document.hidden !== undefined) { - const sendActivityState = () => this.emit('UPDATE_SESSION', { active: !document.hidden, }) + const sendActivityState = (): void => this.emit('UPDATE_SESSION', { active: !document.hidden, }) app.attachEventListener( document, 'visibilitychange', @@ -111,7 +111,7 @@ export default class Assist { app.session.attachUpdateCallback(sessInfo => this.emit('UPDATE_SESSION', sessInfo)) } - private emit(ev: string, ...args) { + private emit(ev: string, ...args): void { this.socket && this.socket.emit(ev, ...args) } @@ -119,14 +119,17 @@ export default class Assist { return Object.keys(this.agents).length > 0 } - private notifyCallEnd() { - this.emit('call_end') + private readonly setCallingState = (newState: CallingState): void => { + this.callingState = newState } - private onRemoteCallEnd = () => {} private onStart() { const app = this.app - const peerID = `${app.getProjectKey()}-${app.getSessionID()}` + const sessionId = app.getSessionID() + if (!sessionId) { + return app.debug.error('No session ID') + } + const peerID = `${app.getProjectKey()}-${sessionId}` // SocketIO const socket = this.socket = connect(app.getHost(), { @@ -187,51 +190,65 @@ export default class Assist { socket.on('NEW_AGENT', (id: string, info) => { this.agents[id] = { - onDisconnect: this.options.onAgentConnect && this.options.onAgentConnect(), + onDisconnect: this.options.onAgentConnect?.(), ...info, // TODO } this.assistDemandedRestart = true this.app.stop() - this.app.start().then(() => { this.assistDemandedRestart = false }) + this.app.start().then(() => { this.assistDemandedRestart = false }).catch(e => app.debug.error(e)) }) socket.on('AGENTS_CONNECTED', (ids: string[]) => { ids.forEach(id =>{ this.agents[id] = { - onDisconnect: this.options.onAgentConnect && this.options.onAgentConnect(), + onDisconnect: this.options.onAgentConnect?.(), } }) this.assistDemandedRestart = true this.app.stop() - this.app.start().then(() => { this.assistDemandedRestart = false }) + this.app.start().then(() => { this.assistDemandedRestart = false }).catch(e => app.debug.error(e)) remoteControl.reconnect(ids) }) - let confirmCall:ConfirmWindow | null = null - socket.on('AGENT_DISCONNECTED', (id) => { remoteControl.releaseControl(id) - // close the call also - if (callingAgent === id) { - confirmCall?.remove() - this.onRemoteCallEnd() - } - - // @ts-ignore (wtf, typescript?!) - this.agents[id] && this.agents[id].onDisconnect != null && this.agents[id].onDisconnect() + this.agents[id]?.onDisconnect?.() delete this.agents[id] + + endAgentCall(id) }) socket.on('NO_AGENT', () => { + Object.values(this.agents).forEach(a => a.onDisconnect?.()) this.agents = {} }) - socket.on('call_end', () => this.onRemoteCallEnd()) // TODO: check if agent calling id + socket.on('call_end', (id) => { + if (!callingAgents.has(id)) { + app.debug.warn('Received call_end from unknown agent', id) + return + } + endAgentCall(id) + }) - // TODO: fix the code - let agentName = '' - let callingAgent = '' - socket.on('_agent_name',(id, name) => { agentName = name; callingAgent = id }) + socket.on('_agent_name', (id, name) => { + callingAgents.set(id, name) + updateCallerNames() + }) + const callingAgents: Map = new Map() // !! uses socket.io ID + // TODO: merge peerId & socket.io id (simplest way - send peerId with the name) + const calls: Record = {} // !! uses peerJS ID + const lStreams: Record = {} + // const callingPeers: Map = new Map() // Maybe + function endAgentCall(id: string) { + callingAgents.delete(id) + if (callingAgents.size === 0) { + handleCallEnd() + } else { + updateCallerNames() + //TODO: close() specific call and corresponding lStreams (after connecting peerId & socket.io id) + } + } // PeerJS call (todo: use native WebRTC) const peerOptions = { @@ -244,119 +261,147 @@ export default class Assist { peerOptions['config'] = this.options.config } const peer = this.peer = new Peer(peerID, peerOptions) - // app.debug.log('Peer created: ', peer) - // @ts-ignore + + // @ts-ignore (peerjs typing) peer.on('error', e => app.debug.warn('Peer error: ', e.type, e)) peer.on('disconnected', () => peer.reconnect()) - peer.on('call', (call) => { - app.debug.log('Call: ', call) - if (this.callingState !== CallingState.False) { - call.close() - //this.notifyCallEnd() // TODO: strictly connect calling peer with agent socket.id - app.debug.warn('Call closed instantly bacause line is busy. CallingState: ', this.callingState) - return - } - const setCallingState = (newState: CallingState) => { - if (newState === CallingState.True) { - sessionStorage.setItem(this.options.session_calling_peer_key, call.peer) - } else if (newState === CallingState.False) { - sessionStorage.removeItem(this.options.session_calling_peer_key) - } - this.callingState = newState + // Common for all incoming call requests + let callUI: CallWindow | null = null + function updateCallerNames() { + callUI?.setAssistentName(callingAgents) + } + // TODO: incapsulate + let callConfirmWindow: ConfirmWindow | null = null + let callConfirmAnswer: Promise | null = null + const closeCallConfirmWindow = () => { + if (callConfirmWindow) { + callConfirmWindow.remove() + callConfirmWindow = null + callConfirmAnswer = null } - + } + const requestCallConfirm = () => { + if (callConfirmAnswer) { // Already asking + return callConfirmAnswer + } + callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || { + text: this.options.confirmText, + style: this.options.confirmStyle, + })) // TODO: reuse ? + return callConfirmAnswer = callConfirmWindow.mount().then(answer => { + closeCallConfirmWindow() + return answer + }) + } + let callEndCallback: ReturnType | null = null + const handleCallEnd = () => { // Completle stop and clear all calls + // Streams + Object.values(calls).forEach(call => call.close()) + Object.keys(calls).forEach(peerId => delete calls[peerId]) + Object.values(lStreams).forEach((stream) => { stream.stop() }) + Object.keys(lStreams).forEach((peerId: string) => { delete lStreams[peerId] }) + + // UI + closeCallConfirmWindow() + callUI?.remove() + annot?.remove() + callUI = null + annot = null + + this.emit('UPDATE_SESSION', { agentIds: [], isCallActive: false, }) + this.setCallingState(CallingState.False) + sessionStorage.removeItem(this.options.session_calling_peer_key) + callEndCallback?.() + } + const initiateCallEnd = () => { + this.emit('call_end') + handleCallEnd() + } + + peer.on('call', (call) => { + app.debug.log('Incoming call: ', call) let confirmAnswer: Promise - const callingPeer = sessionStorage.getItem(this.options.session_calling_peer_key) - if (callingPeer === call.peer) { + const callingPeerIds = JSON.parse(sessionStorage.getItem(this.options.session_calling_peer_key) || '[]') + if (callingPeerIds.includes(call.peer) || this.callingState === CallingState.True) { confirmAnswer = Promise.resolve(true) } else { - setCallingState(CallingState.Requesting) - confirmCall = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || { - text: this.options.confirmText, - style: this.options.confirmStyle, - })) - confirmAnswer = confirmCall.mount() - this.playNotificationSound() - this.onRemoteCallEnd = () => { // if call cancelled by a caller before confirmation - app.debug.log('Received call_end during confirm window opened') - confirmCall?.remove() - setCallingState(CallingState.False) - call.close() - } + this.setCallingState(CallingState.Requesting) + confirmAnswer = requestCallConfirm() + this.playNotificationSound() // For every new agent during confirmation here + + // TODO: only one (latest) timeout setTimeout(() => { if (this.callingState !== CallingState.Requesting) { return } - call.close() - confirmCall?.remove() - this.notifyCallEnd() - setCallingState(CallingState.False) + initiateCallEnd() }, 30000) } - confirmAnswer.then(agreed => { + confirmAnswer.then(async agreed => { if (!agreed) { - call.close() - this.notifyCallEnd() - setCallingState(CallingState.False) + initiateCallEnd() + return + } + // Request local stream for the new connection + try { + // lStreams are reusable so fare we don't delete them in the `endAgentCall` + if (!lStreams[call.peer]) { + app.debug.log('starting new stream for', call.peer) + lStreams[call.peer] = await RequestLocalStream() + } + calls[call.peer] = call + } catch (e) { + app.debug.error('Audio mediadevice request error:', e) + initiateCallEnd() return } - const callUI = new CallWindow() - annot = new AnnotationCanvas() - annot.mount() - callUI.setAssistentName(agentName) - - const onCallEnd = this.options.onCallStart() - const handleCallEnd = () => { - app.debug.log('Handle Call End') - call.close() - callUI.remove() - annot && annot.remove() - annot = null - setCallingState(CallingState.False) - onCallEnd && onCallEnd() + // UI + if (!callUI) { + callUI = new CallWindow(app.debug.error) + // TODO: as constructor options + callUI.setCallEndAction(initiateCallEnd) + callUI.setLocalStreams(Object.values(lStreams)) } - const initiateCallEnd = () => { - this.notifyCallEnd() - handleCallEnd() + if (!annot) { + annot = new AnnotationCanvas() + annot.mount() } - this.onRemoteCallEnd = handleCallEnd call.on('error', e => { app.debug.warn('Call error:', e) initiateCallEnd() }) - - RequestLocalStream().then(lStream => { - call.on('stream', function(rStream) { - callUI.setRemoteStream(rStream) - const onInteraction = () => { // only if hidden? - callUI.playRemote() - document.removeEventListener('click', onInteraction) - } - document.addEventListener('click', onInteraction) - }) - - lStream.onVideoTrack(vTrack => { - const sender = call.peerConnection.getSenders().find(s => s.track?.kind === 'video') - if (!sender) { - app.debug.warn('No video sender found') - return - } - app.debug.log('sender found:', sender) - sender.replaceTrack(vTrack) - }) - - callUI.setCallEndAction(initiateCallEnd) - callUI.setLocalStream(lStream) - call.answer(lStream.stream) - setCallingState(CallingState.True) + call.on('stream', (rStream) => { + callUI?.addRemoteStream(rStream) + const onInteraction = () => { // do only if document.hidden ? + callUI?.playRemote() + document.removeEventListener('click', onInteraction) + } + document.addEventListener('click', onInteraction) }) - .catch(e => { - app.debug.warn('Audio mediadevice request error:', e) - initiateCallEnd() + + // remote video on/off/camera change + lStreams[call.peer].onVideoTrack(vTrack => { + const sender = call.peerConnection.getSenders().find(s => s.track?.kind === 'video') + if (!sender) { + app.debug.warn('No video sender found') + return + } + app.debug.log('sender found:', sender) + void sender.replaceTrack(vTrack) }) - }).catch() // in case of Confirm.remove() without any confirmation/decline + + call.answer(lStreams[call.peer].stream) + this.setCallingState(CallingState.True) + if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() } + + const callingPeerIds = Object.keys(calls) + sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIds)) + this.emit('UPDATE_SESSION', { agentIds: callingPeerIds, isCallActive: true, }) + }).catch(reason => { // in case of Confirm.remove() without user answer (not a error) + app.debug.log(reason) + }) }) } diff --git a/tracker/tracker-assist/src/CallWindow.ts b/tracker/tracker-assist/src/CallWindow.ts index 8804ffa3e..c8f07e0ff 100644 --- a/tracker/tracker-assist/src/CallWindow.ts +++ b/tracker/tracker-assist/src/CallWindow.ts @@ -4,7 +4,7 @@ import attachDND from './dnd.js' const SS_START_TS_KEY = '__openreplay_assist_call_start_ts' export default class CallWindow { - private iframe: HTMLIFrameElement + private readonly iframe: HTMLIFrameElement private vRemote: HTMLVideoElement | null = null private vLocal: HTMLVideoElement | null = null private audioBtn: HTMLElement | null = null @@ -16,9 +16,9 @@ export default class CallWindow { private tsInterval: ReturnType - private load: Promise + private readonly load: Promise - constructor() { + constructor(private readonly logError: (...args: any[]) => void) { const iframe = this.iframe = document.createElement('iframe') Object.assign(iframe.style, { position: 'fixed', @@ -107,8 +107,8 @@ export default class CallWindow { private adjustIframeSize() { const doc = this.iframe.contentDocument if (!doc) { return } - this.iframe.style.height = doc.body.scrollHeight + 'px' - this.iframe.style.width = doc.body.scrollWidth + 'px' + this.iframe.style.height = `${doc.body.scrollHeight}px` + this.iframe.style.width = `${doc.body.scrollWidth}px` } setCallEndAction(endCall: () => void) { @@ -116,40 +116,46 @@ export default class CallWindow { if (this.endCallBtn) { this.endCallBtn.onclick = endCall } - }) + }).catch(e => this.logError(e)) } - private aRemote: HTMLAudioElement | null = null; private checkRemoteVideoInterval: ReturnType - setRemoteStream(rStream: MediaStream) { + private audioContainer: HTMLDivElement | null = null + addRemoteStream(rStream: MediaStream) { this.load.then(() => { + // Video if (this.vRemote && !this.vRemote.srcObject) { this.vRemote.srcObject = rStream if (this.vPlaceholder) { this.vPlaceholder.innerText = 'Video has been paused. Click anywhere to resume.' } - - // Hack for audio. Doesen't work inside the iframe because of some magical reasons (check if it is connected to autoplay?) - this.aRemote = document.createElement('audio') - this.aRemote.autoplay = true - this.aRemote.style.display = 'none' - this.aRemote.srcObject = rStream - document.body.appendChild(this.aRemote) + // Hack to determine if the remote video is enabled + // TODO: pass this info through socket + if (this.checkRemoteVideoInterval) { clearInterval(this.checkRemoteVideoInterval) } // just in case + let enabled = false + this.checkRemoteVideoInterval = setInterval(() => { + const settings = rStream.getVideoTracks()[0]?.getSettings() + const isDummyVideoTrack = !!settings && (settings.width === 2 || settings.frameRate === 0) + const shouldBeEnabled = !isDummyVideoTrack + if (enabled !== shouldBeEnabled) { + this.toggleRemoteVideoUI(enabled=shouldBeEnabled) + } + }, 1000) } - // Hack to determine if the remote video is enabled - if (this.checkRemoteVideoInterval) { clearInterval(this.checkRemoteVideoInterval) } // just in case - let enabled = false - this.checkRemoteVideoInterval = setInterval(() => { - const settings = rStream.getVideoTracks()[0]?.getSettings() - //console.log(settings) - const isDummyVideoTrack = !!settings && (settings.width === 2 || settings.frameRate === 0) - const shouldBeEnabled = !isDummyVideoTrack - if (enabled !== shouldBeEnabled) { - this.toggleRemoteVideoUI(enabled=shouldBeEnabled) - } - }, 1000) - }) + // Audio + if (!this.audioContainer) { + this.audioContainer = document.createElement('div') + document.body.appendChild(this.audioContainer) + } + // Hack for audio. Doesen't work inside the iframe + // because of some magical reasons (check if it is connected to autoplay?) + const audioEl = document.createElement('audio') + audioEl.autoplay = true + audioEl.style.display = 'none' + audioEl.srcObject = rStream + this.audioContainer.appendChild(audioEl) + }).catch(e => this.logError(e)) } toggleRemoteVideoUI(enable: boolean) { @@ -162,26 +168,27 @@ export default class CallWindow { } this.adjustIframeSize() } - }) + }).catch(e => this.logError(e)) } - private localStream: LocalStream | null = null; - - // TODO: on construction? - setLocalStream(lStream: LocalStream) { - this.localStream = lStream + private localStreams: LocalStream[] = [] + // !TODO: separate streams manipulation from ui + setLocalStreams(streams: LocalStream[]) { + this.localStreams = streams } playRemote() { this.vRemote && this.vRemote.play() } - setAssistentName(name: string) { + setAssistentName(callingAgents: Map) { this.load.then(() => { if (this.agentNameElem) { - this.agentNameElem.innerText = name + const nameString = Array.from(callingAgents.values()).join(', ') + const safeNames = nameString.length > 20 ? nameString.substring(0, 20) + '...' : nameString + this.agentNameElem.innerText = safeNames } - }) + }).catch(e => this.logError(e)) } @@ -195,7 +202,10 @@ export default class CallWindow { } private toggleAudio() { - const enabled = this.localStream?.toggleAudio() || false + let enabled = false + this.localStreams.forEach(stream => { + enabled = stream.toggleAudio() || false + }) this.toggleAudioUI(enabled) } @@ -211,30 +221,32 @@ export default class CallWindow { this.adjustIframeSize() } - private videoRequested = false private toggleVideo() { - this.localStream?.toggleVideo() - .then(enabled => { - this.toggleVideoUI(enabled) - this.load.then(() => { - if (this.vLocal && this.localStream && !this.vLocal.srcObject) { - this.vLocal.srcObject = this.localStream.stream - } - }) + this.localStreams.forEach(stream => { + stream.toggleVideo() + .then(enabled => { + this.toggleVideoUI(enabled) + this.load.then(() => { + if (this.vLocal && stream && !this.vLocal.srcObject) { + this.vLocal.srcObject = stream.stream + } + }).catch(e => this.logError(e)) + }).catch(e => this.logError(e)) }) } remove() { - this.localStream?.stop() clearInterval(this.tsInterval) clearInterval(this.checkRemoteVideoInterval) - if (this.iframe.parentElement) { - document.body.removeChild(this.iframe) + if (this.audioContainer && this.audioContainer.parentElement) { + this.audioContainer.parentElement.removeChild(this.audioContainer) + this.audioContainer = null } - if (this.aRemote && this.aRemote.parentElement) { - document.body.removeChild(this.aRemote) + if (this.iframe.parentElement) { + this.iframe.parentElement.removeChild(this.iframe) } sessionStorage.removeItem(SS_START_TS_KEY) + this.localStreams = [] } -} \ No newline at end of file +} diff --git a/tracker/tracker-assist/src/ConfirmWindow/ConfirmWindow.ts b/tracker/tracker-assist/src/ConfirmWindow/ConfirmWindow.ts index fd7209689..4de359ac9 100644 --- a/tracker/tracker-assist/src/ConfirmWindow/ConfirmWindow.ts +++ b/tracker/tracker-assist/src/ConfirmWindow/ConfirmWindow.ts @@ -110,17 +110,15 @@ export default class ConfirmWindow { this.wrapper = wrapper confirmBtn.onclick = () => { - this._remove() this.resolve(true) } declineBtn.onclick = () => { - this._remove() this.resolve(false) } } private resolve: (result: boolean) => void = () => {}; - private reject: () => void = () => {}; + private reject: (reason: string) => void = () => {}; mount(): Promise { document.body.appendChild(this.wrapper) @@ -135,10 +133,10 @@ export default class ConfirmWindow { if (!this.wrapper.parentElement) { return } - document.body.removeChild(this.wrapper) + this.wrapper.parentElement.removeChild(this.wrapper) } remove() { this._remove() - this.reject() + this.reject('no answer') } } diff --git a/tracker/tracker-assist/src/LocalStream.ts b/tracker/tracker-assist/src/LocalStream.ts index 7f233108a..78c9ccff8 100644 --- a/tracker/tracker-assist/src/LocalStream.ts +++ b/tracker/tracker-assist/src/LocalStream.ts @@ -22,6 +22,7 @@ export default function RequestLocalStream(): Promise { return navigator.mediaDevices.getUserMedia({ audio:true, }) .then(aStream => { const aTrack = aStream.getAudioTracks()[0] + if (!aTrack) { throw new Error('No audio tracks provided') } return new _LocalStream(aTrack) }) @@ -54,6 +55,7 @@ class _LocalStream { }) .catch(e => { // TODO: log + console.error(e) return false }) } diff --git a/tracker/tracker-assist/src/Mouse.ts b/tracker/tracker-assist/src/Mouse.ts index a6164e153..afb50a1f1 100644 --- a/tracker/tracker-assist/src/Mouse.ts +++ b/tracker/tracker-assist/src/Mouse.ts @@ -2,7 +2,7 @@ type XY = [number, number] export default class Mouse { - private mouse: HTMLDivElement + private readonly mouse: HTMLDivElement private position: [number,number] = [0,0,] constructor() { this.mouse = document.createElement('div') @@ -52,8 +52,8 @@ export default class Mouse { private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct private lastScrEl: Element | 'window' | null = null - private resetLastScrEl = () => { this.lastScrEl = null } - private handleWScroll = e => { + private readonly resetLastScrEl = () => { this.lastScrEl = null } + private readonly handleWScroll = e => { if (e.target !== this.lastScrEl && this.lastScrEl !== 'window') { this.resetLastScrEl() @@ -111,4 +111,4 @@ export default class Mouse { window.removeEventListener('scroll', this.handleWScroll) window.removeEventListener('resize', this.resetLastScrEl) } -} \ No newline at end of file +} diff --git a/tracker/tracker-assist/src/RemoteControl.ts b/tracker/tracker-assist/src/RemoteControl.ts index 4cbc785f3..5cad2ac18 100644 --- a/tracker/tracker-assist/src/RemoteControl.ts +++ b/tracker/tracker-assist/src/RemoteControl.ts @@ -23,9 +23,9 @@ export default class RemoteControl { private agentID: string | null = null constructor( - private options: AssistOptions, - private onGrand: (sting?) => void, - private onRelease: (sting?) => void) {} + private readonly options: AssistOptions, + private readonly onGrand: (sting?) => void, + private readonly onRelease: (sting?) => void) {} reconnect(ids: string[]) { const storedID = sessionStorage.getItem(this.options.session_control_peer_key) @@ -56,7 +56,7 @@ export default class RemoteControl { } else { this.releaseControl(id) } - }).catch() + }).catch(e => console.error(e)) } grantControl = (id: string) => { this.agentID = id @@ -99,4 +99,4 @@ export default class RemoteControl { this.focused.innerText = value } } -} \ No newline at end of file +}