jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ui/tracker/player): handle multiple callers in one assist session

Browse source 
feat(ui/tracker/player): small fixes

feat(ui/tracker/player): fix incoming streams

feat(tracker): some logs

feat(tracker): fix types, fix stream binding

feat(tracker): more stuff for multicall...

feat(tracker): more stuff for multicall...

feat(tracker): more stuff for multicall...

feat(tracker): more stuff for multicall...

feat(tracker): more stuff for multicall...

feat(tracker): more stuff for multicall...

feat(tracker): more stuff for multicall...

feat(tracker): rm async

feat(tracker): rewrite stuff

feat(tracker): rewrite stuff

feat(tracker): rewrite stuff

feat(tracker): rewrite stuff

feat(tracker): rewrite stuff

feat(tracker): rewrite lstream

feat(tracker): rewrite lstream

feat(tracker): rewrite stuff

feat(tracker): rewrite stuff

feat(tracker): fix group call

feat(tracker): fix group call

feat(tracker): fix group call

feat(tracker): fix group call

feat(tracker): add text to ui

feat(tracker): destroy calls obj on call end

feat(tracker): rm unused prop

fix(tracker-assist):simplify addRemoteStream logic

fixup! fix(tracker-assist):simplify addRemoteStream logic

refactor(tracker-assist): make multi-agents call logic more explicite => fixed few bugs
This commit is contained in:
sylenien 2022-07-13 15:46:32 +02:00 • committed by Delirium
parent 334eb69edd
commit e2a10c0751
18 changed files with 429 additions and 280 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
frontend/app/components/Assist/ChatControls/ChatControls.tsx
View file

@ -9,9 +9,10 @@ interface Props {
stream: LocalStream | null, stream: LocalStream | null,
endCall: () => void, endCall: () => void,
videoEnabled: boolean, videoEnabled: boolean,
setVideoEnabled: (boolean) => void isPrestart?: boolean,
setVideoEnabled: (isEnabled: boolean) => void
} }
function ChatControls({ stream, endCall, videoEnabled, setVideoEnabled } : Props) { function ChatControls({ stream, endCall, videoEnabled, setVideoEnabled, isPrestart } : Props) {
const [audioEnabled, setAudioEnabled] = useState(true) const [audioEnabled, setAudioEnabled] = useState(true)
const toggleAudio = () => { const toggleAudio = () => {
@ -25,6 +26,13 @@ function ChatControls({ stream, endCall, videoEnabled, setVideoEnabled } : Props
.then(setVideoEnabled) .then(setVideoEnabled)
} }
/** muting user if he is auto connected to the call */
React.useEffect(() => {
if (isPrestart) {
audioEnabled && toggleAudio();
}
}, [])
return ( return (
<div className={cn(stl.controls, "flex items-center w-full justify-start bottom-0 px-2")}> <div className={cn(stl.controls, "flex items-center w-full justify-start bottom-0 px-2")}>
<div className="flex items-center"> <div className="flex items-center">

27
frontend/app/components/Assist/ChatWindow/ChatWindow.tsx
View file

@ -1,5 +1,4 @@
//@ts-nocheck import React, { useState, useEffect } from 'react'
import React, { useState, FC, useEffect } from 'react'
import VideoContainer from '../components/VideoContainer' import VideoContainer from '../components/VideoContainer'
import cn from 'classnames' import cn from 'classnames'
import Counter from 'App/components/shared/SessionItem/Counter' import Counter from 'App/components/shared/SessionItem/Counter'
@ -8,23 +7,23 @@ import ChatControls from '../ChatControls/ChatControls'
import Draggable from 'react-draggable'; import Draggable from 'react-draggable';
import type { LocalStream } from 'Player/MessageDistributor/managers/LocalStream'; import type { LocalStream } from 'Player/MessageDistributor/managers/LocalStream';
export interface Props { export interface Props {
incomeStream: MediaStream | null, incomeStream: MediaStream[] | null,
localStream: LocalStream | null, localStream: LocalStream | null,
userId: String, userId: string,
isPrestart?: boolean;
endCall: () => void endCall: () => void
} }
const ChatWindow: FC<Props> = function ChatWindow({ userId, incomeStream, localStream, endCall }) { function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }: Props) {
const [localVideoEnabled, setLocalVideoEnabled] = useState(false) const [localVideoEnabled, setLocalVideoEnabled] = useState(false)
const [remoteVideoEnabled, setRemoteVideoEnabled] = useState(false) const [remoteVideoEnabled, setRemoteVideoEnabled] = useState(false)
useEffect(() => { useEffect(() => {
if (!incomeStream) { return } if (!incomeStream || incomeStream.length === 0) { return }
const iid = setInterval(() => { const iid = setInterval(() => {
const settings = incomeStream.getVideoTracks()[0]?.getSettings() const settings = incomeStream.map(stream => stream.getVideoTracks()[0]?.getSettings()).filter(Boolean)
const isDummyVideoTrack = !!settings ? (settings.width === 2 || settings.frameRate === 0) : true const isDummyVideoTrack = settings.length > 0 ? (settings.every(s => s.width === 2 || s.frameRate === 0 || s.frameRate === undefined)) : true
const shouldBeEnabled = !isDummyVideoTrack const shouldBeEnabled = !isDummyVideoTrack
if (shouldBeEnabled !== localVideoEnabled) { if (shouldBeEnabled !== localVideoEnabled) {
setRemoteVideoEnabled(shouldBeEnabled) setRemoteVideoEnabled(shouldBeEnabled)
@ -42,16 +41,20 @@ const ChatWindow: FC<Props> = function ChatWindow({ userId, incomeStream, localS
style={{ width: '280px' }} style={{ width: '280px' }}
> >
<div className="handle flex items-center p-2 cursor-move select-none border-b"> <div className="handle flex items-center p-2 cursor-move select-none border-b">
<div className={stl.headerTitle}><b>Talking to </b> {userId ? userId : 'Anonymous User'}</div> <div className={stl.headerTitle}>
<b>Talking to </b> {userId ? userId : 'Anonymous User'}
{incomeStream && incomeStream.length > 2 ? ' (+ other agents in the call)' : ''}
</div>
<Counter startTime={new Date().getTime() } className="text-sm ml-auto" /> <Counter startTime={new Date().getTime() } className="text-sm ml-auto" />
</div> </div>
<div className={cn(stl.videoWrapper, {'hidden' : minimize}, 'relative')}> <div className={cn(stl.videoWrapper, {'hidden' : minimize}, 'relative')}>
<VideoContainer stream={ incomeStream } /> {!incomeStream && <div className={stl.noVideo}>Error obtaining incoming streams</div>}
{incomeStream && incomeStream.map(stream => <VideoContainer stream={ stream } />)}
<div className="absolute bottom-0 right-0 z-50"> <div className="absolute bottom-0 right-0 z-50">
<VideoContainer stream={ localStream ? localStream.stream : null } muted width={50} /> <VideoContainer stream={ localStream ? localStream.stream : null } muted width={50} />
</div> </div>
</div> </div>
<ChatControls videoEnabled={localVideoEnabled} setVideoEnabled={setLocalVideoEnabled} stream={localStream} endCall={endCall} /> <ChatControls videoEnabled={localVideoEnabled} setVideoEnabled={setLocalVideoEnabled} stream={localStream} endCall={endCall} isPrestart={isPrestart} />
</div> </div>
</Draggable> </Draggable>
) )

66
frontend/app/components/Assist/components/AssistActions/AssistActions.tsx
View file

@ -1,11 +1,12 @@
import React, { useState, useEffect } from 'react'; import React, { useState, useEffect } from 'react';
import { Popup, Icon, Button, IconButton } from 'UI'; import { Popup, Icon, Button, IconButton } from 'UI';
import logger from 'App/logger';
import { connect } from 'react-redux'; import { connect } from 'react-redux';
import cn from 'classnames'; import cn from 'classnames';
import { toggleChatWindow } from 'Duck/sessions'; import { toggleChatWindow } from 'Duck/sessions';
import { connectPlayer } from 'Player/store'; import { connectPlayer } from 'Player/store';
import ChatWindow from '../../ChatWindow'; import ChatWindow from '../../ChatWindow';
import { callPeer, requestReleaseRemoteControl, toggleAnnotation } from 'Player'; import { callPeer, setCallArgs, requestReleaseRemoteControl, toggleAnnotation } from 'Player';
import { CallingState, ConnectionStatus, RemoteControlStatus } from 'Player/MessageDistributor/managers/AssistManager'; import { CallingState, ConnectionStatus, RemoteControlStatus } from 'Player/MessageDistributor/managers/AssistManager';
import RequestLocalStream from 'Player/MessageDistributor/managers/LocalStream'; import RequestLocalStream from 'Player/MessageDistributor/managers/LocalStream';
import type { LocalStream } from 'Player/MessageDistributor/managers/LocalStream'; import type { LocalStream } from 'Player/MessageDistributor/managers/LocalStream';
@ -14,15 +15,12 @@ import { toast } from 'react-toastify';
import { confirm } from 'UI'; import { confirm } from 'UI';
import stl from './AassistActions.module.css'; import stl from './AassistActions.module.css';
function onClose(stream) {
stream.getTracks().forEach((t) => t.stop());
}
function onReject() { function onReject() {
toast.info(`Call was rejected.`); toast.info(`Call was rejected.`);
} }
function onError(e) { function onError(e) {
console.log(e)
toast.error(typeof e === 'string' ? e : e.message); toast.error(typeof e === 'string' ? e : e.message);
} }
@ -35,6 +33,8 @@ interface Props {
remoteControlStatus: RemoteControlStatus; remoteControlStatus: RemoteControlStatus;
hasPermission: boolean; hasPermission: boolean;
isEnterprise: boolean; isEnterprise: boolean;
isCallActive: boolean;
agentIds: string[];
} }
function AssistActions({ function AssistActions({
@ -46,14 +46,21 @@ function AssistActions({
remoteControlStatus, remoteControlStatus,
hasPermission, hasPermission,
isEnterprise, isEnterprise,
isCallActive,
agentIds
}: Props) { }: Props) {
const [incomeStream, setIncomeStream] = useState<MediaStream | null>(null); const [isPrestart, setPrestart] = useState(false);
const [incomeStream, setIncomeStream] = useState<MediaStream[] | null>([]);
const [localStream, setLocalStream] = useState<LocalStream | null>(null); const [localStream, setLocalStream] = useState<LocalStream | null>(null);
const [callObject, setCallObject] = useState<{ end: () => void } | null>(null); const [callObject, setCallObject] = useState<{ end: () => void } | null>(null);
const onCall = calling === CallingState.OnCall || calling === CallingState.Reconnecting;
const cannotCall = peerConnectionStatus !== ConnectionStatus.Connected || (isEnterprise && !hasPermission);
const remoteActive = remoteControlStatus === RemoteControlStatus.Enabled;
useEffect(() => { useEffect(() => {
return callObject?.end(); return callObject?.end()
}, []); }, [])
useEffect(() => { useEffect(() => {
if (peerConnectionStatus == ConnectionStatus.Disconnected) { if (peerConnectionStatus == ConnectionStatus.Disconnected) {
@ -61,15 +68,36 @@ function AssistActions({
} }
}, [peerConnectionStatus]); }, [peerConnectionStatus]);
function call() { const addIncomeStream = (stream: MediaStream) => {
RequestLocalStream() console.log('new stream in component')
.then((lStream) => { setIncomeStream(oldState => [...oldState, stream]);
setLocalStream(lStream);
setCallObject(callPeer(lStream, setIncomeStream, lStream.stop.bind(lStream), onReject, onError));
})
.catch(onError);
} }
function call(agentIds?: string[]) {
RequestLocalStream().then(lStream => {
setLocalStream(lStream);
setCallArgs(
lStream,
addIncomeStream,
lStream.stop.bind(lStream),
onReject,
onError
)
setCallObject(callPeer());
if (agentIds) {
callPeer(agentIds)
}
}).catch(onError)
}
React.useEffect(() => {
if (!onCall && isCallActive && agentIds) {
logger.log('joinig the party', agentIds)
setPrestart(true);
call(agentIds)
}
}, [agentIds, isCallActive])
const confirmCall = async () => { const confirmCall = async () => {
if ( if (
await confirm({ await confirm({
@ -82,10 +110,6 @@ function AssistActions({
} }
}; };
const onCall = calling === CallingState.OnCall || calling === CallingState.Reconnecting;
const cannotCall = peerConnectionStatus !== ConnectionStatus.Connected || (isEnterprise && !hasPermission);
const remoteActive = remoteControlStatus === RemoteControlStatus.Enabled;
return ( return (
<div className="flex items-center"> <div className="flex items-center">
{(onCall || remoteActive) && ( {(onCall || remoteActive) && (
@ -123,7 +147,7 @@ function AssistActions({
</div> </div>
<div className={stl.divider} /> <div className={stl.divider} />
<Popup content={cannotCall ? 'You don’t have the permissions to perform this action.' : `Call ${userId ? userId : 'User'}`}> <Popup content={cannotCall ? `You don't have the permissions to perform this action.` : `Call ${userId ? userId : 'User'}`}>
<div <div
className={cn('cursor-pointer p-2 flex items-center', { [stl.disabled]: cannotCall })} className={cn('cursor-pointer p-2 flex items-center', { [stl.disabled]: cannotCall })}
onClick={onCall ? callObject?.end : confirmCall} onClick={onCall ? callObject?.end : confirmCall}
@ -138,7 +162,7 @@ function AssistActions({
<div className="fixed ml-3 left-0 top-0" style={{ zIndex: 999 }}> <div className="fixed ml-3 left-0 top-0" style={{ zIndex: 999 }}>
{onCall && callObject && ( {onCall && callObject && (
<ChatWindow endCall={callObject.end} userId={userId} incomeStream={incomeStream} localStream={localStream} /> <ChatWindow endCall={callObject.end} userId={userId} incomeStream={incomeStream} localStream={localStream} isPrestart={isPrestart} />
)} )}
</div> </div>
</div> </div>

4
frontend/app/components/Session_/PlayerBlockHeader.js
View file

@ -105,7 +105,7 @@ export default class PlayerBlockHeader extends React.PureComponent {
const { hideBack } = this.state; const { hideBack } = this.state;
const { sessionId, userId, userNumericHash, live, metadata } = session; const { sessionId, userId, userNumericHash, live, metadata, isCallActive, agentIds } = session;
let _metaList = Object.keys(metadata) let _metaList = Object.keys(metadata)
.filter((i) => metaList.includes(i)) .filter((i) => metaList.includes(i))
.map((key) => { .map((key) => {
@ -142,7 +142,7 @@ export default class PlayerBlockHeader extends React.PureComponent {
</div> </div>
)} )}
{isAssist && <AssistActions userId={userId} />} {isAssist && <AssistActions userId={userId} isCallActive={isCallActive} agentIds={agentIds} />}
</div> </div>
</div> </div>
{!isAssist && ( {!isAssist && (

11
frontend/app/components/shared/SessionItem/SessionItem.tsx
View file

@ -41,6 +41,8 @@ interface Props {
userSessionsCount: number; userSessionsCount: number;
issueTypes: []; issueTypes: [];
active: boolean; active: boolean;
isCallActive?: boolean;
agentIds?: string[];
}; };
onUserClick?: (userId: string, userAnonymousId: string) => void; onUserClick?: (userId: string, userAnonymousId: string) => void;
hasUserFilter?: boolean; hasUserFilter?: boolean;
@ -168,6 +170,15 @@ function SessionItem(props: RouteComponentProps & Props) {
<div className="flex items-center"> <div className="flex items-center">
<div className={stl.playLink} id="play-button" data-viewed={viewed}> <div className={stl.playLink} id="play-button" data-viewed={viewed}>
{live && session.isCallActive && session.agentIds.length > 0 ? (
<div className="mr-4">
<Label className="bg-gray-lightest p-1 px-2 rounded-lg">
<span className="color-gray-medium text-xs" style={{ whiteSpace: 'nowrap' }}>
CALL IN PROGRESS
</span>
</Label>
</div>
) : null}
{isSessions && ( {isSessions && (
<div className="mr-4 flex-shrink-0 w-24"> <div className="mr-4 flex-shrink-0 w-24">
{isLastPlayed && ( {isLastPlayed && (

147
frontend/app/player/MessageDistributor/managers/AssistManager.ts
View file

@ -136,7 +136,6 @@ export default class AssistManager {
//agentInfo: JSON.stringify({}) //agentInfo: JSON.stringify({})
} }
}) })
//socket.onAny((...args) => console.log(...args))
socket.on("connect", () => { socket.on("connect", () => {
waitingForMessages = true waitingForMessages = true
this.setStatus(ConnectionStatus.WaitingMessages) // TODO: happens frequently on bad network this.setStatus(ConnectionStatus.WaitingMessages) // TODO: happens frequently on bad network
@ -146,7 +145,6 @@ export default class AssistManager {
update({ calling: CallingState.NoCall }) update({ calling: CallingState.NoCall })
}) })
socket.on('messages', messages => { socket.on('messages', messages => {
//console.log(messages.filter(m => m._id === 41 || m._id === 44))
jmr.append(messages) // as RawMessage[] jmr.append(messages) // as RawMessage[]
if (waitingForMessages) { if (waitingForMessages) {
@ -176,11 +174,11 @@ export default class AssistManager {
this.setStatus(ConnectionStatus.Connected) this.setStatus(ConnectionStatus.Connected)
}) })
socket.on('UPDATE_SESSION', ({ active }) => { socket.on('UPDATE_SESSION', (data) => {
showDisconnectTimeout && clearTimeout(showDisconnectTimeout) showDisconnectTimeout && clearTimeout(showDisconnectTimeout)
!inactiveTimeout && this.setStatus(ConnectionStatus.Connected) !inactiveTimeout && this.setStatus(ConnectionStatus.Connected)
if (typeof active === "boolean") { if (typeof data.active === "boolean") {
if (active) { if (data.active) {
inactiveTimeout && clearTimeout(inactiveTimeout) inactiveTimeout && clearTimeout(inactiveTimeout)
this.setStatus(ConnectionStatus.Connected) this.setStatus(ConnectionStatus.Connected)
} else { } else {
@ -305,7 +303,7 @@ export default class AssistManager {
private _peer: Peer | null = null private _peer: Peer | null = null
private connectionAttempts: number = 0 private connectionAttempts: number = 0
private callConnection: MediaConnection | null = null private callConnection: MediaConnection[] = []
private getPeer(): Promise<Peer> { private getPeer(): Promise<Peer> {
if (this._peer && !this._peer.disconnected) { return Promise.resolve(this._peer) } if (this._peer && !this._peer.disconnected) { return Promise.resolve(this._peer) }
@ -326,6 +324,32 @@ export default class AssistManager {
}; };
} }
const peer = this._peer = new Peer(peerOpts) const peer = this._peer = new Peer(peerOpts)
peer.on('call', call => {
console.log('getting call from', call.peer)
call.answer(this.callArgs.localStream.stream)
this.callConnection.push(call)
this.callArgs.localStream.onVideoTrack(vTrack => {
const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video")
if (!sender) {
console.warn("No video sender found")
return
}
sender.replaceTrack(vTrack)
})
call.on('stream', stream => {
this.callArgs && this.callArgs.onStream(stream)
});
call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
call.on("close", this.onRemoteCallEnd)
call.on("error", (e) => {
console.error("PeerJS error (on call):", e)
this.initiateCallEnd();
this.callArgs && this.callArgs.onError && this.callArgs.onError();
});
})
peer.on('error', e => { peer.on('error', e => {
if (e.type === 'disconnected') { if (e.type === 'disconnected') {
return peer.reconnect() return peer.reconnect()
@ -351,21 +375,21 @@ export default class AssistManager {
private handleCallEnd() { private handleCallEnd() {
this.callArgs && this.callArgs.onCallEnd() this.callArgs && this.callArgs.onCallEnd()
this.callConnection && this.callConnection.close() this.callConnection[0] && this.callConnection[0].close()
update({ calling: CallingState.NoCall }) update({ calling: CallingState.NoCall })
this.callArgs = null this.callArgs = null
this.toggleAnnotation(false) this.toggleAnnotation(false)
} }
private initiateCallEnd = () => { private initiateCallEnd = async () => {
this.socket?.emit("call_end") this.socket?.emit("call_end", store.getState().getIn([ 'user', 'account', 'name']))
this.handleCallEnd() this.handleCallEnd()
} }
private onRemoteCallEnd = () => { private onRemoteCallEnd = () => {
if (getState().calling === CallingState.Requesting) { if (getState().calling === CallingState.Requesting) {
this.callArgs && this.callArgs.onReject() this.callArgs && this.callArgs.onReject()
this.callConnection && this.callConnection.close() this.callConnection[0] && this.callConnection[0].close()
update({ calling: CallingState.NoCall }) update({ calling: CallingState.NoCall })
this.callArgs = null this.callArgs = null
this.toggleAnnotation(false) this.toggleAnnotation(false)
@ -379,15 +403,16 @@ export default class AssistManager {
onStream: (s: MediaStream)=>void, onStream: (s: MediaStream)=>void,
onCallEnd: () => void, onCallEnd: () => void,
onReject: () => void, onReject: () => void,
onError?: ()=> void onError?: ()=> void,
} | null = null } | null = null
call( public setCallArgs(
localStream: LocalStream, localStream: LocalStream,
onStream: (s: MediaStream)=>void, onStream: (s: MediaStream)=>void,
onCallEnd: () => void, onCallEnd: () => void,
onReject: () => void, onReject: () => void,
onError?: ()=> void): { end: Function } { onError?: ()=> void,
) {
this.callArgs = { this.callArgs = {
localStream, localStream,
onStream, onStream,
@ -395,12 +420,66 @@ export default class AssistManager {
onReject, onReject,
onError, onError,
} }
this._call() }
public call(thirdPartyPeers?: string[]): { end: Function } {
if (thirdPartyPeers && thirdPartyPeers.length > 0) {
this.addPeerCall(thirdPartyPeers)
} else {
this._callSessionPeer()
}
return { return {
end: this.initiateCallEnd, end: this.initiateCallEnd,
} }
} }
/** Connecting to the other agents that are already
* in the call with the user
*/
public addPeerCall(thirdPartyPeers: string[]) {
thirdPartyPeers.forEach(peer => this._peerConnection(peer))
}
/** Connecting to the app user */
private _callSessionPeer() {
if (![CallingState.NoCall, CallingState.Reconnecting].includes(getState().calling)) { return }
update({ calling: CallingState.Connecting })
this._peerConnection(this.peerID);
this.socket && this.socket.emit("_agent_name", store.getState().getIn([ 'user', 'account', 'name']))
}
private async _peerConnection(remotePeerId: string) {
try {
const peer = await this.getPeer();
const call = peer.call(remotePeerId, this.callArgs.localStream.stream)
this.callConnection.push(call)
this.callArgs.localStream.onVideoTrack(vTrack => {
const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video")
if (!sender) {
console.warn("No video sender found")
return
}
sender.replaceTrack(vTrack)
})
call.on('stream', stream => {
getState().calling !== CallingState.OnCall && update({ calling: CallingState.OnCall })
this.callArgs && this.callArgs.onStream(stream)
});
call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
call.on("close", this.onRemoteCallEnd)
call.on("error", (e) => {
console.error("PeerJS error (on call):", e)
this.initiateCallEnd();
this.callArgs && this.callArgs.onError && this.callArgs.onError();
});
} catch (e) {
console.error(e)
}
}
toggleAnnotation(enable?: boolean) { toggleAnnotation(enable?: boolean) {
// if (getState().calling !== CallingState.OnCall) { return } // if (getState().calling !== CallingState.OnCall) { return }
if (typeof enable !== "boolean") { if (typeof enable !== "boolean") {
@ -442,44 +521,6 @@ export default class AssistManager {
private annot: AnnotationCanvas | null = null private annot: AnnotationCanvas | null = null
private _call() {
if (![CallingState.NoCall, CallingState.Reconnecting].includes(getState().calling)) { return }
update({ calling: CallingState.Connecting })
this.getPeer().then(peer => {
if (!this.callArgs) { return console.log("No call Args. Must not happen.") }
update({ calling: CallingState.Requesting })
// TODO: in a proper way
this.socket && this.socket.emit("_agent_name", store.getState().getIn([ 'user', 'account', 'name']))
const call = this.callConnection = peer.call(this.peerID, this.callArgs.localStream.stream)
this.callArgs.localStream.onVideoTrack(vTrack => {
const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video")
if (!sender) {
console.warn("No video sender found")
return
}
//logger.log("sender found:", sender)
sender.replaceTrack(vTrack)
})
call.on('stream', stream => {
update({ calling: CallingState.OnCall })
this.callArgs && this.callArgs.onStream(stream)
});
//call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
call.on("close", this.onRemoteCallEnd)
call.on("error", (e) => {
console.error("PeerJS error (on call):", e)
this.initiateCallEnd();
this.callArgs && this.callArgs.onError && this.callArgs.onError();
});
})
}
/* ==== Cleaning ==== */ /* ==== Cleaning ==== */
private cleaned: boolean = false private cleaned: boolean = false
clear() { clear() {
@ -502,5 +543,3 @@ export default class AssistManager {
} }
} }
} }

1
frontend/app/player/MessageDistributor/managers/LocalStream.ts
View file

@ -54,6 +54,7 @@ class _LocalStream {
}) })
.catch(e => { .catch(e => {
// TODO: log // TODO: log
console.error(e)
return false return false
}) })
} }

5
frontend/app/player/singletone.js
View file

@ -2,7 +2,7 @@ import Player from './Player';
import { update, clean as cleanStore, getState } from './store'; import { update, clean as cleanStore, getState } from './store';
import { clean as cleanLists } from './lists'; import { clean as cleanLists } from './lists';
/** @type {Player} */
let instance = null; let instance = null;
const initCheck = method => (...args) => { const initCheck = method => (...args) => {
@ -69,7 +69,10 @@ export const attach = initCheck((...args) => instance.attach(...args));
export const markElement = initCheck((...args) => instance.marker && instance.marker.mark(...args)); export const markElement = initCheck((...args) => instance.marker && instance.marker.mark(...args));
export const scale = initCheck(() => instance.scale()); export const scale = initCheck(() => instance.scale());
export const toggleInspectorMode = initCheck((...args) => instance.toggleInspectorMode(...args)); export const toggleInspectorMode = initCheck((...args) => instance.toggleInspectorMode(...args));
/** @type {Player.assistManager.call} */
export const callPeer = initCheck((...args) => instance.assistManager.call(...args)) export const callPeer = initCheck((...args) => instance.assistManager.call(...args))
/** @type {Player.assistManager.setCallArgs} */
export const setCallArgs = initCheck((...args) => instance.assistManager.setCallArgs(...args))
export const requestReleaseRemoteControl = initCheck((...args) => instance.assistManager.requestReleaseRemoteControl(...args)) export const requestReleaseRemoteControl = initCheck((...args) => instance.assistManager.requestReleaseRemoteControl(...args))
export const markTargets = initCheck((...args) => instance.markTargets(...args)) export const markTargets = initCheck((...args) => instance.markTargets(...args))
export const activeTarget = initCheck((...args) => instance.activeTarget(...args)) export const activeTarget = initCheck((...args) => instance.activeTarget(...args))

2
frontend/app/types/session/session.ts
View file

@ -79,6 +79,8 @@ export default Record({
isIOS: false, isIOS: false,
revId: '', revId: '',
userSessionsCount: 0, userSessionsCount: 0,
agentIds: [],
isCallActive: false
}, { }, {
fromJS:({ fromJS:({
startTs=0, startTs=0,

1
frontend/package.json
View file

@ -83,6 +83,7 @@
"@openreplay/sourcemap-uploader": "^3.0.0", "@openreplay/sourcemap-uploader": "^3.0.0",
"@types/react": "^18.0.9", "@types/react": "^18.0.9",
"@types/react-dom": "^18.0.4", "@types/react-dom": "^18.0.4",
"@types/react-redux": "^7.1.24",
"@types/react-router-dom": "^5.3.3", "@types/react-router-dom": "^5.3.3",
"@typescript-eslint/eslint-plugin": "^5.24.0", "@typescript-eslint/eslint-plugin": "^5.24.0",
"@typescript-eslint/parser": "^5.24.0", "@typescript-eslint/parser": "^5.24.0",

6
tracker/tracker-assist/.eslintrc.cjs
View file

@ -24,12 +24,14 @@ module.exports = {
'@typescript-eslint/camelcase': 'off', '@typescript-eslint/camelcase': 'off',
'@typescript-eslint/no-explicit-any': 'off', '@typescript-eslint/no-explicit-any': 'off',
'@typescript-eslint/unbound-method': 'off', '@typescript-eslint/unbound-method': 'off',
'@typescript-eslint/explicit-function-return-type': 'warn',
'@typescript-eslint/prefer-readonly': 'warn', '@typescript-eslint/prefer-readonly': 'warn',
'@typescript-eslint/ban-ts-comment': 'off', '@typescript-eslint/ban-ts-comment': 'off',
'@typescript-eslint/no-unsafe-assignment': 'off', '@typescript-eslint/no-unsafe-assignment': 'off',
'@typescript-eslint/no-unsafe-member-access': 'off', '@typescript-eslint/no-unsafe-member-access': 'off',
'@typescript-eslint/no-unused-expressions': 'off',
'@typescript-eslint/no-unsafe-call': 'off', '@typescript-eslint/no-unsafe-call': 'off',
'@typescript-eslint/no-unsafe-argument': 'off',
'@typescript-eslint/explicit-function-return-type': 'off',
'@typescript-eslint/restrict-plus-operands': 'warn', '@typescript-eslint/restrict-plus-operands': 'warn',
'@typescript-eslint/no-unsafe-return': 'warn', '@typescript-eslint/no-unsafe-return': 'warn',
'no-useless-escape': 'warn', 'no-useless-escape': 'warn',
@ -38,9 +40,7 @@ module.exports = {
'@typescript-eslint/no-useless-constructor': 'warn', '@typescript-eslint/no-useless-constructor': 'warn',
'@typescript-eslint/no-this-alias': 'off', '@typescript-eslint/no-this-alias': 'off',
'@typescript-eslint/no-floating-promises': 'warn', '@typescript-eslint/no-floating-promises': 'warn',
'@typescript-eslint/no-unsafe-argument': 'warn',
'no-unused-expressions': 'off', 'no-unused-expressions': 'off',
'@typescript-eslint/no-unused-expressions': 'warn',
'@typescript-eslint/no-useless-constructor': 'warn', '@typescript-eslint/no-useless-constructor': 'warn',
'semi': ["error", "never"], 'semi': ["error", "never"],
'quotes': ["error", "single"], 'quotes': ["error", "single"],

2
tracker/tracker-assist/package.json
View file

@ -34,6 +34,7 @@
"@openreplay/tracker": "^3.5.3" "@openreplay/tracker": "^3.5.3"
}, },
"devDependencies": { "devDependencies": {
"@openreplay/tracker": "file:../tracker",
"@typescript-eslint/eslint-plugin": "^5.30.0", "@typescript-eslint/eslint-plugin": "^5.30.0",
"@typescript-eslint/parser": "^5.30.0", "@typescript-eslint/parser": "^5.30.0",
"eslint": "^7.8.0", "eslint": "^7.8.0",
@ -42,7 +43,6 @@
"husky": "^8.0.1", "husky": "^8.0.1",
"lint-staged": "^13.0.3", "lint-staged": "^13.0.3",
"prettier": "^2.7.1", "prettier": "^2.7.1",
"@openreplay/tracker": "file:../tracker",
"replace-in-files-cli": "^1.0.0", "replace-in-files-cli": "^1.0.0",
"typescript": "^4.6.0-dev.20211126" "typescript": "^4.6.0-dev.20211126"
}, },

279
tracker/tracker-assist/src/Assist.ts
View file

@ -1,11 +1,11 @@
/* eslint-disable @typescript-eslint/no-empty-function */ /* eslint-disable @typescript-eslint/no-empty-function */
import type { Socket, } from 'socket.io-client' import type { Socket, } from 'socket.io-client'
import { connect, } from 'socket.io-client' import { connect, } from 'socket.io-client'
import Peer from 'peerjs' import Peer, { MediaConnection, } from 'peerjs'
import type { Properties, } from 'csstype' import type { Properties, } from 'csstype'
import { App, } from '@openreplay/tracker' import { App, } from '@openreplay/tracker'
import RequestLocalStream from './LocalStream.js' import RequestLocalStream, { LocalStream, } from './LocalStream.js'
import RemoteControl from './RemoteControl.js' import RemoteControl from './RemoteControl.js'
import CallWindow from './CallWindow.js' import CallWindow from './CallWindow.js'
import AnnotationCanvas from './AnnotationCanvas.js' import AnnotationCanvas from './AnnotationCanvas.js'
@ -13,7 +13,7 @@ import ConfirmWindow from './ConfirmWindow/ConfirmWindow.js'
import { callConfirmDefault, } from './ConfirmWindow/defaults.js' import { callConfirmDefault, } from './ConfirmWindow/defaults.js'
import type { Options as ConfirmOptions, } from './ConfirmWindow/defaults.js' import type { Options as ConfirmOptions, } from './ConfirmWindow/defaults.js'
// TODO: fully specified strict check (everywhere) // TODO: fully specified strict check with no-any (everywhere)
type StartEndCallback = () => ((()=>Record<string, unknown>) | void) type StartEndCallback = () => ((()=>Record<string, unknown>) | void)
@ -45,7 +45,7 @@ type OptionalCallback = (()=>Record<string, unknown>) | void
type Agent = { type Agent = {
onDisconnect?: OptionalCallback, onDisconnect?: OptionalCallback,
onControlReleased?: OptionalCallback, onControlReleased?: OptionalCallback,
name?: string //name?: string
// //
} }
@ -78,7 +78,7 @@ export default class Assist {
) )
if (document.hidden !== undefined) { if (document.hidden !== undefined) {
const sendActivityState = () => this.emit('UPDATE_SESSION', { active: !document.hidden, }) const sendActivityState = (): void => this.emit('UPDATE_SESSION', { active: !document.hidden, })
app.attachEventListener( app.attachEventListener(
document, document,
'visibilitychange', 'visibilitychange',
@ -111,7 +111,7 @@ export default class Assist {
app.session.attachUpdateCallback(sessInfo => this.emit('UPDATE_SESSION', sessInfo)) app.session.attachUpdateCallback(sessInfo => this.emit('UPDATE_SESSION', sessInfo))
} }
private emit(ev: string, ...args) { private emit(ev: string, ...args): void {
this.socket && this.socket.emit(ev, ...args) this.socket && this.socket.emit(ev, ...args)
} }
@ -119,14 +119,17 @@ export default class Assist {
return Object.keys(this.agents).length > 0 return Object.keys(this.agents).length > 0
} }
private notifyCallEnd() { private readonly setCallingState = (newState: CallingState): void => {
this.emit('call_end') this.callingState = newState
} }
private onRemoteCallEnd = () => {}
private onStart() { private onStart() {
const app = this.app const app = this.app
const peerID = `${app.getProjectKey()}-${app.getSessionID()}` const sessionId = app.getSessionID()
if (!sessionId) {
return app.debug.error('No session ID')
}
const peerID = `${app.getProjectKey()}-${sessionId}`
// SocketIO // SocketIO
const socket = this.socket = connect(app.getHost(), { const socket = this.socket = connect(app.getHost(), {
@ -187,51 +190,65 @@ export default class Assist {
socket.on('NEW_AGENT', (id: string, info) => { socket.on('NEW_AGENT', (id: string, info) => {
this.agents[id] = { this.agents[id] = {
onDisconnect: this.options.onAgentConnect && this.options.onAgentConnect(), onDisconnect: this.options.onAgentConnect?.(),
...info, // TODO ...info, // TODO
} }
this.assistDemandedRestart = true this.assistDemandedRestart = true
this.app.stop() this.app.stop()
this.app.start().then(() => { this.assistDemandedRestart = false }) this.app.start().then(() => { this.assistDemandedRestart = false }).catch(e => app.debug.error(e))
}) })
socket.on('AGENTS_CONNECTED', (ids: string[]) => { socket.on('AGENTS_CONNECTED', (ids: string[]) => {
ids.forEach(id =>{ ids.forEach(id =>{
this.agents[id] = { this.agents[id] = {
onDisconnect: this.options.onAgentConnect && this.options.onAgentConnect(), onDisconnect: this.options.onAgentConnect?.(),
} }
}) })
this.assistDemandedRestart = true this.assistDemandedRestart = true
this.app.stop() this.app.stop()
this.app.start().then(() => { this.assistDemandedRestart = false }) this.app.start().then(() => { this.assistDemandedRestart = false }).catch(e => app.debug.error(e))
remoteControl.reconnect(ids) remoteControl.reconnect(ids)
}) })
let confirmCall:ConfirmWindow | null = null
socket.on('AGENT_DISCONNECTED', (id) => { socket.on('AGENT_DISCONNECTED', (id) => {
remoteControl.releaseControl(id) remoteControl.releaseControl(id)
// close the call also this.agents[id]?.onDisconnect?.()
if (callingAgent === id) {
confirmCall?.remove()
this.onRemoteCallEnd()
}
// @ts-ignore (wtf, typescript?!)
this.agents[id] && this.agents[id].onDisconnect != null && this.agents[id].onDisconnect()
delete this.agents[id] delete this.agents[id]
endAgentCall(id)
}) })
socket.on('NO_AGENT', () => { socket.on('NO_AGENT', () => {
Object.values(this.agents).forEach(a => a.onDisconnect?.())
this.agents = {} this.agents = {}
}) })
socket.on('call_end', () => this.onRemoteCallEnd()) // TODO: check if agent calling id socket.on('call_end', (id) => {
if (!callingAgents.has(id)) {
app.debug.warn('Received call_end from unknown agent', id)
return
}
endAgentCall(id)
})
// TODO: fix the code socket.on('_agent_name', (id, name) => {
let agentName = '' callingAgents.set(id, name)
let callingAgent = '' updateCallerNames()
socket.on('_agent_name',(id, name) => { agentName = name; callingAgent = id }) })
const callingAgents: Map<string, string> = new Map() // !! uses socket.io ID
// TODO: merge peerId & socket.io id (simplest way - send peerId with the name)
const calls: Record<string, MediaConnection> = {} // !! uses peerJS ID
const lStreams: Record<string, LocalStream> = {}
// const callingPeers: Map<string, { call: MediaConnection, lStream: LocalStream }> = new Map() // Maybe
function endAgentCall(id: string) {
callingAgents.delete(id)
if (callingAgents.size === 0) {
handleCallEnd()
} else {
updateCallerNames()
//TODO: close() specific call and corresponding lStreams (after connecting peerId & socket.io id)
}
}
// PeerJS call (todo: use native WebRTC) // PeerJS call (todo: use native WebRTC)
const peerOptions = { const peerOptions = {
@ -244,119 +261,147 @@ export default class Assist {
peerOptions['config'] = this.options.config peerOptions['config'] = this.options.config
} }
const peer = this.peer = new Peer(peerID, peerOptions) const peer = this.peer = new Peer(peerID, peerOptions)
// app.debug.log('Peer created: ', peer)
// @ts-ignore // @ts-ignore (peerjs typing)
peer.on('error', e => app.debug.warn('Peer error: ', e.type, e)) peer.on('error', e => app.debug.warn('Peer error: ', e.type, e))
peer.on('disconnected', () => peer.reconnect()) peer.on('disconnected', () => peer.reconnect())
peer.on('call', (call) => {
app.debug.log('Call: ', call)
if (this.callingState !== CallingState.False) {
call.close()
//this.notifyCallEnd() // TODO: strictly connect calling peer with agent socket.id
app.debug.warn('Call closed instantly bacause line is busy. CallingState: ', this.callingState)
return
}
const setCallingState = (newState: CallingState) => { // Common for all incoming call requests
if (newState === CallingState.True) { let callUI: CallWindow | null = null
sessionStorage.setItem(this.options.session_calling_peer_key, call.peer) function updateCallerNames() {
} else if (newState === CallingState.False) { callUI?.setAssistentName(callingAgents)
sessionStorage.removeItem(this.options.session_calling_peer_key) }
} // TODO: incapsulate
this.callingState = newState let callConfirmWindow: ConfirmWindow | null = null
let callConfirmAnswer: Promise<boolean> | null = null
const closeCallConfirmWindow = () => {
if (callConfirmWindow) {
callConfirmWindow.remove()
callConfirmWindow = null
callConfirmAnswer = null
} }
}
const requestCallConfirm = () => {
if (callConfirmAnswer) { // Already asking
return callConfirmAnswer
}
callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || {
text: this.options.confirmText,
style: this.options.confirmStyle,
})) // TODO: reuse ?
return callConfirmAnswer = callConfirmWindow.mount().then(answer => {
closeCallConfirmWindow()
return answer
})
}
let callEndCallback: ReturnType<StartEndCallback> | null = null
const handleCallEnd = () => { // Completle stop and clear all calls
// Streams
Object.values(calls).forEach(call => call.close())
Object.keys(calls).forEach(peerId => delete calls[peerId])
Object.values(lStreams).forEach((stream) => { stream.stop() })
Object.keys(lStreams).forEach((peerId: string) => { delete lStreams[peerId] })
// UI
closeCallConfirmWindow()
callUI?.remove()
annot?.remove()
callUI = null
annot = null
this.emit('UPDATE_SESSION', { agentIds: [], isCallActive: false, })
this.setCallingState(CallingState.False)
sessionStorage.removeItem(this.options.session_calling_peer_key)
callEndCallback?.()
}
const initiateCallEnd = () => {
this.emit('call_end')
handleCallEnd()
}
peer.on('call', (call) => {
app.debug.log('Incoming call: ', call)
let confirmAnswer: Promise<boolean> let confirmAnswer: Promise<boolean>
const callingPeer = sessionStorage.getItem(this.options.session_calling_peer_key) const callingPeerIds = JSON.parse(sessionStorage.getItem(this.options.session_calling_peer_key) || '[]')
if (callingPeer === call.peer) { if (callingPeerIds.includes(call.peer) || this.callingState === CallingState.True) {
confirmAnswer = Promise.resolve(true) confirmAnswer = Promise.resolve(true)
} else { } else {
setCallingState(CallingState.Requesting) this.setCallingState(CallingState.Requesting)
confirmCall = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || { confirmAnswer = requestCallConfirm()
text: this.options.confirmText, this.playNotificationSound() // For every new agent during confirmation here
style: this.options.confirmStyle,
})) // TODO: only one (latest) timeout
confirmAnswer = confirmCall.mount()
this.playNotificationSound()
this.onRemoteCallEnd = () => { // if call cancelled by a caller before confirmation
app.debug.log('Received call_end during confirm window opened')
confirmCall?.remove()
setCallingState(CallingState.False)
call.close()
}
setTimeout(() => { setTimeout(() => {
if (this.callingState !== CallingState.Requesting) { return } if (this.callingState !== CallingState.Requesting) { return }
call.close() initiateCallEnd()
confirmCall?.remove()
this.notifyCallEnd()
setCallingState(CallingState.False)
}, 30000) }, 30000)
} }
confirmAnswer.then(agreed => { confirmAnswer.then(async agreed => {
if (!agreed) { if (!agreed) {
call.close() initiateCallEnd()
this.notifyCallEnd() return
setCallingState(CallingState.False) }
// Request local stream for the new connection
try {
// lStreams are reusable so fare we don't delete them in the `endAgentCall`
if (!lStreams[call.peer]) {
app.debug.log('starting new stream for', call.peer)
lStreams[call.peer] = await RequestLocalStream()
}
calls[call.peer] = call
} catch (e) {
app.debug.error('Audio mediadevice request error:', e)
initiateCallEnd()
return return
} }
const callUI = new CallWindow() // UI
annot = new AnnotationCanvas() if (!callUI) {
annot.mount() callUI = new CallWindow(app.debug.error)
callUI.setAssistentName(agentName) // TODO: as constructor options
callUI.setCallEndAction(initiateCallEnd)
const onCallEnd = this.options.onCallStart() callUI.setLocalStreams(Object.values(lStreams))
const handleCallEnd = () => {
app.debug.log('Handle Call End')
call.close()
callUI.remove()
annot && annot.remove()
annot = null
setCallingState(CallingState.False)
onCallEnd && onCallEnd()
} }
const initiateCallEnd = () => { if (!annot) {
this.notifyCallEnd() annot = new AnnotationCanvas()
handleCallEnd() annot.mount()
} }
this.onRemoteCallEnd = handleCallEnd
call.on('error', e => { call.on('error', e => {
app.debug.warn('Call error:', e) app.debug.warn('Call error:', e)
initiateCallEnd() initiateCallEnd()
}) })
call.on('stream', (rStream) => {
RequestLocalStream().then(lStream => { callUI?.addRemoteStream(rStream)
call.on('stream', function(rStream) { const onInteraction = () => { // do only if document.hidden ?
callUI.setRemoteStream(rStream) callUI?.playRemote()
const onInteraction = () => { // only if hidden? document.removeEventListener('click', onInteraction)
callUI.playRemote() }
document.removeEventListener('click', onInteraction) document.addEventListener('click', onInteraction)
}
document.addEventListener('click', onInteraction)
})
lStream.onVideoTrack(vTrack => {
const sender = call.peerConnection.getSenders().find(s => s.track?.kind === 'video')
if (!sender) {
app.debug.warn('No video sender found')
return
}
app.debug.log('sender found:', sender)
sender.replaceTrack(vTrack)
})
callUI.setCallEndAction(initiateCallEnd)
callUI.setLocalStream(lStream)
call.answer(lStream.stream)
setCallingState(CallingState.True)
}) })
.catch(e => {
app.debug.warn('Audio mediadevice request error:', e) // remote video on/off/camera change
initiateCallEnd() lStreams[call.peer].onVideoTrack(vTrack => {
const sender = call.peerConnection.getSenders().find(s => s.track?.kind === 'video')
if (!sender) {
app.debug.warn('No video sender found')
return
}
app.debug.log('sender found:', sender)
void sender.replaceTrack(vTrack)
}) })
}).catch() // in case of Confirm.remove() without any confirmation/decline
call.answer(lStreams[call.peer].stream)
this.setCallingState(CallingState.True)
if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() }
const callingPeerIds = Object.keys(calls)
sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIds))
this.emit('UPDATE_SESSION', { agentIds: callingPeerIds, isCallActive: true, })
}).catch(reason => { // in case of Confirm.remove() without user answer (not a error)
app.debug.log(reason)
})
}) })
} }

118
tracker/tracker-assist/src/CallWindow.ts
View file

@ -4,7 +4,7 @@ import attachDND from './dnd.js'
const SS_START_TS_KEY = '__openreplay_assist_call_start_ts' const SS_START_TS_KEY = '__openreplay_assist_call_start_ts'
export default class CallWindow { export default class CallWindow {
private iframe: HTMLIFrameElement private readonly iframe: HTMLIFrameElement
private vRemote: HTMLVideoElement | null = null private vRemote: HTMLVideoElement | null = null
private vLocal: HTMLVideoElement | null = null private vLocal: HTMLVideoElement | null = null
private audioBtn: HTMLElement | null = null private audioBtn: HTMLElement | null = null
@ -16,9 +16,9 @@ export default class CallWindow {
private tsInterval: ReturnType<typeof setInterval> private tsInterval: ReturnType<typeof setInterval>
private load: Promise<void> private readonly load: Promise<void>
constructor() { constructor(private readonly logError: (...args: any[]) => void) {
const iframe = this.iframe = document.createElement('iframe') const iframe = this.iframe = document.createElement('iframe')
Object.assign(iframe.style, { Object.assign(iframe.style, {
position: 'fixed', position: 'fixed',
@ -107,8 +107,8 @@ export default class CallWindow {
private adjustIframeSize() { private adjustIframeSize() {
const doc = this.iframe.contentDocument const doc = this.iframe.contentDocument
if (!doc) { return } if (!doc) { return }
this.iframe.style.height = doc.body.scrollHeight + 'px' this.iframe.style.height = `${doc.body.scrollHeight}px`
this.iframe.style.width = doc.body.scrollWidth + 'px' this.iframe.style.width = `${doc.body.scrollWidth}px`
} }
setCallEndAction(endCall: () => void) { setCallEndAction(endCall: () => void) {
@ -116,40 +116,46 @@ export default class CallWindow {
if (this.endCallBtn) { if (this.endCallBtn) {
this.endCallBtn.onclick = endCall this.endCallBtn.onclick = endCall
} }
}) }).catch(e => this.logError(e))
} }
private aRemote: HTMLAudioElement | null = null;
private checkRemoteVideoInterval: ReturnType<typeof setInterval> private checkRemoteVideoInterval: ReturnType<typeof setInterval>
setRemoteStream(rStream: MediaStream) { private audioContainer: HTMLDivElement | null = null
addRemoteStream(rStream: MediaStream) {
this.load.then(() => { this.load.then(() => {
// Video
if (this.vRemote && !this.vRemote.srcObject) { if (this.vRemote && !this.vRemote.srcObject) {
this.vRemote.srcObject = rStream this.vRemote.srcObject = rStream
if (this.vPlaceholder) { if (this.vPlaceholder) {
this.vPlaceholder.innerText = 'Video has been paused. Click anywhere to resume.' this.vPlaceholder.innerText = 'Video has been paused. Click anywhere to resume.'
} }
// Hack to determine if the remote video is enabled
// Hack for audio. Doesen't work inside the iframe because of some magical reasons (check if it is connected to autoplay?) // TODO: pass this info through socket
this.aRemote = document.createElement('audio') if (this.checkRemoteVideoInterval) { clearInterval(this.checkRemoteVideoInterval) } // just in case
this.aRemote.autoplay = true let enabled = false
this.aRemote.style.display = 'none' this.checkRemoteVideoInterval = setInterval(() => {
this.aRemote.srcObject = rStream const settings = rStream.getVideoTracks()[0]?.getSettings()
document.body.appendChild(this.aRemote) const isDummyVideoTrack = !!settings && (settings.width === 2 || settings.frameRate === 0)
const shouldBeEnabled = !isDummyVideoTrack
if (enabled !== shouldBeEnabled) {
this.toggleRemoteVideoUI(enabled=shouldBeEnabled)
}
}, 1000)
} }
// Hack to determine if the remote video is enabled // Audio
if (this.checkRemoteVideoInterval) { clearInterval(this.checkRemoteVideoInterval) } // just in case if (!this.audioContainer) {
let enabled = false this.audioContainer = document.createElement('div')
this.checkRemoteVideoInterval = setInterval(() => { document.body.appendChild(this.audioContainer)
const settings = rStream.getVideoTracks()[0]?.getSettings() }
//console.log(settings) // Hack for audio. Doesen't work inside the iframe
const isDummyVideoTrack = !!settings && (settings.width === 2 || settings.frameRate === 0) // because of some magical reasons (check if it is connected to autoplay?)
const shouldBeEnabled = !isDummyVideoTrack const audioEl = document.createElement('audio')
if (enabled !== shouldBeEnabled) { audioEl.autoplay = true
this.toggleRemoteVideoUI(enabled=shouldBeEnabled) audioEl.style.display = 'none'
} audioEl.srcObject = rStream
}, 1000) this.audioContainer.appendChild(audioEl)
}) }).catch(e => this.logError(e))
} }
toggleRemoteVideoUI(enable: boolean) { toggleRemoteVideoUI(enable: boolean) {
@ -162,26 +168,27 @@ export default class CallWindow {
} }
this.adjustIframeSize() this.adjustIframeSize()
} }
}) }).catch(e => this.logError(e))
} }
private localStream: LocalStream | null = null; private localStreams: LocalStream[] = []
// !TODO: separate streams manipulation from ui
// TODO: on construction? setLocalStreams(streams: LocalStream[]) {
setLocalStream(lStream: LocalStream) { this.localStreams = streams
this.localStream = lStream
} }
playRemote() { playRemote() {
this.vRemote && this.vRemote.play() this.vRemote && this.vRemote.play()
} }
setAssistentName(name: string) { setAssistentName(callingAgents: Map<string, string>) {
this.load.then(() => { this.load.then(() => {
if (this.agentNameElem) { if (this.agentNameElem) {
this.agentNameElem.innerText = name const nameString = Array.from(callingAgents.values()).join(', ')
const safeNames = nameString.length > 20 ? nameString.substring(0, 20) + '...' : nameString
this.agentNameElem.innerText = safeNames
} }
}) }).catch(e => this.logError(e))
} }
@ -195,7 +202,10 @@ export default class CallWindow {
} }
private toggleAudio() { private toggleAudio() {
const enabled = this.localStream?.toggleAudio() || false let enabled = false
this.localStreams.forEach(stream => {
enabled = stream.toggleAudio() || false
})
this.toggleAudioUI(enabled) this.toggleAudioUI(enabled)
} }
@ -211,30 +221,32 @@ export default class CallWindow {
this.adjustIframeSize() this.adjustIframeSize()
} }
private videoRequested = false
private toggleVideo() { private toggleVideo() {
this.localStream?.toggleVideo() this.localStreams.forEach(stream => {
.then(enabled => { stream.toggleVideo()
this.toggleVideoUI(enabled) .then(enabled => {
this.load.then(() => { this.toggleVideoUI(enabled)
if (this.vLocal && this.localStream && !this.vLocal.srcObject) { this.load.then(() => {
this.vLocal.srcObject = this.localStream.stream if (this.vLocal && stream && !this.vLocal.srcObject) {
} this.vLocal.srcObject = stream.stream
}) }
}).catch(e => this.logError(e))
}).catch(e => this.logError(e))
}) })
} }
remove() { remove() {
this.localStream?.stop()
clearInterval(this.tsInterval) clearInterval(this.tsInterval)
clearInterval(this.checkRemoteVideoInterval) clearInterval(this.checkRemoteVideoInterval)
if (this.iframe.parentElement) { if (this.audioContainer && this.audioContainer.parentElement) {
document.body.removeChild(this.iframe) this.audioContainer.parentElement.removeChild(this.audioContainer)
this.audioContainer = null
} }
if (this.aRemote && this.aRemote.parentElement) { if (this.iframe.parentElement) {
document.body.removeChild(this.aRemote) this.iframe.parentElement.removeChild(this.iframe)
} }
sessionStorage.removeItem(SS_START_TS_KEY) sessionStorage.removeItem(SS_START_TS_KEY)
this.localStreams = []
} }
} }

8
tracker/tracker-assist/src/ConfirmWindow/ConfirmWindow.ts
View file

@ -110,17 +110,15 @@ export default class ConfirmWindow {
this.wrapper = wrapper this.wrapper = wrapper
confirmBtn.onclick = () => { confirmBtn.onclick = () => {
this._remove()
this.resolve(true) this.resolve(true)
} }
declineBtn.onclick = () => { declineBtn.onclick = () => {
this._remove()
this.resolve(false) this.resolve(false)
} }
} }
private resolve: (result: boolean) => void = () => {}; private resolve: (result: boolean) => void = () => {};
private reject: () => void = () => {}; private reject: (reason: string) => void = () => {};
mount(): Promise<boolean> { mount(): Promise<boolean> {
document.body.appendChild(this.wrapper) document.body.appendChild(this.wrapper)
@ -135,10 +133,10 @@ export default class ConfirmWindow {
if (!this.wrapper.parentElement) { if (!this.wrapper.parentElement) {
return return
} }
document.body.removeChild(this.wrapper) this.wrapper.parentElement.removeChild(this.wrapper)
} }
remove() { remove() {
this._remove() this._remove()
this.reject() this.reject('no answer')
} }
} }

2
tracker/tracker-assist/src/LocalStream.ts
View file

@ -22,6 +22,7 @@ export default function RequestLocalStream(): Promise<LocalStream> {
return navigator.mediaDevices.getUserMedia({ audio:true, }) return navigator.mediaDevices.getUserMedia({ audio:true, })
.then(aStream => { .then(aStream => {
const aTrack = aStream.getAudioTracks()[0] const aTrack = aStream.getAudioTracks()[0]
if (!aTrack) { throw new Error('No audio tracks provided') } if (!aTrack) { throw new Error('No audio tracks provided') }
return new _LocalStream(aTrack) return new _LocalStream(aTrack)
}) })
@ -54,6 +55,7 @@ class _LocalStream {
}) })
.catch(e => { .catch(e => {
// TODO: log // TODO: log
console.error(e)
return false return false
}) })
} }

8
tracker/tracker-assist/src/Mouse.ts
View file

@ -2,7 +2,7 @@ type XY = [number, number]
export default class Mouse { export default class Mouse {
private mouse: HTMLDivElement private readonly mouse: HTMLDivElement
private position: [number,number] = [0,0,] private position: [number,number] = [0,0,]
constructor() { constructor() {
this.mouse = document.createElement('div') this.mouse = document.createElement('div')
@ -52,8 +52,8 @@ export default class Mouse {
private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct
private lastScrEl: Element | 'window' | null = null private lastScrEl: Element | 'window' | null = null
private resetLastScrEl = () => { this.lastScrEl = null } private readonly resetLastScrEl = () => { this.lastScrEl = null }
private handleWScroll = e => { private readonly handleWScroll = e => {
if (e.target !== this.lastScrEl && if (e.target !== this.lastScrEl &&
this.lastScrEl !== 'window') { this.lastScrEl !== 'window') {
this.resetLastScrEl() this.resetLastScrEl()
@ -111,4 +111,4 @@ export default class Mouse {
window.removeEventListener('scroll', this.handleWScroll) window.removeEventListener('scroll', this.handleWScroll)
window.removeEventListener('resize', this.resetLastScrEl) window.removeEventListener('resize', this.resetLastScrEl)
} }
} }

10
tracker/tracker-assist/src/RemoteControl.ts
View file

@ -23,9 +23,9 @@ export default class RemoteControl {
private agentID: string | null = null private agentID: string | null = null
constructor( constructor(
private options: AssistOptions, private readonly options: AssistOptions,
private onGrand: (sting?) => void, private readonly onGrand: (sting?) => void,
private onRelease: (sting?) => void) {} private readonly onRelease: (sting?) => void) {}
reconnect(ids: string[]) { reconnect(ids: string[]) {
const storedID = sessionStorage.getItem(this.options.session_control_peer_key) const storedID = sessionStorage.getItem(this.options.session_control_peer_key)
@ -56,7 +56,7 @@ export default class RemoteControl {
} else { } else {
this.releaseControl(id) this.releaseControl(id)
} }
}).catch() }).catch(e => console.error(e))
} }
grantControl = (id: string) => { grantControl = (id: string) => {
this.agentID = id this.agentID = id
@ -99,4 +99,4 @@ export default class RemoteControl {
this.focused.innerText = value this.focused.innerText = value
} }
} }
} }