jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Migrate to webrtc (#3051)

Browse source 
* resolved conflicts

* resolved conflicts

* translated comments

* changed console.log message lang

* changed console to logs

* implementing conference call

* add isAgent flag

* add webrtc handlers

* add conference call

* removed conference calls

* fix lint error

---------

Co-authored-by: Andrey Babushkin <a.babushkin@lemon-ai.com>
This commit is contained in:
Andrey Babushkin 2025-02-27 10:12:27 +01:00 • committed by GitHub
parent c793d9d177
commit fd76f7c302
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 15646 additions and 2532 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
frontend/app/components/Assist/ChatWindow/ChatWindow.tsx
View file

@ -9,7 +9,7 @@ import type { LocalStream } from 'Player';
import { PlayerContext } from 'App/components/Session/playerContext'; import { PlayerContext } from 'App/components/Session/playerContext';
export interface Props { export interface Props {
incomeStream: MediaStream[] | null; incomeStream: { stream: MediaStream, isAgent: boolean }[] | null;
localStream: LocalStream | null; localStream: LocalStream | null;
userId: string; userId: string;
isPrestart?: boolean; isPrestart?: boolean;
@ -50,8 +50,8 @@ function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }:
> >
{incomeStream ? ( {incomeStream ? (
incomeStream.map((stream) => ( incomeStream.map((stream) => (
<React.Fragment key={stream.id}> <React.Fragment key={stream.stream.id}>
<VideoContainer stream={stream} setRemoteEnabled={setRemoteEnabled} /> <VideoContainer stream={stream.stream} setRemoteEnabled={setRemoteEnabled} isAgent={stream.isAgent} />
</React.Fragment> </React.Fragment>
)) ))
) : ( ) : (
@ -62,6 +62,7 @@ function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }:
stream={localStream ? localStream.stream : null} stream={localStream ? localStream.stream : null}
muted muted
height={anyRemoteEnabled ? 50 : 'unset'} height={anyRemoteEnabled ? 50 : 'unset'}
local
/> />
</div> </div>
</div> </div>

28
frontend/app/components/Assist/components/AssistActions/AssistActions.tsx
View file

@ -80,7 +80,7 @@ function AssistActions({
} = store.get(); } = store.get();
const [isPrestart, setPrestart] = useState(false); const [isPrestart, setPrestart] = useState(false);
const [incomeStream, setIncomeStream] = useState<MediaStream[] | null>([]); const [incomeStream, setIncomeStream] = useState<{ stream: MediaStream; isAgent: boolean }[] | null>([]);
const [localStream, setLocalStream] = useState<LocalStream | null>(null); const [localStream, setLocalStream] = useState<LocalStream | null>(null);
const [callObject, setCallObject] = useState<{ end: () => void } | null>(null); const [callObject, setCallObject] = useState<{ end: () => void } | null>(null);
@ -131,18 +131,25 @@ function AssistActions({
} }
}, [peerConnectionStatus]); }, [peerConnectionStatus]);
const addIncomeStream = (stream: MediaStream) => { const addIncomeStream = (stream: MediaStream, isAgent: boolean) => {
setIncomeStream((oldState) => { setIncomeStream((oldState) => {
if (oldState === null) return [stream]; if (oldState === null) return [{ stream, isAgent }];
if (!oldState.find((existingStream) => existingStream.id === stream.id)) { if (!oldState.find((existingStream) => existingStream.stream.id === stream.id)) {
audioContextManager.mergeAudioStreams(stream); audioContextManager.mergeAudioStreams(stream);
return [...oldState, stream]; return [...oldState, { stream, isAgent }];
} }
return oldState; return oldState;
}); });
}; };
function call(additionalAgentIds?: string[]) { const removeIncomeStream = (stream: MediaStream) => {
setIncomeStream((prevState) => {
if (!prevState) return [];
return prevState.filter((existingStream) => existingStream.stream.id !== stream.id);
});
};
function call() {
RequestLocalStream() RequestLocalStream()
.then((lStream) => { .then((lStream) => {
setLocalStream(lStream); setLocalStream(lStream);
@ -152,15 +159,16 @@ function AssistActions({
addIncomeStream, addIncomeStream,
() => { () => {
player.assistManager.ping(AssistActionsPing.call.end, agentId) player.assistManager.ping(AssistActionsPing.call.end, agentId)
lStream.stop.bind(lStream); lStream.stop.apply(lStream);
removeIncomeStream(lStream.stream);
}, },
onReject, onReject,
onError onError
); );
setCallObject(callPeer()); setCallObject(callPeer());
if (additionalAgentIds) { // if (additionalAgentIds) {
callPeer(additionalAgentIds); // callPeer(additionalAgentIds);
} // }
}) })
.catch(onError); .catch(onError);
} }

25
frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx
View file

@ -5,9 +5,18 @@ interface Props {
muted?: boolean; muted?: boolean;
height?: number | string; height?: number | string;
setRemoteEnabled?: (isEnabled: boolean) => void; setRemoteEnabled?: (isEnabled: boolean) => void;
local?: boolean;
isAgent?: boolean;
} }
function VideoContainer({ stream, muted = false, height = 280, setRemoteEnabled }: Props) { function VideoContainer({
stream,
muted = false,
height = 280,
setRemoteEnabled,
local,
isAgent,
}: Props) {
const ref = useRef<HTMLVideoElement>(null); const ref = useRef<HTMLVideoElement>(null);
const [isEnabled, setEnabled] = React.useState(false); const [isEnabled, setEnabled] = React.useState(false);
@ -15,14 +24,14 @@ function VideoContainer({ stream, muted = false, height = 280, setRemoteEnabled
if (ref.current) { if (ref.current) {
ref.current.srcObject = stream; ref.current.srcObject = stream;
} }
}, [ref.current, stream, stream.getVideoTracks()[0]?.getSettings().width]); }, [ref.current, stream, stream?.getVideoTracks()[0]?.getSettings().width]);
useEffect(() => { useEffect(() => {
if (!stream) { if (!stream) {
return; return;
} }
const iid = setInterval(() => { const iid = setInterval(() => {
const track = stream.getVideoTracks()[0] const track = stream.getVideoTracks()[0];
const settings = track?.getSettings(); const settings = track?.getSettings();
const isDummyVideoTrack = settings const isDummyVideoTrack = settings
? settings.width === 2 || ? settings.width === 2 ||
@ -47,9 +56,19 @@ function VideoContainer({ stream, muted = false, height = 280, setRemoteEnabled
width: isEnabled ? undefined : '0px!important', width: isEnabled ? undefined : '0px!important',
height: isEnabled ? undefined : '0px!important', height: isEnabled ? undefined : '0px!important',
border: '1px solid grey', border: '1px solid grey',
transform: local ? 'scaleX(-1)' : undefined,
}} }}
> >
<video autoPlay ref={ref} muted={muted} style={{ height: height }} /> <video autoPlay ref={ref} muted={muted} style={{ height: height }} />
{isAgent ? (
<div
style={{
position: 'absolute',
}}
>
Agent
</div>
) : null}
</div> </div>
); );
} }

2
frontend/app/components/Session/Player/LivePlayer/LivePlayerBlockHeader.tsx
View file

@ -87,7 +87,7 @@ function LivePlayerBlockHeader({
</div> </div>
)} )}
<AssistActions userId={userId} isCallActive={isCallActive} agentIds={agentIds} /> <AssistActions userId={userId} isCallActive={isCallActive} agentIds={agentIds ?? []} />
</div> </div>
</div> </div>
</div> </div>

1
frontend/app/player/web/WebLivePlayer.ts
View file

@ -39,6 +39,7 @@ export default class WebLivePlayer extends WebPlayer {
config, config,
wpState, wpState,
(id) => this.messageManager.getNode(id), (id) => this.messageManager.getNode(id),
agentId,
uiErrorHandler uiErrorHandler
); );
this.assistManager.connect(session.agentToken!, agentId, projectId); this.assistManager.connect(session.agentToken!, agentId, projectId);

25
frontend/app/player/web/assist/AssistManager.ts
View file

@ -67,6 +67,7 @@ export default class AssistManager {
...RemoteControl.INITIAL_STATE, ...RemoteControl.INITIAL_STATE,
...ScreenRecording.INITIAL_STATE, ...ScreenRecording.INITIAL_STATE,
}; };
private agentIds: string[] = [];
// TODO: Session type // TODO: Session type
constructor( constructor(
@ -77,9 +78,10 @@ export default class AssistManager {
private config: RTCIceServer[] | null, private config: RTCIceServer[] | null,
private store: Store<typeof AssistManager.INITIAL_STATE>, private store: Store<typeof AssistManager.INITIAL_STATE>,
private getNode: MessageManager['getNode'], private getNode: MessageManager['getNode'],
public readonly agentId: number,
public readonly uiErrorHandler?: { public readonly uiErrorHandler?: {
error: (msg: string) => void; error: (msg: string) => void;
} },
) {} ) {}
public getAssistVersion = () => this.assistVersion; public getAssistVersion = () => this.assistVersion;
@ -192,6 +194,12 @@ export default class AssistManager {
}), }),
}, },
})); }));
// socket.onAny((event, ...args) => {
// logger.log(`📩 Socket: ${event}`, args);
// });
socket.on('connect', () => { socket.on('connect', () => {
waitingForMessages = true; waitingForMessages = true;
// TODO: reconnect happens frequently on bad network // TODO: reconnect happens frequently on bad network
@ -268,6 +276,10 @@ export default class AssistManager {
} }
} }
} }
if (data.agentIds) {
const filteredAgentIds = this.agentIds.filter((id: string) => id.split('-')[3] !== agentId.toString());
this.agentIds = filteredAgentIds;
}
}); });
socket.on('SESSION_DISCONNECTED', (e) => { socket.on('SESSION_DISCONNECTED', (e) => {
waitingForMessages = true; waitingForMessages = true;
@ -288,7 +300,12 @@ export default class AssistManager {
socket, socket,
this.config, this.config,
this.peerID, this.peerID,
this.getAssistVersion this.getAssistVersion,
{
...this.session.agentInfo,
id: agentId,
},
this.agentIds,
); );
this.remoteControl = new RemoteControl( this.remoteControl = new RemoteControl(
this.store, this.store,
@ -309,7 +326,7 @@ export default class AssistManager {
this.canvasReceiver = new CanvasReceiver(this.peerID, this.config, this.getNode, { this.canvasReceiver = new CanvasReceiver(this.peerID, this.config, this.getNode, {
...this.session.agentInfo, ...this.session.agentInfo,
id: agentId, id: agentId,
}); }, socket);
document.addEventListener('visibilitychange', this.onVisChange); document.addEventListener('visibilitychange', this.onVisChange);
}); });
@ -318,7 +335,7 @@ export default class AssistManager {
/** /**
* Sends event ping to stats service * Sends event ping to stats service
* */ * */
public ping(event: StatsEvent, id: number) { public ping(event: StatsEvent, id: string) {
this.socket?.emit(event, id); this.socket?.emit(event, id);
} }

479
frontend/app/player/web/assist/Call.ts
View file

@ -1,10 +1,8 @@
import type Peer from 'peerjs';
import type { MediaConnection } from 'peerjs';
import type { LocalStream } from './LocalStream'; import type { LocalStream } from './LocalStream';
import type { Socket } from './types'; import type { Socket } from './types';
import type { Store } from '../../common/types'; import type { Store } from '../../common/types';
import { userStore } from "App/mstore"; import { userStore } from "App/mstore";
import logger from '@/logger';
export enum CallingState { export enum CallingState {
NoCall, NoCall,
@ -19,28 +17,59 @@ export interface State {
currentTab?: string; currentTab?: string;
} }
const WEBRTC_CALL_AGENT_EVENT_TYPES = {
OFFER: 'offer',
ANSWER: 'answer',
ICE_CANDIDATE: 'ice-candidate',
}
export default class Call { export default class Call {
private assistVersion = 1; private assistVersion = 1;
static readonly INITIAL_STATE: Readonly<State> = { static readonly INITIAL_STATE: Readonly<State> = {
calling: CallingState.NoCall, calling: CallingState.NoCall,
}; };
private _peer: Peer | null = null; private connections: Record<string, RTCPeerConnection> = {};
private connectionAttempts: number = 0; private connectAttempts = 0;
private callConnection: MediaConnection[] = [];
private videoStreams: Record<string, MediaStreamTrack> = {}; private videoStreams: Record<string, MediaStreamTrack> = {};
private callID: string;
private agentInCallIds: string[] = [];
constructor( constructor(
private store: Store<State & { tabs: Set<string> }>, private store: Store<State & { tabs: Set<string> }>,
private socket: Socket, private socket: Socket,
private config: RTCIceServer[] | null, private config: RTCIceServer[] | null,
private peerID: string, private peerID: string,
private getAssistVersion: () => number private getAssistVersion: () => number,
private agent: Record<string, any>,
private agentIds: string[],
) { ) {
socket.on('WEBRTC_AGENT_CALL', (data) => {
switch (data.type) {
case WEBRTC_CALL_AGENT_EVENT_TYPES.OFFER:
this.handleOffer(data, true);
break;
case WEBRTC_CALL_AGENT_EVENT_TYPES.ICE_CANDIDATE:
this.handleIceCandidate(data);
break;
case WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER:
this.handleAnswer(data, true);
default:
break;
}
})
socket.on('UPDATE_SESSION', (data: { data: { agentIds: string[] }}) => {
this.callAgentsInSession({ agentIds: data.data.agentIds });
});
socket.on('call_end', () => { socket.on('call_end', () => {
this.onRemoteCallEnd() this.onRemoteCallEnd()
}); });
socket.on('videofeed', ({ streamId, enabled }) => {
socket.on('videofeed', (data: { data: { streamId: string; enabled: boolean }}) => {
const { streamId, enabled } = data.data;
if (this.videoStreams[streamId]) { if (this.videoStreams[streamId]) {
this.videoStreams[streamId].enabled = enabled; this.videoStreams[streamId].enabled = enabled;
} }
@ -56,14 +85,13 @@ export default class Call {
}); });
socket.on('messages_gz', () => { socket.on('messages_gz', () => {
if (reconnecting) { if (reconnecting) {
// 'messages' come frequently, so it is better to have Reconnecting // When the connection is restored, we initiate a re-creation of the connection
this._callSessionPeer(); this._callSessionPeer();
reconnecting = false; reconnecting = false;
} }
}) })
socket.on('messages', () => { socket.on('messages', () => {
if (reconnecting) { if (reconnecting) {
// 'messages' come frequently, so it is better to have Reconnecting
this._callSessionPeer(); this._callSessionPeer();
reconnecting = false; reconnecting = false;
} }
@ -71,111 +99,239 @@ export default class Call {
socket.on('disconnect', () => { socket.on('disconnect', () => {
this.store.update({ calling: CallingState.NoCall }); this.store.update({ calling: CallingState.NoCall });
}); });
socket.on('webrtc_call_offer', (data: { data: { from: string, offer: RTCSessionDescriptionInit } }) => {
this.handleOffer(data.data);
});
socket.on('webrtc_call_answer', (data: { data: { from: string, answer: RTCSessionDescriptionInit } }) => {
this.handleAnswer(data.data);
});
socket.on('webrtc_call_ice_candidate', (data: { data: { from: string, candidate: RTCIceCandidateInit } }) => {
this.handleIceCandidate({ candidate: data.data.candidate, from: data.data.from });
});
this.assistVersion = this.getAssistVersion(); this.assistVersion = this.getAssistVersion();
} }
private getPeer(): Promise<Peer> { // CREATE A LOCAL PEER
if (this._peer && !this._peer.disconnected) { private async createPeerConnection({ remotePeerId, localPeerId, isAgent }: { remotePeerId: string, isAgent?: boolean, localPeerId?: string }): Promise<RTCPeerConnection> {
return Promise.resolve(this._peer); // create pc with ice config
const pc = new RTCPeerConnection({
iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
});
// If there is a local stream, add its tracks to the connection
if (this.callArgs && this.callArgs.localStream && this.callArgs.localStream.stream) {
this.callArgs.localStream.stream.getTracks().forEach((track) => {
pc.addTrack(track, this.callArgs!.localStream.stream);
});
} }
// @ts-ignore // when ice is ready we send it
const urlObject = new URL(window.env.API_EDP || window.location.origin); pc.onicecandidate = (event) => {
if (event.candidate) {
// @ts-ignore TODO: set module in ts settings if (isAgent) {
return import('peerjs').then(({ default: Peer }) => { this.socket.emit('WEBRTC_AGENT_CALL', { from: localPeerId, candidate: event.candidate, toAgentId: getSocketIdByCallId(remotePeerId), type: WEBRTC_CALL_AGENT_EVENT_TYPES.ICE_CANDIDATE });
if (this.cleaned) { } else {
return Promise.reject('Already cleaned'); this.socket.emit('webrtc_call_ice_candidate', { from: remotePeerId, candidate: event.candidate });
}
const peerOpts: Peer.PeerJSOption = {
host: urlObject.hostname,
path: '/assist',
port:
urlObject.port === ''
? location.protocol === 'https:'
? 443
: 80
: parseInt(urlObject.port),
};
if (this.config) {
peerOpts['config'] = {
iceServers: this.config,
//@ts-ignore
sdpSemantics: 'unified-plan',
iceTransportPolicy: 'all',
};
}
const peer = (this._peer = new Peer(peerOpts));
peer.on('call', (call) => {
console.log('getting call from', call.peer);
call.answer(this.callArgs?.localStream.stream);
this.callConnection.push(call);
this.callArgs?.localStream.onVideoTrack((vTrack) => {
const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video');
if (!sender) {
console.warn('No video sender found');
return;
}
sender.replaceTrack(vTrack);
});
call.on('stream', (stream) => {
this.videoStreams[call.peer] = stream.getVideoTracks()[0];
this.callArgs && this.callArgs.onStream(stream);
});
call.on('close', this.onRemoteCallEnd);
call.on('error', (e) => {
console.error('PeerJS error (on call):', e);
this.initiateCallEnd();
this.callArgs && this.callArgs.onError && this.callArgs.onError();
});
});
peer.on('error', (e) => {
if (e.type === 'disconnected') {
return peer.reconnect();
} else if (e.type !== 'peer-unavailable') {
console.error(`PeerJS error (on peer). Type ${e.type}`, e);
} }
}); } else {
logger.log("ICE candidate gathering complete");
}
};
return new Promise((resolve) => { // when we receive a remote track, we write it to videoStreams[peerId]
peer.on('open', () => resolve(peer)); pc.ontrack = (event) => {
const stream = event.streams[0];
if (stream && !this.videoStreams[remotePeerId]) {
const clonnedStream = stream.clone();
this.videoStreams[remotePeerId] = clonnedStream.getVideoTracks()[0];
if (this.store.get().calling !== CallingState.OnCall) {
this.store.update({ calling: CallingState.OnCall });
}
if (this.callArgs) {
this.callArgs.onStream(stream, remotePeerId !== this.callID && isAgentId(remotePeerId));
}
}
};
// If the connection is lost, we end the call
pc.onconnectionstatechange = () => {
if (pc.connectionState === "disconnected" || pc.connectionState === "failed") {
this.onRemoteCallEnd();
}
};
// Handle track replacement when local video changes
if (this.callArgs && this.callArgs.localStream) {
this.callArgs.localStream.onVideoTrack((vTrack: MediaStreamTrack) => {
const sender = pc.getSenders().find((s) => s.track?.kind === 'video');
if (!sender) {
logger.warn('No video sender found');
return;
}
sender.replaceTrack(vTrack);
}); });
}); }
return pc;
} }
// ESTABLISHING A CONNECTION
private async _peerConnection({ remotePeerId, isAgent, socketId, localPeerId }: { remotePeerId: string, isAgent?: boolean, socketId?: string, localPeerId?: string }) {
try {
// Create RTCPeerConnection with client
const pc = await this.createPeerConnection({ remotePeerId, localPeerId, isAgent });
this.connections[remotePeerId] = pc;
// Create an SDP offer
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
// Sending offer
if (isAgent) {
this.socket.emit('WEBRTC_AGENT_CALL', { from: localPeerId, offer, toAgentId: socketId, type: WEBRTC_CALL_AGENT_EVENT_TYPES.OFFER });
} else {
this.socket.emit('webrtc_call_offer', { from: remotePeerId, offer });
}
this.connectAttempts = 0;
} catch (e: any) {
logger.error(e);
// Trying to reconnect
const tryReconnect = async (error: any) => {
if (error.type === 'peer-unavailable' && this.connectAttempts < 5) {
this.connectAttempts++;
logger.log('reconnecting', this.connectAttempts);
await new Promise((resolve) => setTimeout(resolve, 250));
await this._peerConnection({ remotePeerId });
} else {
logger.log('error', this.connectAttempts);
this.callArgs?.onError?.('Could not establish a connection with the peer after 5 attempts');
}
};
await tryReconnect(e);
}
}
// Process the received offer to answer
private async handleOffer(data: { from: string, offer: RTCSessionDescriptionInit }, isAgent?: boolean) {
// set to remotePeerId data.from
logger.log("RECEIVED OFFER", data);
const fromCallId = data.from;
let pc = this.connections[fromCallId];
if (!pc) {
if (isAgent) {
this.connections[fromCallId] = await this.createPeerConnection({ remotePeerId: fromCallId, isAgent, localPeerId: this.callID });
pc = this.connections[fromCallId];
} else {
logger.error("No connection found for remote peer", fromCallId);
return;
}
}
try {
// if the connection is not established yet, then set remoteDescription to peer
if (!pc.localDescription) {
await pc.setRemoteDescription(new RTCSessionDescription(data.offer));
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
if (isAgent) {
this.socket.emit('WEBRTC_AGENT_CALL', { from: this.callID, answer, toAgentId: getSocketIdByCallId(fromCallId), type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER });
} else {
this.socket.emit('webrtc_call_answer', { from: fromCallId, answer });
}
} else {
logger.warn("Skipping setRemoteDescription: Already in stable state");
}
} catch (e) {
logger.error("Error setting remote description from answer", e);
this.callArgs?.onError?.(e);
}
}
// Process the received answer to offer
private async handleAnswer(data: { from: string, answer: RTCSessionDescriptionInit }, isAgent?: boolean) {
// set to remotePeerId data.from
logger.log("RECEIVED ANSWER", data);
if (this.agentInCallIds.includes(data.from) && !isAgent) {
return;
}
const callId = data.from;
const pc = this.connections[callId];
if (!pc) {
logger.error("No connection found for remote peer", callId, this.connections);
return;
}
try {
// if the connection is not established yet, then set remoteDescription to peer
if (pc.signalingState !== "stable") {
await pc.setRemoteDescription(new RTCSessionDescription(data.answer));
} else {
logger.warn("Skipping setRemoteDescription: Already in stable state");
}
} catch (e) {
logger.error("Error setting remote description from answer", e);
this.callArgs?.onError?.(e);
}
}
// process the received iceCandidate
private async handleIceCandidate(data: { from: string, candidate: RTCIceCandidateInit }) {
const callId = data.from;
const pc = this.connections[callId];
if (!pc) return;
// if there are ice candidates then add candidate to peer
if (data.candidate && (data.candidate.sdpMid || data.candidate.sdpMLineIndex !== null)) {
try {
await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
} catch (e) {
logger.error("Error adding ICE candidate", e);
}
} else {
logger.warn("Invalid ICE candidate skipped:", data.candidate);
}
}
// handle call ends
private handleCallEnd() { private handleCallEnd() {
if (this.store.get().calling !== CallingState.NoCall) this.callArgs && this.callArgs.onCallEnd(); // If the call is not completed, then call onCallEnd
if (this.store.get().calling !== CallingState.NoCall) {
this.callArgs && this.callArgs.onCallEnd();
}
// change state to NoCall
this.store.update({ calling: CallingState.NoCall }); this.store.update({ calling: CallingState.NoCall });
this.callConnection[0] && this.callConnection[0].close(); // Close all created RTCPeerConnection
Object.values(this.connections).forEach((pc) => pc.close());
this.callArgs?.onCallEnd();
// Clear connections
this.connections = {};
this.callArgs = null;
this.videoStreams = {};
this.callArgs = null; this.callArgs = null;
// TODO: We have it separated, right? (check)
//this.toggleAnnotation(false)
} }
// Call completion event handler by signal
private onRemoteCallEnd = () => { private onRemoteCallEnd = () => {
if ([CallingState.Requesting, CallingState.Connecting].includes(this.store.get().calling)) { if ([CallingState.Requesting, CallingState.Connecting].includes(this.store.get().calling)) {
// If the call has not started yet, then call onReject
this.callArgs && this.callArgs.onReject(); this.callArgs && this.callArgs.onReject();
this.callConnection[0] && this.callConnection[0].close(); // Close all connections and reset callArgs
Object.values(this.connections).forEach((pc) => pc.close());
this.connections = {};
this.callArgs?.onCallEnd();
this.store.update({ calling: CallingState.NoCall }); this.store.update({ calling: CallingState.NoCall });
this.callArgs = null; this.callArgs = null;
} else { } else {
// Call the full call completion handler
this.handleCallEnd(); this.handleCallEnd();
} }
}; };
// Ends the call and sends the call_end signal
initiateCallEnd = async () => { initiateCallEnd = async () => {
const userName = userStore.account.name; this.emitData('call_end', this.callID);
this.emitData('call_end', userName);
this.handleCallEnd(); this.handleCallEnd();
// TODO: We have it separated, right? (check)
// const remoteControl = this.store.get().remoteControl
// if (remoteControl === RemoteControlStatus.Enabled) {
// this.socket.emit("release_control")
// this.toggleRemoteControl(false)
// }
}; };
private emitData = (event: string, data?: any) => { private emitData = (event: string, data?: any) => {
@ -188,7 +344,7 @@ export default class Call {
private callArgs: { private callArgs: {
localStream: LocalStream; localStream: LocalStream;
onStream: (s: MediaStream) => void; onStream: (s: MediaStream, isAgent: boolean) => void;
onCallEnd: () => void; onCallEnd: () => void;
onReject: () => void; onReject: () => void;
onError?: (arg?: any) => void; onError?: (arg?: any) => void;
@ -196,7 +352,7 @@ export default class Call {
setCallArgs( setCallArgs(
localStream: LocalStream, localStream: LocalStream,
onStream: (s: MediaStream) => void, onStream: (s: MediaStream, isAgent: boolean) => void,
onCallEnd: () => void, onCallEnd: () => void,
onReject: () => void, onReject: () => void,
onError?: (e?: any) => void onError?: (e?: any) => void
@ -210,117 +366,88 @@ export default class Call {
}; };
} }
call(thirdPartyPeers?: string[]): { end: () => void } { // Initiates a call
if (thirdPartyPeers && thirdPartyPeers.length > 0) { call(): { end: () => void } {
this.addPeerCall(thirdPartyPeers); this._callSessionPeer();
} else { // this.callAgentsInSession({ agentIds: this.agentInCallIds });
this._callSessionPeer();
}
return { return {
end: this.initiateCallEnd, end: this.initiateCallEnd,
}; };
} }
// Notify peers of local video state change
toggleVideoLocalStream(enabled: boolean) { toggleVideoLocalStream(enabled: boolean) {
this.getPeer().then((peer) => { this.emitData('videofeed', { streamId: this.callID, enabled });
this.emitData('videofeed', { streamId: peer.id, enabled });
});
} }
/** Connecting to the other agents that are already // Calls the method to create a connection with a peer
* in the call with the user
*/
addPeerCall(thirdPartyPeers: string[]) {
thirdPartyPeers.forEach((peer) => this._peerConnection(peer));
}
/** Connecting to the app user */
private _callSessionPeer() { private _callSessionPeer() {
if (![CallingState.NoCall, CallingState.Reconnecting].includes(this.store.get().calling)) { if (![CallingState.NoCall, CallingState.Reconnecting].includes(this.store.get().calling)) {
return; return;
} }
this.store.update({ calling: CallingState.Connecting }); this.store.update({ calling: CallingState.Connecting });
const tab = this.store.get().currentTab; const tab = this.store.get().currentTab;
if (!this.store.get().currentTab) { if (!tab) {
console.warn('No tab data to connect to peer'); logger.warn('No tab data to connect to peer');
} }
const peerId =
this.getAssistVersion() === 1
? this.peerID
: `${this.peerID}-${tab || Object.keys(this.store.get().tabs)[0]}`;
// Generate a peer identifier depending on the assist version
this.callID = this.getCallId();
const userName = userStore.account.name; const userName = userStore.account.name;
this.emitData('_agent_name', userName); this.emitData('_agent_name', userName);
void this._peerConnection(peerId); void this._peerConnection({ remotePeerId: this.callID });
} }
connectAttempts = 0; private callAgentsInSession({ agentIds }: { agentIds: string[] }) {
private async _peerConnection(remotePeerId: string) { if (agentIds) {
try { const filteredAgentIds = agentIds.filter((id: string) => id.split('-')[3] !== this.agent.id.toString());
const peer = await this.getPeer(); const newIds = filteredAgentIds.filter((id: string) => !this.agentInCallIds.includes(id));
// let canCall = false const removedIds = this.agentInCallIds.filter((id: string) => !filteredAgentIds.includes(id));
removedIds.forEach((id: string) => this.agentDisconnected(id));
const tryReconnect = async (e: any) => { if (this.store.get().calling === CallingState.OnCall) {
peer.off('error', tryReconnect) newIds.forEach((id: string) => {
console.log(e.type, this.connectAttempts); const socketId = getSocketIdByCallId(id);
if (e.type === 'peer-unavailable' && this.connectAttempts < 5) { this._peerConnection({ remotePeerId: id, isAgent: true, socketId, localPeerId: this.callID });
this.connectAttempts++;
console.log('reconnecting', this.connectAttempts);
await new Promise((resolve) => setTimeout(resolve, 250));
await this._peerConnection(remotePeerId);
} else {
console.log('error', this.connectAttempts);
this.callArgs?.onError?.('Could not establish a connection with the peer after 5 attempts');
}
}
const call = peer.call(remotePeerId, this.callArgs!.localStream.stream);
peer.on('error', tryReconnect);
peer.on('connection', () => {
this.callConnection.push(call);
this.connectAttempts = 0;
this.callArgs?.localStream.onVideoTrack((vTrack) => {
const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video');
if (!sender) {
console.warn('No video sender found');
return;
}
sender.replaceTrack(vTrack);
}); });
}) }
call.on('stream', (stream) => { this.agentInCallIds = filteredAgentIds;
this.store.get().calling !== CallingState.OnCall &&
this.store.update({ calling: CallingState.OnCall });
this.videoStreams[call.peer] = stream.getVideoTracks()[0];
this.callArgs && this.callArgs.onStream(stream);
});
call.on('close', this.onRemoteCallEnd);
call.on('error', (e) => {
console.error('PeerJS error (on call):', e);
this.initiateCallEnd();
this.callArgs && this.callArgs.onError && this.callArgs.onError();
});
} catch (e) {
console.error(e);
} }
} }
private cleaned: boolean = false; private getCallId() {
const tab = this.store.get().currentTab;
clean() { if (!tab) {
this.cleaned = true; // sometimes cleaned before modules loaded logger.warn('No tab data to connect to peer');
void this.initiateCallEnd();
if (this._peer) {
console.log('destroying peer...');
const peer = this._peer; // otherwise it calls reconnection on data chan close
this._peer = null;
peer.disconnect();
peer.destroy();
} }
// Generate a peer identifier depending on the assist version
return `${this.peerID}-${tab || Array.from(this.store.get().tabs)[0]}-${this.agent.id}-${this.socket.id}-agent`;
}
agentDisconnected(agentId: string) {
this.connections[agentId]?.close();
delete this.connections[agentId];
}
// Method for clearing resources
clean() {
void this.initiateCallEnd();
Object.values(this.connections).forEach((pc) => pc.close());
this.connections = {};
this.callArgs?.onCallEnd();
}
}
function isAgentId(id: string): boolean {
return id.endsWith('_agent');
}
function getSocketIdByCallId(callId?: string): string | undefined {
const socketIdRegex = /-\d{2}-(.*?)\-agent/;
const match = callId?.match(socketIdRegex);
if (match) {
return match[1];
} }
} }

124
frontend/app/player/web/assist/CanvasReceiver.ts
View file

@ -1,6 +1,6 @@
import Peer from 'peerjs';
import { VElement } from 'Player/web/managers/DOM/VirtualDOM'; import { VElement } from 'Player/web/managers/DOM/VirtualDOM';
import MessageManager from 'Player/web/MessageManager'; import MessageManager from 'Player/web/MessageManager';
import { Socket } from 'socket.io-client';
let frameCounter = 0; let frameCounter = 0;
@ -18,70 +18,102 @@ function draw(
export default class CanvasReceiver { export default class CanvasReceiver {
private streams: Map<string, MediaStream> = new Map(); private streams: Map<string, MediaStream> = new Map();
private peer: Peer | null = null; // Store RTCPeerConnection for each remote peer
private connections: Map<string, RTCPeerConnection> = new Map();
private cId: string;
// sendSignal – for sending signals (offer/answer/ICE)
constructor( constructor(
private readonly peerIdPrefix: string, private readonly peerIdPrefix: string,
private readonly config: RTCIceServer[] | null, private readonly config: RTCIceServer[] | null,
private readonly getNode: MessageManager['getNode'], private readonly getNode: MessageManager['getNode'],
private readonly agentInfo: Record<string, any> private readonly agentInfo: Record<string, any>,
private readonly socket: Socket,
) { ) {
// @ts-ignore // Form an id like in PeerJS
const urlObject = new URL(window.env.API_EDP || window.location.origin); this.cId = `${this.peerIdPrefix}-${this.agentInfo.id}-canvas`;
const peerOpts: Peer.PeerJSOption = {
host: urlObject.hostname, this.socket.on('webrtc_canvas_offer', (data: { data: { offer: RTCSessionDescriptionInit, id: string }}) => {
path: '/assist', const { offer, id } = data.data;
port: if (checkId(id, this.cId)) {
urlObject.port === '' this.handleOffer(offer, id);
? location.protocol === 'https:' }
? 443 });
: 80
: parseInt(urlObject.port), this.socket.on('webrtc_canvas_ice_candidate', (data: { data: { candidate: RTCIceCandidateInit, id: string }}) => {
const {candidate, id } = data.data;
if (checkId(id, this.cId)) {
this.handleCandidate(candidate, id);
}
});
this.socket.on('webrtc_canvas_restart', () => {
this.clear();
});
}
async handleOffer(offer: RTCSessionDescriptionInit, id: string): Promise<void> {
const pc = new RTCPeerConnection({
iceServers: this.config ? this.config : [{ urls: "stun:stun.l.google.com:19302" }],
});
// Save the connection
this.connections.set(id, pc);
pc.onicecandidate = (event) => {
if (event.candidate) {
this.socket.emit('webrtc_canvas_ice_candidate', ({ candidate: event.candidate, id }));
}
}; };
if (this.config) {
peerOpts['config'] = { pc.ontrack = (event) => {
iceServers: this.config,
//@ts-ignore const stream = event.streams[0];
sdpSemantics: 'unified-plan', if (stream) {
iceTransportPolicy: 'all', // Detect canvasId from remote peer id
}; const canvasId = id.split('-')[4];
}
const id = `${this.peerIdPrefix}-${this.agentInfo.id}-canvas`;
const canvasPeer = new Peer(id, peerOpts);
this.peer = canvasPeer;
canvasPeer.on('error', (err) => console.error('canvas peer error', err));
canvasPeer.on('call', (call) => {
call.answer();
const canvasId = call.peer.split('-')[2];
call.on('stream', (stream) => {
this.streams.set(canvasId, stream); this.streams.set(canvasId, stream);
setTimeout(() => { setTimeout(() => {
const node = this.getNode(parseInt(canvasId, 10)); const node = this.getNode(parseInt(canvasId, 10));
const videoEl = spawnVideo( const videoEl = spawnVideo(stream.clone() as MediaStream, node as VElement);
this.streams.get(canvasId)?.clone() as MediaStream,
node as VElement
);
if (node) { if (node) {
draw( draw(
videoEl, videoEl,
node.node as HTMLCanvasElement, node.node as HTMLCanvasElement,
(node.node as HTMLCanvasElement).getContext('2d') as CanvasRenderingContext2D (node.node as HTMLCanvasElement).getContext('2d') as CanvasRenderingContext2D
); );
} else {
console.log('NODE', canvasId, 'IS NOT FOUND');
} }
}, 250); }, 250);
}); }
call.on('error', (err) => console.error('canvas call error', err)); };
});
await pc.setRemoteDescription(new RTCSessionDescription(offer));
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
this.socket.emit('webrtc_canvas_answer', { answer: answer, id });
}
async handleCandidate(candidate: RTCIceCandidateInit, id: string): Promise<void> {
const pc = this.connections.get(id);
if (pc) {
try {
await pc.addIceCandidate(new RTCIceCandidate(candidate));
} catch (e) {
console.error('Error adding ICE candidate', e);
}
}
} }
clear() { clear() {
if (this.peer) { this.connections.forEach((pc) => {
// otherwise it calls reconnection on data chan close pc.close();
const peer = this.peer; });
this.peer = null; this.connections.clear();
peer.disconnect(); this.streams.clear();
peer.destroy();
}
} }
} }
@ -157,6 +189,10 @@ function spawnDebugVideo(stream: MediaStream, node: VElement) {
}); });
} }
function checkId(id: string, cId: string): boolean {
return id.includes(cId);
}
/** simple peer example /** simple peer example
* // @ts-ignore * // @ts-ignore
* const peer = new SLPeer({ initiator: false }) * const peer = new SLPeer({ initiator: false })

1
frontend/package.json
View file

@ -51,7 +51,6 @@
"mobx": "^6.13.3", "mobx": "^6.13.3",
"mobx-persist-store": "^1.1.5", "mobx-persist-store": "^1.1.5",
"mobx-react-lite": "^4.0.7", "mobx-react-lite": "^4.0.7",
"peerjs": "1.3.2",
"prismjs": "^1.29.0", "prismjs": "^1.29.0",
"rc-time-picker": "^3.7.3", "rc-time-picker": "^3.7.3",
"react": "^18.2.0", "react": "^18.2.0",

3375
frontend/yarn.lock
View file

File diff suppressed because it is too large Load diff

BIN
tracker/tracker-assist/.yarn/install-state.gz
View file

Binary file not shown.

933
tracker/tracker-assist/layout/index-chat.html
View file

@ -1,490 +1,545 @@
<!doctype html> <!doctype html>
<html lang="en"> <html lang="en">
<head>
<!-- Required meta tags -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>OpenReplay | Assist</title>
<!--CSS --> <head>
<!-- <link href="css/styles.css" rel="stylesheet"> --> <!-- Required meta tags -->
<style> <meta charset="utf-8">
body { <meta name="viewport" content="width=device-width, initial-scale=1">
margin: 0; <title>OpenReplay | Assist</title>
padding: 0;
}
.text-uppercase {
text-transform: uppercase;
}
.connecting-message {
/* margin-top: 50%; */
font-size: 20px;
color: #aaa;
text-align: center;
display: none;
font-family: sans-serif;
}
.status-connecting .connecting-message { <!--CSS -->
/* display: block; */ <!-- <link href="css/styles.css" rel="stylesheet"> -->
} <style>
.status-connecting .card { body {
/* display: none; */ margin: 0;
} padding: 0;
}
.card{ .text-uppercase {
font: 14px 'Roboto', sans-serif; text-transform: uppercase;
/* min-width: 324px; */ }
width: 300px;
/* max-width: 800px; */
/* border: solid thin #ccc; */
/* box-shadow: 0 0 10px #aaa; */
border: solid 4px rgba(0, 0, 0, 0.2);
border-radius: .5rem;
}
.card-footers { .connecting-message {
display: flex; /* margin-top: 50%; */
border-bottom: solid thin #CCC; font-size: 20px;
padding: 5px 5px; color: #aaa;
justify-content: space-between; text-align: center;
} display: none;
font-family: sans-serif;
}
.card-footers .assist-controls { .status-connecting .connecting-message {
display: flex; /* display: block; */
align-items: center; }
}
.btn-danger { .status-connecting .card {
background-color: #CC0000 !important; /* display: none; */
color: white; }
}
.btn-danger:hover { .card {
background-color: #FF0000 !important; font: 14px 'Roboto', sans-serif;
color: white; /* min-width: 324px; */
} width: 300px;
/* max-width: 800px; */
/* border: solid thin #ccc; */
/* box-shadow: 0 0 10px #aaa; */
border: solid 4px rgba(0, 0, 0, 0.2);
border-radius: .5rem;
}
.btn { .card-footers {
padding: 5px 8px; display: flex;
font-size: 14px; border-bottom: solid thin #CCC;
border-radius: .5rem; padding: 5px 5px;
background-color: transparent; justify-content: space-between;
border: none; }
cursor: pointer;
display: flex;
align-items: center;
}
.btn span { .card-footers .assist-controls {
margin-left: 10px; display: flex;
} align-items: center;
}
.btn:hover { .btn-danger {
filter: brightness(0.9); background-color: #CC0000 !important;
} color: white;
}
.card .card-header{ .btn-danger:hover {
cursor: move; background-color: #FF0000 !important;
padding: 14px 18px; color: white;
display: flex; }
justify-content: space-between;
border-bottom: solid thin #ccc;
}
#agent-name, #duration{ .btn {
cursor:default; padding: 5px 8px;
} font-size: 14px;
border-radius: .5rem;
background-color: transparent;
border: none;
cursor: pointer;
display: flex;
align-items: center;
}
#video-container { .btn span {
background-color: rgb(90, 90, 90); margin-left: 10px;
position: relative; }
overflow: hidden;
/* width: 300px; */
}
#video-container video { .btn:hover {
width: 100% !important; filter: brightness(0.9);
height: auto; }
object-fit: cover;
}
#local-stream, #remote-stream {
/* display:none; */ /* TODO uncomment this line */
}
#video-container.remote #remote-stream {
display: block;
}
#video-container.local {
min-height: 100px;
}
#video-container.local #local-stream {
display: block;
}
#local-stream{ .card .card-header {
width: 35%; cursor: move;
/* top: 50%; */ padding: 14px 18px;
/* left: 70%; */ display: flex;
position: absolute; justify-content: space-between;
z-index: 99; border-bottom: solid thin #ccc;
bottom: 5px; }
right: 5px;
border: thin solid rgba(255,255,255, .3);
overflow: hidden;
}
#audio-btn { #agent-name,
margin-right: 10px; #duration {
} cursor: default;
#audio-btn .bi-mic { }
fill: #CC0000;
} #video-container {
#audio-btn .bi-mic-mute { background-color: rgb(90, 90, 90);
display:none; position: relative;
} overflow: hidden;
#audio-btn:after { /* width: 300px; */
/* text-transform: capitalize; */ }
color: #CC0000;
content: 'Mute'; #video-container video {
padding-left: 5px; width: 100% !important;
} height: auto;
#audio-btn.muted .bi-mic-mute { object-fit: cover;
display: inline-block; }
}
#audio-btn.muted .bi-mic { #local-stream,
display:none; #remote-stream {
} /* display:none; */
#audio-btn.muted:after { /* TODO uncomment this line */
content: 'Unmute'; }
padding-left: 5px;
} #video-container.remote #remote-stream {
display: block;
}
#video-container.local {
min-height: 100px;
}
#video-container.local #local-stream {
display: block;
}
#local-stream {
width: 35%;
/* top: 50%; */
/* left: 70%; */
position: absolute;
z-index: 99;
bottom: 5px;
right: 5px;
border: thin solid rgba(255, 255, 255, .3);
overflow: hidden;
}
#audio-btn {
margin-right: 10px;
}
#audio-btn .bi-mic {
fill: #CC0000;
}
#audio-btn .bi-mic-mute {
display: none;
}
#audio-btn:after {
/* text-transform: capitalize; */
color: #CC0000;
content: 'Mute';
padding-left: 5px;
}
#audio-btn.muted .bi-mic-mute {
display: inline-block;
}
#audio-btn.muted .bi-mic {
display: none;
}
#audio-btn.muted:after {
content: 'Unmute';
padding-left: 5px;
}
#video-btn .bi-camera-video { #video-btn .bi-camera-video {
fill: #CC0000; fill: #CC0000;
} }
#video-btn .bi-camera-video-off {
display:none;
}
#video-btn:after {
/* text-transform: capitalize; */
color: #CC0000;
content: 'Stop Video';
padding-left: 5px;
}
#video-btn.off:after {
content: 'Start Video';
padding-left: 5px;
}
#video-btn.off .bi-camera-video-off {
display: inline-block;
}
#video-btn.off .bi-camera-video {
display:none;
}
/* CHART */ #video-btn .bi-camera-video-off {
#chat-card { display: none;
display: flex; }
flex-direction: column;
font-size: 14px;
background-color: white;
}
#chat-card .chat-messages { display: none; } #video-btn:after {
#chat-card .chat-input { display: none; } /* text-transform: capitalize; */
#chat-card .chat-header .arrow-state { transform: rotate(180deg); } color: #CC0000;
#chat-card.active .chat-messages { display: flex; } content: 'Stop Video';
#chat-card.active .chat-input { display: flex; } padding-left: 5px;
#chat-card.active .chat-header .arrow-state { transform: rotate(0deg); } }
#chat-card .chat-header { #video-btn.off:after {
border-bottom: solid thin #ccc; content: 'Start Video';
padding: 8px 16px; padding-left: 5px;
display: flex; }
justify-content: space-between;
cursor: pointer;
}
#chat-card .chat-header .chat-title { #video-btn.off .bi-camera-video-off {
display: flex; display: inline-block;
align-items: center; }
}
#chat-card .chat-header .chat-title span { #video-btn.off .bi-camera-video {
margin-left: 6px; display: none;
} }
#chat-card .chat-messages { /* CHART */
padding: 8px 16px; #chat-card {
overflow-y: auto; display: flex;
height: 250px; flex-direction: column;
overflow-y: auto; font-size: 14px;
flex-direction: column; background-color: white;
justify-content: flex-end; }
}
#chat-card .message-text { #chat-card .chat-messages {
padding: 8px 16px; display: none;
border-radius: 20px; }
color: #666666;
margin-bottom: 2px;
}
#chat-card .message .message-text { #chat-card .chat-input {
/* max-width: 70%; */ display: none;
width: fit-content; }
}
#chat-card .message {
margin-bottom: 15px;
}
#chat-card .chat-messages .message.left .message-text { #chat-card .chat-header .arrow-state {
text-align: left; transform: rotate(180deg);
background: #D7E2E2; }
border-radius: 0px 30px 30px 30px;
}
#chat-card .message .message-user { #chat-card.active .chat-messages {
font-size: 12px; display: flex;
font-weight: bold; }
color: #999999;
}
#chat-card .message .message-time {
font-size: 12px;
color: #999999;
margin-left: 4px;
}
#chat-card .chat-messages .message.right { #chat-card.active .chat-input {
margin-left: auto; display: flex;
text-align: right; }
}
#chat-card .chat-messages .message.right .message-text { #chat-card.active .chat-header .arrow-state {
background: #E4E4E4; transform: rotate(0deg);
box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.15); }
border-radius: 30px 30px 0px 30px;
}
#chat-card .chat-input { #chat-card .chat-header {
margin: 10px; border-bottom: solid thin #ccc;
border-radius: .5rem; padding: 8px 16px;
box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.15); display: flex;
background-color: #DDDDDD; justify-content: space-between;
position: relative; cursor: pointer;
} }
#chat-card .chat-input .input { #chat-card .chat-header .chat-title {
width: 100%; display: flex;
border: none; align-items: center;
border-radius: 0px; }
padding: 8px 16px;
font-size: 16px;
color: #333;
background-color: transparent;
}
.send-btn { #chat-card .chat-header .chat-title span {
width: 26px; margin-left: 6px;
height: 26px; }
background-color: #AAA;
position: absolute;
right: 5px;
top: 0;
bottom: 0;
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
margin: auto;
cursor: pointer;
}
.send-btn:hover {
background-color: #999;
}
.send-btn svg {
fill: #DDDDDD;
}
.confirm-window .title { #chat-card .chat-messages {
margin-bottom: 10px; padding: 8px 16px;
} overflow-y: auto;
.confirm-window { height: 250px;
font: 14px 'Roboto', sans-serif; overflow-y: auto;
padding: 20px; flex-direction: column;
background-color: #F3F3F3; justify-content: flex-end;
border-radius: .5rem; }
/* position: absolute; */
width: fit-content;
color: #666666;
display: none;
}
.confirm-window .actions {
background-color: white;
padding: 10px;
display: flex;
box-shadow: 0px 0px 3.99778px 1.99889px rgba(0, 0, 0, 0.1);
border-radius: 6px;
}
.btn-lg { #chat-card .message-text {
font-size: 14px; padding: 8px 16px;
padding: 10px 14px; border-radius: 20px;
} color: #666666;
margin-bottom: 2px;
}
.btn-success { #chat-card .message .message-text {
background: rgba(0, 167, 47, 1); /* max-width: 70%; */
color: white; width: fit-content;
} }
/* .btn-error:hover, #chat-card .message {
margin-bottom: 15px;
}
#chat-card .chat-messages .message.left .message-text {
text-align: left;
background: #D7E2E2;
border-radius: 0px 30px 30px 30px;
}
#chat-card .message .message-user {
font-size: 12px;
font-weight: bold;
color: #999999;
}
#chat-card .message .message-time {
font-size: 12px;
color: #999999;
margin-left: 4px;
}
#chat-card .chat-messages .message.right {
margin-left: auto;
text-align: right;
}
#chat-card .chat-messages .message.right .message-text {
background: #E4E4E4;
box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.15);
border-radius: 30px 30px 0px 30px;
}
#chat-card .chat-input {
margin: 10px;
border-radius: .5rem;
box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.15);
background-color: #DDDDDD;
position: relative;
}
#chat-card .chat-input .input {
width: 100%;
border: none;
border-radius: 0px;
padding: 8px 16px;
font-size: 16px;
color: #333;
background-color: transparent;
}
.send-btn {
width: 26px;
height: 26px;
background-color: #AAA;
position: absolute;
right: 5px;
top: 0;
bottom: 0;
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
margin: auto;
cursor: pointer;
}
.send-btn:hover {
background-color: #999;
}
.send-btn svg {
fill: #DDDDDD;
}
.confirm-window .title {
margin-bottom: 10px;
}
.confirm-window {
font: 14px 'Roboto', sans-serif;
padding: 20px;
background-color: #F3F3F3;
border-radius: .5rem;
/* position: absolute; */
width: fit-content;
color: #666666;
display: none;
}
.confirm-window .actions {
background-color: white;
padding: 10px;
display: flex;
box-shadow: 0px 0px 3.99778px 1.99889px rgba(0, 0, 0, 0.1);
border-radius: 6px;
}
.btn-lg {
font-size: 14px;
padding: 10px 14px;
}
.btn-success {
background: rgba(0, 167, 47, 1);
color: white;
}
/* .btn-error:hover,
.btn-success:hover { .btn-success:hover {
filter: brightness(0.9); filter: brightness(0.9);
} */ } */
.btn-error { .btn-error {
background: #FFE9E9; background: #FFE9E9;
/* border-color: #d43f3a; */ /* border-color: #d43f3a; */
color: #CC0000; color: #CC0000;
} }
</style> </style>
</head> </head>
<body data-openreplay-hidden> <body data-openreplay-hidden>
<div id="remote-control-confirm" class="confirm-window"> <div id="remote-control-confirm" class="confirm-window">
<div class="title">The agent is requesting remote control</div> <div class="title">The agent is requesting remote control</div>
<div class="actions"> <div class="actions">
<button class="text-uppercase btn btn-lg btn-success" style="margin-right: 10px">Grant remote access</button> <button class="text-uppercase btn btn-lg btn-success" style="margin-right: 10px">Grant remote access</button>
<button class="text-uppercase btn btn-lg btn-error">Reject</button> <button class="text-uppercase btn btn-lg btn-error">Reject</button>
</div>
</div> </div>
</div>
<div id="call-confirm" class="confirm-window"> <div id="call-confirm" class="confirm-window">
<div class="title">Answer the call so the agent can assist.</div> <div class="title">Answer the call so the agent can assist.</div>
<div class="actions"> <div class="actions">
<button class="text-uppercase btn btn-lg btn-success" style="margin-right: 10px"> <button class="text-uppercase btn btn-lg btn-success" style="margin-right: 10px">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-telephone" viewBox="0 0 16 16"> <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-telephone"
<path d="M3.654 1.328a.678.678 0 0 0-1.015-.063L1.605 2.3c-.483.484-.661 1.169-.45 1.77a17.568 17.568 0 0 0 4.168 6.608 17.569 17.569 0 0 0 6.608 4.168c.601.211 1.286.033 1.77-.45l1.034-1.034a.678.678 0 0 0-.063-1.015l-2.307-1.794a.678.678 0 0 0-.58-.122l-2.19.547a1.745 1.745 0 0 1-1.657-.459L5.482 8.062a1.745 1.745 0 0 1-.46-1.657l.548-2.19a.678.678 0 0 0-.122-.58L3.654 1.328zM1.884.511a1.745 1.745 0 0 1 2.612.163L6.29 2.98c.329.423.445.974.315 1.494l-.547 2.19a.678.678 0 0 0 .178.643l2.457 2.457a.678.678 0 0 0 .644.178l2.189-.547a1.745 1.745 0 0 1 1.494.315l2.306 1.794c.829.645.905 1.87.163 2.611l-1.034 1.034c-.74.74-1.846 1.065-2.877.702a18.634 18.634 0 0 1-7.01-4.42 18.634 18.634 0 0 1-4.42-7.009c-.362-1.03-.037-2.137.703-2.877L1.885.511z"/> viewBox="0 0 16 16">
</svg> <path
<span>Answer</span> d="M3.654 1.328a.678.678 0 0 0-1.015-.063L1.605 2.3c-.483.484-.661 1.169-.45 1.77a17.568 17.568 0 0 0 4.168 6.608 17.569 17.569 0 0 0 6.608 4.168c.601.211 1.286.033 1.77-.45l1.034-1.034a.678.678 0 0 0-.063-1.015l-2.307-1.794a.678.678 0 0 0-.58-.122l-2.19.547a1.745 1.745 0 0 1-1.657-.459L5.482 8.062a1.745 1.745 0 0 1-.46-1.657l.548-2.19a.678.678 0 0 0-.122-.58L3.654 1.328zM1.884.511a1.745 1.745 0 0 1 2.612.163L6.29 2.98c.329.423.445.974.315 1.494l-.547 2.19a.678.678 0 0 0 .178.643l2.457 2.457a.678.678 0 0 0 .644.178l2.189-.547a1.745 1.745 0 0 1 1.494.315l2.306 1.794c.829.645.905 1.87.163 2.611l-1.034 1.034c-.74.74-1.846 1.065-2.877.702a18.634 18.634 0 0 1-7.01-4.42 18.634 18.634 0 0 1-4.42-7.009c-.362-1.03-.037-2.137.703-2.877L1.885.511z" />
</button> </svg>
<button class="text-uppercase btn btn-lg btn-error">Reject</button> <span>Answer</span>
</div> </button>
<button class="text-uppercase btn btn-lg btn-error">Reject</button>
</div> </div>
<section id="or-assist" class="status-connecting"> </div>
<div class="connecting-message"> Connecting... </div> <section id="or-assist" class="status-connecting">
<div class="card shadow"> <div class="connecting-message"> Connecting... </div>
<div class="drag-area card-header d-flex justify-content-between"> <div class="card shadow">
<div class="user-info"> <div class="drag-area card-header d-flex justify-content-between">
<span>Call with</span> <div class="user-info">
<!-- User Name --> <span>Call with</span>
<span id="agent-name" class="person-name fw-light" >Support Agent</span> <!-- User Name -->
</div> <span id="agent-name" class="person-name fw-light">Support Agent</span>
<div class="call-duration">
<!--Call Duration. -->
<span id="duration" class="card-subtitle mb-2 text-muted fw-light" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Duration">00:00</span>
</div>
</div>
<div id="video-container" class="card-body bg-dark p-0 d-flex align-items-center position-relative">
<div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow">
<!-- <p class="text-white m-auto text-center">Starting video...</p> -->
<video id="video-local" autoplay muted></video>
</div>
<div id="remote-stream" class="ratio ratio-4x3 m-0 p-0">
<!-- <p id="remote-stream-placeholder" class="text-white m-auto text-center">Starting video...</p> -->
<video id="video-remote" autoplay></video>
</div>
</div>
<div class="card-footers">
<div class="assist-controls">
<!-- Add class .muted to #audio-btn when user mutes audio -->
<button
href="#"
id="audio-btn"
class="btn btn-light btn-sm text-uppercase me-2"
>
<i>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" class="bi bi-mic" viewBox="0 0 16 16">
<path d="M3.5 6.5A.5.5 0 0 1 4 7v1a4 4 0 0 0 8 0V7a.5.5 0 0 1 1 0v1a5 5 0 0 1-4.5 4.975V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 .5-.5z"/>
<path d="M10 8a2 2 0 1 1-4 0V3a2 2 0 1 1 4 0v5zM8 0a3 3 0 0 0-3 3v5a3 3 0 0 0 6 0V3a3 3 0 0 0-3-3z"/>
</svg>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" class="bi bi-mic-mute" viewBox="0 0 16 16">
<path d="M13 8c0 .564-.094 1.107-.266 1.613l-.814-.814A4.02 4.02 0 0 0 12 8V7a.5.5 0 0 1 1 0v1zm-5 4c.818 0 1.578-.245 2.212-.667l.718.719a4.973 4.973 0 0 1-2.43.923V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 1 0v1a4 4 0 0 0 4 4zm3-9v4.879l-1-1V3a2 2 0 0 0-3.997-.118l-.845-.845A3.001 3.001 0 0 1 11 3z"/>
<path d="m9.486 10.607-.748-.748A2 2 0 0 1 6 8v-.878l-1-1V8a3 3 0 0 0 4.486 2.607zm-7.84-9.253 12 12 .708-.708-12-12-.708.708z"/>
</svg>
</i>
</button>
<!--Add class .off to #video-btn when user stops video -->
<button
href="#"
id="video-btn"
class="btn btn-light btn-sm text-uppercase ms-2"
>
<i>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-camera-video" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M0 5a2 2 0 0 1 2-2h7.5a2 2 0 0 1 1.983 1.738l3.11-1.382A1 1 0 0 1 16 4.269v7.462a1 1 0 0 1-1.406.913l-3.111-1.382A2 2 0 0 1 9.5 13H2a2 2 0 0 1-2-2V5zm11.5 5.175 3.5 1.556V4.269l-3.5 1.556v4.35zM2 4a1 1 0 0 0-1 1v6a1 1 0 0 0 1 1h7.5a1 1 0 0 0 1-1V5a1 1 0 0 0-1-1H2z"/>
</svg>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-camera-video-off" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M10.961 12.365a1.99 1.99 0 0 0 .522-1.103l3.11 1.382A1 1 0 0 0 16 11.731V4.269a1 1 0 0 0-1.406-.913l-3.111 1.382A2 2 0 0 0 9.5 3H4.272l.714 1H9.5a1 1 0 0 1 1 1v6a1 1 0 0 1-.144.518l.605.847zM1.428 4.18A.999.999 0 0 0 1 5v6a1 1 0 0 0 1 1h5.014l.714 1H2a2 2 0 0 1-2-2V5c0-.675.334-1.272.847-1.634l.58.814zM15 11.73l-3.5-1.555v-4.35L15 4.269v7.462zm-4.407 3.56-10-14 .814-.58 10 14-.814.58z"/>
</svg>
</i>
</button>
</div>
<button id="end-call-btn" href="#" class="btn btn-danger btn-sm text-uppercase" style="margin-right: 8px;">End</button>
</div>
<!-- CHAT - add .active class to show the messages and input -->
<div id="chat-card" class="active">
<div class="chat-header">
<div class="chat-title">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="bi bi-chat" viewBox="0 0 16 16">
<path d="M2.678 11.894a1 1 0 0 1 .287.801 10.97 10.97 0 0 1-.398 2c1.395-.323 2.247-.697 2.634-.893a1 1 0 0 1 .71-.074A8.06 8.06 0 0 0 8 14c3.996 0 7-2.807 7-6 0-3.192-3.004-6-7-6S1 4.808 1 8c0 1.468.617 2.83 1.678 3.894zm-.493 3.905a21.682 21.682 0 0 1-.713.129c-.2.032-.352-.176-.273-.362a9.68 9.68 0 0 0 .244-.637l.003-.01c.248-.72.45-1.548.524-2.319C.743 11.37 0 9.76 0 8c0-3.866 3.582-7 8-7s8 3.134 8 7-3.582 7-8 7a9.06 9.06 0 0 1-2.347-.306c-.52.263-1.639.742-3.468 1.105z"/>
</svg>
<span>Chat</span>
</div>
<div>
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" class="bi bi-chevron-up arrow-state" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M7.646 4.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1-.708.708L8 5.707l-5.646 5.647a.5.5 0 0 1-.708-.708l6-6z"/>
</svg>
</div>
</div>
<div class="chat-messages">
<div class="message left">
<div class="message-text"> Hey, did you get the key? </div>
<div>
<span class="message-user">Username</span>
<span class="message-time"> 00:00 </span>
</div>
</div>
<div class="message right">
<div class="message-text">
Oui, merci!
</div>
<div>
<span class="message-user">Username</span>
<span class="message-time">00:00</span>
</div>
</div>
</div>
<div class="chat-input">
<input type="text" class="input" placeholder="Type a message...">
<div class="send-btn">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" class="bi bi-arrow-right-short" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z"/>
</svg>
</div>
</div>
</div>
</div> </div>
</section> <div class="call-duration">
</body> <!--Call Duration. -->
</html> <span id="duration" class="card-subtitle mb-2 text-muted fw-light" data-bs-toggle="tooltip"
data-bs-placement="bottom" title="Duration">00:00</span>
</div>
</div>
<div id="video-container" class="card-body bg-dark p-0 d-flex align-items-center position-relative">
<div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow scale-x-[-1]">
<!-- <p class="text-white m-auto text-center">Starting video...</p> -->
<video id="video-local" autoplay muted class="scale-x-[-1]"></video>
</div>
<div id="remote-stream" class="ratio ratio-4x3 m-0 p-0">
<!-- <p id="remote-stream-placeholder" class="text-white m-auto text-center">Starting video...</p> -->
<video id="video-remote" autoplay></video>
</div>
</div>
<div class="card-footers">
<div class="assist-controls">
<!-- Add class .muted to #audio-btn when user mutes audio -->
<button href="#" id="audio-btn" class="btn btn-light btn-sm text-uppercase me-2">
<i>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" class="bi bi-mic" viewBox="0 0 16 16">
<path
d="M3.5 6.5A.5.5 0 0 1 4 7v1a4 4 0 0 0 8 0V7a.5.5 0 0 1 1 0v1a5 5 0 0 1-4.5 4.975V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 .5-.5z" />
<path d="M10 8a2 2 0 1 1-4 0V3a2 2 0 1 1 4 0v5zM8 0a3 3 0 0 0-3 3v5a3 3 0 0 0 6 0V3a3 3 0 0 0-3-3z" />
</svg>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" class="bi bi-mic-mute" viewBox="0 0 16 16">
<path
d="M13 8c0 .564-.094 1.107-.266 1.613l-.814-.814A4.02 4.02 0 0 0 12 8V7a.5.5 0 0 1 1 0v1zm-5 4c.818 0 1.578-.245 2.212-.667l.718.719a4.973 4.973 0 0 1-2.43.923V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 1 0v1a4 4 0 0 0 4 4zm3-9v4.879l-1-1V3a2 2 0 0 0-3.997-.118l-.845-.845A3.001 3.001 0 0 1 11 3z" />
<path
d="m9.486 10.607-.748-.748A2 2 0 0 1 6 8v-.878l-1-1V8a3 3 0 0 0 4.486 2.607zm-7.84-9.253 12 12 .708-.708-12-12-.708.708z" />
</svg>
</i>
</button>
<!--Add class .off to #video-btn when user stops video -->
<button href="#" id="video-btn" class="btn btn-light btn-sm text-uppercase ms-2">
<i>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
class="bi bi-camera-video" viewBox="0 0 16 16">
<path fill-rule="evenodd"
d="M0 5a2 2 0 0 1 2-2h7.5a2 2 0 0 1 1.983 1.738l3.11-1.382A1 1 0 0 1 16 4.269v7.462a1 1 0 0 1-1.406.913l-3.111-1.382A2 2 0 0 1 9.5 13H2a2 2 0 0 1-2-2V5zm11.5 5.175 3.5 1.556V4.269l-3.5 1.556v4.35zM2 4a1 1 0 0 0-1 1v6a1 1 0 0 0 1 1h7.5a1 1 0 0 0 1-1V5a1 1 0 0 0-1-1H2z" />
</svg>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
class="bi bi-camera-video-off" viewBox="0 0 16 16">
<path fill-rule="evenodd"
d="M10.961 12.365a1.99 1.99 0 0 0 .522-1.103l3.11 1.382A1 1 0 0 0 16 11.731V4.269a1 1 0 0 0-1.406-.913l-3.111 1.382A2 2 0 0 0 9.5 3H4.272l.714 1H9.5a1 1 0 0 1 1 1v6a1 1 0 0 1-.144.518l.605.847zM1.428 4.18A.999.999 0 0 0 1 5v6a1 1 0 0 0 1 1h5.014l.714 1H2a2 2 0 0 1-2-2V5c0-.675.334-1.272.847-1.634l.58.814zM15 11.73l-3.5-1.555v-4.35L15 4.269v7.462zm-4.407 3.56-10-14 .814-.58 10 14-.814.58z" />
</svg>
</i>
</button>
</div>
<button id="end-call-btn" href="#" class="btn btn-danger btn-sm text-uppercase"
style="margin-right: 8px;">End</button>
</div>
<!-- CHAT - add .active class to show the messages and input -->
<div id="chat-card" class="active">
<div class="chat-header">
<div class="chat-title">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="bi bi-chat"
viewBox="0 0 16 16">
<path
d="M2.678 11.894a1 1 0 0 1 .287.801 10.97 10.97 0 0 1-.398 2c1.395-.323 2.247-.697 2.634-.893a1 1 0 0 1 .71-.074A8.06 8.06 0 0 0 8 14c3.996 0 7-2.807 7-6 0-3.192-3.004-6-7-6S1 4.808 1 8c0 1.468.617 2.83 1.678 3.894zm-.493 3.905a21.682 21.682 0 0 1-.713.129c-.2.032-.352-.176-.273-.362a9.68 9.68 0 0 0 .244-.637l.003-.01c.248-.72.45-1.548.524-2.319C.743 11.37 0 9.76 0 8c0-3.866 3.582-7 8-7s8 3.134 8 7-3.582 7-8 7a9.06 9.06 0 0 1-2.347-.306c-.52.263-1.639.742-3.468 1.105z" />
</svg>
<span>Chat</span>
</div>
<div>
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" class="bi bi-chevron-up arrow-state"
viewBox="0 0 16 16">
<path fill-rule="evenodd"
d="M7.646 4.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1-.708.708L8 5.707l-5.646 5.647a.5.5 0 0 1-.708-.708l6-6z" />
</svg>
</div>
</div>
<div class="chat-messages">
<div class="message left">
<div class="message-text"> Hey, did you get the key? </div>
<div>
<span class="message-user">Username</span>
<span class="message-time"> 00:00 </span>
</div>
</div>
<div class="message right">
<div class="message-text">
Oui, merci!
</div>
<div>
<span class="message-user">Username</span>
<span class="message-time">00:00</span>
</div>
</div>
</div>
<div class="chat-input">
<input type="text" class="input" placeholder="Type a message...">
<div class="send-btn">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" class="bi bi-arrow-right-short"
viewBox="0 0 16 16">
<path fill-rule="evenodd"
d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" />
</svg>
</div>
</div>
</div>
</div>
</section>
</body>
</html>

5
tracker/tracker-assist/layout/index.html
View file

@ -152,8 +152,9 @@
</div> </div>
</div> </div>
<div id="video-container" class="card-body bg-dark p-0 d-flex align-items-center position-relative"> <div id="video-container" class="card-body bg-dark p-0 d-flex align-items-center position-relative">
<div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow"> <div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow scale-x-[-1]">
<video id="video-local" autoplay muted></video> <!-- fix horizontal mirroring -->
<video id="video-local" autoplay muted class="scale-x-[-1]"></video>
</div> </div>
<div id="remote-stream" class="ratio ratio-4x3 m-0 p-0"> <div id="remote-stream" class="ratio ratio-4x3 m-0 p-0">

1
tracker/tracker-assist/package.json
View file

@ -30,7 +30,6 @@
"dependencies": { "dependencies": {
"csstype": "^3.0.10", "csstype": "^3.0.10",
"fflate": "^0.8.2", "fflate": "^0.8.2",
"peerjs": "1.5.4",
"socket.io-client": "^4.8.1" "socket.io-client": "^4.8.1"
}, },
"peerDependencies": { "peerDependencies": {

599
tracker/tracker-assist/src/Assist.ts
View file

@ -1,25 +1,21 @@
/* eslint-disable @typescript-eslint/no-empty-function */ /* eslint-disable @typescript-eslint/no-empty-function */
import type { Socket, } from 'socket.io-client' import type { Socket } from 'socket.io-client'
import { connect, } from 'socket.io-client' import { connect } from 'socket.io-client'
import Peer, { MediaConnection, } from 'peerjs' import type { Properties } from 'csstype'
import type { Properties, } from 'csstype' import { App } from '@openreplay/tracker'
import { App, } from '@openreplay/tracker'
import RequestLocalStream, { LocalStream, } from './LocalStream.js' import RequestLocalStream, { LocalStream } from './LocalStream.js'
import {hasTag,} from './guards.js' import { hasTag } from './guards.js'
import RemoteControl, { RCStatus, } from './RemoteControl.js' import RemoteControl, { RCStatus } from './RemoteControl.js'
import CallWindow from './CallWindow.js' import CallWindow from './CallWindow.js'
import AnnotationCanvas from './AnnotationCanvas.js' import AnnotationCanvas from './AnnotationCanvas.js'
import ConfirmWindow from './ConfirmWindow/ConfirmWindow.js' import ConfirmWindow from './ConfirmWindow/ConfirmWindow.js'
import { callConfirmDefault, } from './ConfirmWindow/defaults.js' import { callConfirmDefault } from './ConfirmWindow/defaults.js'
import type { Options as ConfirmOptions, } from './ConfirmWindow/defaults.js' import type { Options as ConfirmOptions } from './ConfirmWindow/defaults.js'
import ScreenRecordingState from './ScreenRecordingState.js' import ScreenRecordingState from './ScreenRecordingState.js'
import { pkgVersion, } from './version.js' import { pkgVersion } from './version.js'
import Canvas from './Canvas.js' import Canvas from './Canvas.js'
import { gzip, } from 'fflate' import { gzip } from 'fflate'
// TODO: fully specified strict check with no-any (everywhere)
// @ts-ignore
const safeCastedPeer = Peer.default || Peer
type StartEndCallback = (agentInfo?: Record<string, any>) => ((() => any) | void) type StartEndCallback = (agentInfo?: Record<string, any>) => ((() => any) | void)
@ -52,26 +48,23 @@ export interface Options {
confirmStyle?: Properties; confirmStyle?: Properties;
config: RTCConfiguration; config: RTCConfiguration;
serverURL: string serverURL: string;
callUITemplate?: string; callUITemplate?: string;
compressionEnabled: boolean; compressionEnabled: boolean;
/** /**
* Minimum amount of messages in a batch to trigger compression run * Minimum amount of messages in a batch to trigger compression run
* @default 5000 * @default 5000
* */ */
compressionMinBatchSize: number compressionMinBatchSize: number;
} }
enum CallingState { enum CallingState {
Requesting, Requesting,
True, True,
False, False,
} }
type OptionalCallback = (() => Record<string, unknown>) | void
// TODO typing????
type OptionalCallback = (()=>Record<string, unknown>) | void
type Agent = { type Agent = {
onDisconnect?: OptionalCallback, onDisconnect?: OptionalCallback,
onControlReleased?: OptionalCallback, onControlReleased?: OptionalCallback,
@ -84,8 +77,8 @@ export default class Assist {
readonly version = pkgVersion readonly version = pkgVersion
private socket: Socket | null = null private socket: Socket | null = null
private peer: Peer | null = null private calls: Map<string, RTCPeerConnection> = new Map();
private canvasPeers: Record<number, Peer | null> = {} private canvasPeers: { [id: number]: RTCPeerConnection | null } = {}
private canvasNodeCheckers: Map<number, any> = new Map() private canvasNodeCheckers: Map<number, any> = new Map()
private assistDemandedRestart = false private assistDemandedRestart = false
private callingState: CallingState = CallingState.False private callingState: CallingState = CallingState.False
@ -103,20 +96,20 @@ export default class Assist {
// @ts-ignore // @ts-ignore
window.__OR_ASSIST_VERSION = this.version window.__OR_ASSIST_VERSION = this.version
this.options = Object.assign({ this.options = Object.assign({
session_calling_peer_key: '__openreplay_calling_peer', session_calling_peer_key: '__openreplay_calling_peer',
session_control_peer_key: '__openreplay_control_peer', session_control_peer_key: '__openreplay_control_peer',
config: null, config: null,
serverURL: null, serverURL: null,
onCallStart: ()=>{}, onCallStart: () => { },
onAgentConnect: ()=>{}, onAgentConnect: () => { },
onRemoteControlStart: ()=>{}, onRemoteControlStart: () => { },
callConfirm: {}, callConfirm: {},
controlConfirm: {}, // TODO: clear options passing/merging/overwriting controlConfirm: {}, // TODO: clear options passing/merging/overwriting
recordingConfirm: {}, recordingConfirm: {},
socketHost: '', socketHost: '',
compressionEnabled: false, compressionEnabled: false,
compressionMinBatchSize: 5000, compressionMinBatchSize: 5000,
}, },
options, options,
) )
@ -155,7 +148,7 @@ export default class Assist {
if (this.agentsConnected) { if (this.agentsConnected) {
const batchSize = messages.length const batchSize = messages.length
// @ts-ignore No need in statistics messages. TODO proper filter // @ts-ignore No need in statistics messages. TODO proper filter
if (batchSize === 2 && messages[0]._id === 0 && messages[1]._id === 49) { return } if (batchSize === 2 && messages[0]._id === 0 && messages[1]._id === 49) { return }
if (batchSize > this.options.compressionMinBatchSize && this.options.compressionEnabled) { if (batchSize > this.options.compressionMinBatchSize && this.options.compressionEnabled) {
const toSend: any[] = [] const toSend: any[] = []
if (batchSize > 10000) { if (batchSize > 10000) {
@ -198,17 +191,17 @@ export default class Assist {
private readonly setCallingState = (newState: CallingState): void => { private readonly setCallingState = (newState: CallingState): void => {
this.callingState = newState this.callingState = newState
} }
private getHost():string{ private getHost(): string {
if (this.options.socketHost) { if (this.options.socketHost) {
return this.options.socketHost return this.options.socketHost
} }
if (this.options.serverURL){ if (this.options.serverURL) {
return new URL(this.options.serverURL).host return new URL(this.options.serverURL).host
} }
return this.app.getHost() return this.app.getHost()
} }
private getBasePrefixUrl(): string{ private getBasePrefixUrl(): string {
if (this.options.serverURL){ if (this.options.serverURL) {
return new URL(this.options.serverURL).pathname return new URL(this.options.serverURL).pathname
} }
return '' return ''
@ -232,7 +225,7 @@ export default class Assist {
// SocketIO // SocketIO
const socket = this.socket = connect(this.getHost(), { const socket = this.socket = connect(this.getHost(), {
path: this.getBasePrefixUrl()+'/ws-assist/socket', path: this.getBasePrefixUrl() + '/ws-assist/socket',
query: { query: {
'peerId': peerID, 'peerId': peerID,
'identity': 'session', 'identity': 'session',
@ -257,14 +250,19 @@ export default class Assist {
if (args[0] === 'messages' || args[0] === 'UPDATE_SESSION') { if (args[0] === 'messages' || args[0] === 'UPDATE_SESSION') {
return return
} }
app.debug.log('Socket:', ...args) if (args[0] !== 'webrtc_call_ice_candidate') {
app.debug.log('Socket:', ...args)
};
socket.on('close', (e) => {
app.debug.warn('Socket closed:', e);
})
}) })
const onGrand = (id: string) => { const onGrand = (id: string) => {
if (!callUI) { if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate) callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
} }
if (this.remoteControl){ if (this.remoteControl) {
callUI?.showRemoteControl(this.remoteControl.releaseControl) callUI?.showRemoteControl(this.remoteControl.releaseControl)
} }
this.agents[id] = { ...this.agents[id], onControlReleased: this.options.onRemoteControlStart(this.agents[id]?.agentInfo), } this.agents[id] = { ...this.agents[id], onControlReleased: this.options.onRemoteControlStart(this.agents[id]?.agentInfo), }
@ -274,26 +272,24 @@ export default class Assist {
return callingAgents.get(id) return callingAgents.get(id)
} }
const onRelease = (id?: string | null, isDenied?: boolean) => { const onRelease = (id?: string | null, isDenied?: boolean) => {
{ if (id) {
if (id) { const cb = this.agents[id].onControlReleased
const cb = this.agents[id].onControlReleased delete this.agents[id].onControlReleased
delete this.agents[id].onControlReleased typeof cb === 'function' && cb()
typeof cb === 'function' && cb() this.emit('control_rejected', id)
this.emit('control_rejected', id) }
} if (annot != null) {
if (annot != null) { annot.remove()
annot.remove() annot = null
annot = null }
} callUI?.hideRemoteControl()
callUI?.hideRemoteControl() if (this.callingState !== CallingState.True) {
if (this.callingState !== CallingState.True) { callUI?.remove()
callUI?.remove() callUI = null
callUI = null }
} if (isDenied) {
if (isDenied) { const info = id ? this.agents[id]?.agentInfo : {}
const info = id ? this.agents[id]?.agentInfo : {} this.options.onRemoteControlDeny?.(info || {})
this.options.onRemoteControlDeny?.(info || {})
}
} }
} }
@ -342,11 +338,12 @@ export default class Assist {
// TODO: restrict by id // TODO: restrict by id
socket.on('moveAnnotation', (id, event) => processEvent(id, event, (_, d) => annot && annot.move(d))) socket.on('moveAnnotation', (id, event) => processEvent(id, event, (_, d) => annot && annot.move(d)))
socket.on('startAnnotation', (id, event) => processEvent(id, event, (_, d) => annot?.start(d))) socket.on('startAnnotation', (id, event) => processEvent(id, event, (_, d) => annot?.start(d)))
socket.on('stopAnnotation', (id, event) => processEvent(id, event, annot?.stop)) socket.on('stopAnnotation', (id, event) => processEvent(id, event, annot?.stop))
socket.on('NEW_AGENT', (id: string, info: AgentInfo) => { socket.on('NEW_AGENT', (id: string, info: AgentInfo) => {
this.cleanCanvasConnections();
this.agents[id] = { this.agents[id] = {
onDisconnect: this.options.onAgentConnect?.(info), onDisconnect: this.options.onAgentConnect?.(info),
agentInfo: info, // TODO ? agentInfo: info, // TODO ?
@ -369,8 +366,10 @@ export default class Assist {
}) })
} }
}) })
socket.on('AGENTS_CONNECTED', (ids: string[]) => { socket.on('AGENTS_CONNECTED', (ids: string[]) => {
ids.forEach(id =>{ this.cleanCanvasConnections();
ids.forEach(id => {
const agentInfo = this.agents[id]?.agentInfo const agentInfo = this.agents[id]?.agentInfo
this.agents[id] = { this.agents[id] = {
agentInfo, agentInfo,
@ -385,7 +384,7 @@ export default class Assist {
this.app.allowAppStart() this.app.allowAppStart()
setTimeout(() => { setTimeout(() => {
this.app.start().then(() => { this.assistDemandedRestart = false }) this.app.start().then(() => { this.assistDemandedRestart = false })
.then(() => { .then(() => {
this.remoteControl?.reconnect(ids) this.remoteControl?.reconnect(ids)
}) })
.catch(e => app.debug.error(e)) .catch(e => app.debug.error(e))
@ -400,37 +399,65 @@ export default class Assist {
this.agents[id]?.onDisconnect?.() this.agents[id]?.onDisconnect?.()
delete this.agents[id] delete this.agents[id]
Object.values(this.calls).forEach(pc => pc.close())
this.calls.clear();
recordingState.stopAgentRecording(id) recordingState.stopAgentRecording(id)
endAgentCall(id) endAgentCall({ socketId: id })
}) })
socket.on('NO_AGENT', () => { socket.on('NO_AGENT', () => {
Object.values(this.agents).forEach(a => a.onDisconnect?.()) Object.values(this.agents).forEach(a => a.onDisconnect?.())
this.cleanCanvasConnections();
this.agents = {} this.agents = {}
if (recordingState.isActive) recordingState.stopRecording() if (recordingState.isActive) recordingState.stopRecording()
}) })
socket.on('call_end', (id) => {
if (!callingAgents.has(id)) { socket.on('call_end', (socketId, { data: callId }) => {
app.debug.warn('Received call_end from unknown agent', id) if (!callingAgents.has(socketId)) {
app.debug.warn('Received call_end from unknown agent', socketId)
return return
} }
endAgentCall(id)
endAgentCall({ socketId, callId })
}) })
socket.on('_agent_name', (id, info) => { socket.on('_agent_name', (id, info) => {
if (app.getTabId() !== info.meta.tabId) return if (app.getTabId() !== info.meta.tabId) return
const name = info.data const name = info.data
callingAgents.set(id, name) callingAgents.set(id, name)
if (!this.peer) {
setupPeer()
}
updateCallerNames() updateCallerNames()
}) })
socket.on('webrtc_canvas_answer', async (_, data: { answer, id }) => {
const pc = this.canvasPeers[data.id];
if (pc) {
try {
await pc.setRemoteDescription(new RTCSessionDescription(data.answer));
} catch (e) {
app.debug.error('Error adding ICE candidate', e);
}
}
})
socket.on('webrtc_canvas_ice_candidate', async (_, data: { candidate, id }) => {
const pc = this.canvasPeers[data.id];
if (pc) {
try {
await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
} catch (e) {
app.debug.error('Error adding ICE candidate', e);
}
}
})
// If a videofeed arrives, then we show the video in the ui
socket.on('videofeed', (_, info) => { socket.on('videofeed', (_, info) => {
if (app.getTabId() !== info.meta.tabId) return if (app.getTabId() !== info.meta.tabId) return
const feedState = info.data const feedState = info.data
callUI?.toggleVideoStream(feedState) callUI?.toggleVideoStream(feedState)
}) })
socket.on('request_recording', (id, info) => { socket.on('request_recording', (id, info) => {
if (app.getTabId() !== info.meta.tabId) return if (app.getTabId() !== info.meta.tabId) return
const agentData = info.data const agentData = info.data
@ -448,29 +475,52 @@ export default class Assist {
} }
}) })
socket.on('webrtc_call_offer', async (_, data: { from: string, offer: RTCSessionDescriptionInit }) => {
if (!this.calls.has(data.from)) {
await handleIncomingCallOffer(data.from, data.offer);
}
});
socket.on('webrtc_call_ice_candidate', async (data: { from: string, candidate: RTCIceCandidateInit }) => {
const pc = this.calls[data.from];
if (pc) {
try {
await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
} catch (e) {
app.debug.error('Error adding ICE candidate', e);
}
}
});
const callingAgents: Map<string, string> = new Map() // !! uses socket.io ID const callingAgents: Map<string, string> = new Map() // !! uses socket.io ID
// TODO: merge peerId & socket.io id (simplest way - send peerId with the name) // TODO: merge peerId & socket.io id (simplest way - send peerId with the name)
const calls: Record<string, MediaConnection> = {} // !! uses peerJS ID
const lStreams: Record<string, LocalStream> = {} const lStreams: Record<string, LocalStream> = {}
function updateCallerNames() { function updateCallerNames() {
callUI?.setAssistentName(callingAgents) callUI?.setAssistentName(callingAgents)
} }
function endAgentCall(id: string) { function endAgentCall({ socketId, callId }: { socketId: string, callId?: string }) {
callingAgents.delete(id) callingAgents.delete(socketId)
if (callingAgents.size === 0) { if (callingAgents.size === 0) {
handleCallEnd() handleCallEnd()
} else { } else {
updateCallerNames() updateCallerNames()
//TODO: close() specific call and corresponding lStreams (after connecting peerId & socket.io id) if (callId) {
handleCallEndWithAgent(callId)
}
} }
} }
const handleCallEnd = () => { // Complete stop and clear all calls
// Streams const handleCallEndWithAgent = (id: string) => {
Object.values(calls).forEach(call => call.close()) this.calls.get(id)?.close()
Object.keys(calls).forEach(peerId => { this.calls.delete(id)
delete calls[peerId] }
})
// call end handling
const handleCallEnd = () => {
Object.values(this.calls).forEach(pc => pc.close())
this.calls.clear();
Object.values(lStreams).forEach((stream) => { stream.stop() }) Object.values(lStreams).forEach((stream) => { stream.stop() })
Object.keys(lStreams).forEach((peerId: string) => { delete lStreams[peerId] }) Object.keys(lStreams).forEach((peerId: string) => { delete lStreams[peerId] })
// UI // UI
@ -484,7 +534,7 @@ export default class Assist {
callUI?.hideControls() callUI?.hideControls()
} }
this.emit('UPDATE_SESSION', { agentIds: [], isCallActive: false, }) this.emit('UPDATE_SESSION', { agentIds: [], isCallActive: false })
this.setCallingState(CallingState.False) this.setCallingState(CallingState.False)
sessionStorage.removeItem(this.options.session_calling_peer_key) sessionStorage.removeItem(this.options.session_calling_peer_key)
@ -498,175 +548,192 @@ export default class Assist {
} }
} }
// PeerJS call (todo: use native WebRTC) const handleIncomingCallOffer = async (from: string, offer: RTCSessionDescriptionInit) => {
const peerOptions = { app.debug.log('handleIncomingCallOffer', from)
host: this.getHost(), let confirmAnswer: Promise<boolean>
path: this.getBasePrefixUrl()+'/assist', const callingPeerIds = JSON.parse(sessionStorage.getItem(this.options.session_calling_peer_key) || '[]')
port: location.protocol === 'http:' && this.noSecureMode ? 80 : 443, // if the caller is already in the list, then we immediately accept the call without ui
debug: 2, //appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs. if (callingPeerIds.includes(from) || this.callingState === CallingState.True) {
} confirmAnswer = Promise.resolve(true)
const setupPeer = () => { } else {
if (this.options.config) { // set the state to wait for confirmation
peerOptions['config'] = this.options.config this.setCallingState(CallingState.Requesting)
// call the call confirmation window
confirmAnswer = requestCallConfirm()
// sound notification of a call
this.playNotificationSound()
// after 30 seconds we drop the call
setTimeout(() => {
if (this.callingState !== CallingState.Requesting) { return }
initiateCallEnd()
}, 30000)
} }
const peer = new safeCastedPeer(peerID, peerOptions) as Peer try {
this.peer = peer // waiting for a decision on accepting the challenge
let peerReconnectAttempts = 0 const agreed = await confirmAnswer
// @ts-ignore (peerjs typing) // if rejected, then terminate the call
peer.on('error', e => app.debug.warn('Peer error: ', e.type, e)) if (!agreed) {
peer.on('disconnected', () => { initiateCallEnd()
if (peerReconnectAttempts < 30) { this.options.onCallDeny?.()
this.peerReconnectTimeout = setTimeout(() => { return
if (this.app.active() && !peer.destroyed) {
peer.reconnect()
}
}, Math.min(peerReconnectAttempts, 8) * 2 * 1000)
peerReconnectAttempts += 1
} }
}) if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
callUI.setVideoToggleCallback((args: { enabled: boolean }) =>
const requestCallConfirm = () => { this.emit('videofeed', { streamId: from, enabled: args.enabled })
if (callConfirmAnswer) { // Already asking );
return callConfirmAnswer
} }
callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || { // show buttons in the call window
text: this.options.confirmText, callUI.showControls(initiateCallEnd)
style: this.options.confirmStyle, if (!annot) {
})) // TODO: reuse ? annot = new AnnotationCanvas()
return callConfirmAnswer = callConfirmWindow.mount().then(answer => { annot.mount()
closeCallConfirmWindow()
return answer
})
}
const initiateCallEnd = () => {
this.emit('call_end')
handleCallEnd()
}
const updateVideoFeed = ({ enabled, }) => this.emit('videofeed', { streamId: this.peer?.id, enabled, })
peer.on('call', (call) => {
app.debug.log('Incoming call from', call.peer)
let confirmAnswer: Promise<boolean>
const callingPeerIds = JSON.parse(sessionStorage.getItem(this.options.session_calling_peer_key) || '[]')
if (callingPeerIds.includes(call.peer) || this.callingState === CallingState.True) {
confirmAnswer = Promise.resolve(true)
} else {
this.setCallingState(CallingState.Requesting)
confirmAnswer = requestCallConfirm()
this.playNotificationSound() // For every new agent during confirmation here
// TODO: only one (latest) timeout
setTimeout(() => {
if (this.callingState !== CallingState.Requesting) { return }
initiateCallEnd()
}, 30000)
} }
confirmAnswer.then(async agreed => { // callUI.setLocalStreams(Object.values(lStreams))
if (!agreed) { try {
initiateCallEnd() // if there are no local streams in lStrems then we set
this.options.onCallDeny?.() if (!lStreams[from]) {
return app.debug.log('starting new stream for', from)
// request a local stream, and set it to lStreams
lStreams[from] = await RequestLocalStream()
} }
// Request local stream for the new connection // we pass the received tracks to Call ui
try {
// lStreams are reusable so fare we don't delete them in the `endAgentCall`
if (!lStreams[call.peer]) {
app.debug.log('starting new stream for', call.peer)
lStreams[call.peer] = await RequestLocalStream()
}
calls[call.peer] = call
} catch (e) {
app.debug.error('Audio media device request error:', e)
initiateCallEnd()
return
}
if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
callUI.setVideoToggleCallback(updateVideoFeed)
}
callUI.showControls(initiateCallEnd)
if (!annot) {
annot = new AnnotationCanvas()
annot.mount()
}
// have to be updated
callUI.setLocalStreams(Object.values(lStreams)) callUI.setLocalStreams(Object.values(lStreams))
} catch (e) {
call.on('error', e => { app.debug.error('Error requesting local stream', e);
app.debug.warn('Call error:', e) // if something didn't work out, we terminate the call
initiateCallEnd() initiateCallEnd();
}) return;
call.on('stream', (rStream) => {
callUI?.addRemoteStream(rStream, call.peer)
const onInteraction = () => { // do only if document.hidden ?
callUI?.playRemote()
document.removeEventListener('click', onInteraction)
}
document.addEventListener('click', onInteraction)
})
// remote video on/off/camera change
lStreams[call.peer].onVideoTrack(vTrack => {
const sender = call.peerConnection.getSenders().find(s => s.track?.kind === 'video')
if (!sender) {
app.debug.warn('No video sender found')
return
}
app.debug.log('sender found:', sender)
void sender.replaceTrack(vTrack)
})
call.answer(lStreams[call.peer].stream)
document.addEventListener('visibilitychange', () => {
initiateCallEnd()
})
this.setCallingState(CallingState.True)
if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() }
const callingPeerIds = Object.keys(calls)
sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIds))
this.emit('UPDATE_SESSION', { agentIds: callingPeerIds, isCallActive: true, })
}).catch(reason => { // in case of Confirm.remove() without user answer (not an error)
app.debug.log(reason)
})
})
}
const startCanvasStream = (stream: MediaStream, id: number) => {
const canvasPID = `${app.getProjectKey()}-${sessionId}-${id}`
if (!this.canvasPeers[id]) {
this.canvasPeers[id] = new safeCastedPeer(canvasPID, peerOptions) as Peer
}
this.canvasPeers[id]?.on('error', (e) => app.debug.error(e))
Object.values(this.agents).forEach(agent => {
if (agent.agentInfo) {
const target = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas`
const connection = this.canvasPeers[id]?.connect(target)
connection?.on('open', () => {
if (agent.agentInfo) {
const call = this.canvasPeers[id]?.call(target, stream.clone())
call?.on('error', app.debug.error)
}
})
connection?.on('error', (e) => app.debug.error(e))
} else {
app.debug.error('Assist: cant establish canvas peer to agent, no agent info')
} }
}) // create a new RTCPeerConnection with ice server config
const pc = new RTCPeerConnection({
iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
});
// get all local tracks and add them to RTCPeerConnection
lStreams[from].stream.getTracks().forEach(track => {
pc.addTrack(track, lStreams[from].stream);
});
// When we receive local ice candidates, we emit them via socket
pc.onicecandidate = (event) => {
if (event.candidate) {
socket.emit('webrtc_call_ice_candidate', { from, candidate: event.candidate });
}
};
// when we get a remote stream, add it to call ui
pc.ontrack = (event) => {
const rStream = event.streams[0];
if (rStream && callUI) {
callUI.addRemoteStream(rStream, from);
const onInteraction = () => {
callUI?.playRemote();
document.removeEventListener('click', onInteraction);
};
document.addEventListener('click', onInteraction);
}
};
// Keep connection with the caller
this.calls.set(from, pc);
// set remote description on incoming request
await pc.setRemoteDescription(new RTCSessionDescription(offer));
// create a response to the incoming request
const answer = await pc.createAnswer();
// set answer as local description
await pc.setLocalDescription(answer);
// set the response as local
socket.emit('webrtc_call_answer', { from, answer });
// If the state changes to an error, we terminate the call
// pc.onconnectionstatechange = () => {
// if (pc.connectionState === 'disconnected' || pc.connectionState === 'failed') {
// initiateCallEnd();
// }
// };
// Update track when local video changes
lStreams[from].onVideoTrack(vTrack => {
const sender = pc.getSenders().find(s => s.track?.kind === 'video');
if (!sender) {
app.debug.warn('No video sender found')
return
}
sender.replaceTrack(vTrack)
})
// if the user closed the tab or switched, then we end the call
document.addEventListener('visibilitychange', () => {
initiateCallEnd()
})
// when everything is set, we change the state to true
this.setCallingState(CallingState.True)
if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() }
const callingPeerIdsNow = Array.from(this.calls.keys())
// in session storage we write down everyone with whom the call is established
sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIdsNow))
this.emit('UPDATE_SESSION', { agentIds: callingPeerIdsNow, isCallActive: true })
} catch (reason) {
app.debug.log(reason);
}
};
// Functions for requesting confirmation, ending a call, notifying, etc.
const requestCallConfirm = () => {
if (callConfirmAnswer) { // If confirmation has already been requested
return callConfirmAnswer;
}
callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || {
text: this.options.confirmText,
style: this.options.confirmStyle,
}));
return callConfirmAnswer = callConfirmWindow.mount().then(answer => {
closeCallConfirmWindow();
return answer;
});
};
const initiateCallEnd = () => {
this.emit('call_end');
handleCallEnd();
};
const startCanvasStream = async (stream: MediaStream, id: number) => {
for (const agent of Object.values(this.agents)) {
if (!agent.agentInfo) return;
const uniqueId = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas-${id}`;
if (!this.canvasPeers[uniqueId]) {
this.canvasPeers[uniqueId] = new RTCPeerConnection({
iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
});
this.setupPeerListeners(uniqueId);
stream.getTracks().forEach((track) => {
this.canvasPeers[uniqueId]?.addTrack(track, stream);
});
// Create SDP offer
const offer = await this.canvasPeers[uniqueId].createOffer();
await this.canvasPeers[uniqueId].setLocalDescription(offer);
// Send offer via signaling server
socket.emit('webrtc_canvas_offer', { offer, id: uniqueId });
}
}
} }
app.nodes.attachNodeCallback((node) => { app.nodes.attachNodeCallback((node) => {
const id = app.nodes.getID(node) const id = app.nodes.getID(node)
if (id && hasTag(node, 'canvas')) { if (id && hasTag(node, 'canvas') && !app.sanitizer.isHidden(id)) {
app.debug.log(`Creating stream for canvas ${id}`) app.debug.log(`Creating stream for canvas ${id}`)
const canvasHandler = new Canvas( const canvasHandler = new Canvas(
node as unknown as HTMLCanvasElement, node as unknown as HTMLCanvasElement,
@ -686,14 +753,30 @@ export default class Assist {
if (!isPresent) { if (!isPresent) {
canvasHandler.stop() canvasHandler.stop()
this.canvasMap.delete(id) this.canvasMap.delete(id)
this.canvasPeers[id]?.destroy() if (this.canvasPeers[id]) {
this.canvasPeers[id] = null this.canvasPeers[id]?.close()
this.canvasPeers[id] = null
}
clearInterval(int) clearInterval(int)
} }
}, 5000) }, 5000)
this.canvasNodeCheckers.set(id, int) this.canvasNodeCheckers.set(id, int)
} }
}) });
}
private setupPeerListeners(id: string) {
const peer = this.canvasPeers[id];
if (!peer) return;
// ICE candidates
peer.onicecandidate = (event) => {
if (event.candidate && this.socket) {
this.socket.emit('webrtc_canvas_ice_candidate', {
candidate: event.candidate,
id,
});
}
};
} }
private playNotificationSound() { private playNotificationSound() {
@ -706,26 +789,32 @@ export default class Assist {
} }
} }
// clear all data
private clean() { private clean() {
// sometimes means new agent connected, so we keep id for control // sometimes means new agent connected, so we keep id for control
this.remoteControl?.releaseControl(false, true) this.remoteControl?.releaseControl(false, true);
if (this.peerReconnectTimeout) { if (this.peerReconnectTimeout) {
clearTimeout(this.peerReconnectTimeout) clearTimeout(this.peerReconnectTimeout)
this.peerReconnectTimeout = null this.peerReconnectTimeout = null
} }
if (this.peer) { this.cleanCanvasConnections();
this.peer.destroy() Object.values(this.calls).forEach(pc => pc.close())
this.app.debug.log('Peer destroyed') this.calls.clear();
}
if (this.socket) { if (this.socket) {
this.socket.disconnect() this.socket.disconnect()
this.app.debug.log('Socket disconnected') this.app.debug.log('Socket disconnected')
} }
this.canvasMap.clear() this.canvasMap.clear()
this.canvasPeers = [] this.canvasPeers = {}
this.canvasNodeCheckers.forEach((int) => clearInterval(int)) this.canvasNodeCheckers.forEach((int) => clearInterval(int))
this.canvasNodeCheckers.clear() this.canvasNodeCheckers.clear()
} }
private cleanCanvasConnections() {
Object.values(this.canvasPeers).forEach(pc => pc?.close())
this.canvasPeers = {}
this.socket?.emit('webrtc_canvas_restart')
}
} }
/** simple peers impl /** simple peers impl

7
tracker/tracker-assist/src/CallWindow.ts
View file

@ -12,6 +12,7 @@ export default class CallWindow {
private videoBtn: HTMLElement | null = null private videoBtn: HTMLElement | null = null
private endCallBtn: HTMLElement | null = null private endCallBtn: HTMLElement | null = null
private agentNameElem: HTMLElement | null = null private agentNameElem: HTMLElement | null = null
private remoteStreamVideoContainerSample: HTMLElement | null = null
private videoContainer: HTMLElement | null = null private videoContainer: HTMLElement | null = null
private vPlaceholder: HTMLElement | null = null private vPlaceholder: HTMLElement | null = null
private remoteControlContainer: HTMLElement | null = null private remoteControlContainer: HTMLElement | null = null
@ -62,7 +63,6 @@ export default class CallWindow {
this.adjustIframeSize() this.adjustIframeSize()
iframe.onload = null iframe.onload = null
} }
// ? // ?
text = text.replace(/href="css/g, `href="${baseHref}/css`) text = text.replace(/href="css/g, `href="${baseHref}/css`)
doc.open() doc.open()
@ -71,8 +71,9 @@ export default class CallWindow {
this.vLocal = doc.getElementById('video-local') as HTMLVideoElement | null this.vLocal = doc.getElementById('video-local') as HTMLVideoElement | null
this.vRemote = doc.getElementById('video-remote') as HTMLVideoElement | null this.vRemote = doc.getElementById('video-remote') as HTMLVideoElement | null
this.videoContainer = doc.getElementById('video-container') this.videoContainer = doc.getElementById('video-container')
this.audioBtn = doc.getElementById('audio-btn') this.audioBtn = doc.getElementById('audio-btn')
if (this.audioBtn) { if (this.audioBtn) {
this.audioBtn.onclick = () => this.toggleAudio() this.audioBtn.onclick = () => this.toggleAudio()
@ -334,4 +335,4 @@ export default class CallWindow {
this.toggleRemoteVideoUI(enabled) this.toggleRemoteVideoUI(enabled)
} }
} }
} }

10
tracker/tracker-assist/src/RemoteControl.ts
View file

@ -63,10 +63,10 @@ export default class RemoteControl {
this.releaseControl(true) this.releaseControl(true)
} }
}) })
.then(() => { .then(() => {
this.confirm?.remove() this.confirm?.remove()
}) })
.catch(e => { .catch(e => {
this.confirm?.remove() this.confirm?.remove()
console.error(e) console.error(e)
}) })
@ -113,7 +113,7 @@ export default class RemoteControl {
scroll = (id, d) => { id === this.agentID && this.mouse?.scroll(d) } scroll = (id, d) => { id === this.agentID && this.mouse?.scroll(d) }
move = (id, xy) => { move = (id, xy) => {
return id === this.agentID && this.mouse?.move(xy) return id === this.agentID && this.mouse?.move(xy)
} }
private focused: HTMLElement | null = null private focused: HTMLElement | null = null
click = (id, xy) => { click = (id, xy) => {

10418
tracker/tracker-redux/.pnp.cjs generated Executable file
View file

File diff suppressed because one or more lines are too long

2126
tracker/tracker-redux/.pnp.loader.mjs generated Normal file
View file

File diff suppressed because it is too large Load diff

BIN
tracker/tracker-redux/.yarn/install-state.gz Normal file
View file

Binary file not shown.

12
tracker/tracker-redux/package.json
View file

@ -28,14 +28,16 @@
"redux": "^4.0.0" "redux": "^4.0.0"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "^7.26.8",
"@openreplay/tracker": "file:../tracker", "@openreplay/tracker": "file:../tracker",
"prettier": "^1.18.2",
"replace-in-files-cli": "^1.0.0",
"typescript": "^4.6.0-dev.20211126",
"@rollup/plugin-babel": "^6.0.4", "@rollup/plugin-babel": "^6.0.4",
"@rollup/plugin-node-resolve": "^15.2.3", "@rollup/plugin-node-resolve": "^15.2.3",
"prettier": "^1.18.2",
"replace-in-files": "^3.0.0", "replace-in-files": "^3.0.0",
"replace-in-files-cli": "^1.0.0",
"rollup": "^4.14.0", "rollup": "^4.14.0",
"rollup-plugin-terser": "^7.0.2" "rollup-plugin-terser": "^7.0.2",
} "typescript": "^4.6.0-dev.20211126"
},
"packageManager": "yarn@4.6.0+sha512.5383cc12567a95f1d668fbe762dfe0075c595b4bfff433be478dbbe24e05251a8e8c3eb992a986667c1d53b6c3a9c85b8398c35a960587fbd9fa3a0915406728"
} }