jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Migrate to webrtc (#3051)

Browse source 
* resolved conflicts

* resolved conflicts

* translated comments

* changed console.log message lang

* changed console to logs

* implementing conference call

* add isAgent flag

* add webrtc handlers

* add conference call

* removed conference calls

* fix lint error

---------

Co-authored-by: Andrey Babushkin <a.babushkin@lemon-ai.com>
This commit is contained in:
Andrey Babushkin 2025-02-27 10:12:27 +01:00 • committed by GitHub
parent c793d9d177
commit fd76f7c302
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 15646 additions and 2532 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
frontend/app/components/Assist/ChatWindow/ChatWindow.tsx
View file

@ -9,7 +9,7 @@ import type { LocalStream } from 'Player';
import { PlayerContext } from 'App/components/Session/playerContext';
export interface Props {
incomeStream: MediaStream[] | null;
incomeStream: { stream: MediaStream, isAgent: boolean }[] | null;
localStream: LocalStream | null;
userId: string;
isPrestart?: boolean;
@ -50,8 +50,8 @@ function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }:
>
{incomeStream ? (
incomeStream.map((stream) => (
<React.Fragment key={stream.id}>
<VideoContainer stream={stream} setRemoteEnabled={setRemoteEnabled} />
<React.Fragment key={stream.stream.id}>
<VideoContainer stream={stream.stream} setRemoteEnabled={setRemoteEnabled} isAgent={stream.isAgent} />
</React.Fragment>
))
) : (
@ -62,6 +62,7 @@ function ChatWindow({ userId, incomeStream, localStream, endCall, isPrestart }:
stream={localStream ? localStream.stream : null}
muted
height={anyRemoteEnabled ? 50 : 'unset'}
local
/>
</div>
</div>

28
frontend/app/components/Assist/components/AssistActions/AssistActions.tsx
View file

@ -80,7 +80,7 @@ function AssistActions({
} = store.get();
const [isPrestart, setPrestart] = useState(false);
const [incomeStream, setIncomeStream] = useState<MediaStream[] | null>([]);
const [incomeStream, setIncomeStream] = useState<{ stream: MediaStream; isAgent: boolean }[] | null>([]);
const [localStream, setLocalStream] = useState<LocalStream | null>(null);
const [callObject, setCallObject] = useState<{ end: () => void } | null>(null);
@ -131,18 +131,25 @@ function AssistActions({
}
}, [peerConnectionStatus]);
const addIncomeStream = (stream: MediaStream) => {
const addIncomeStream = (stream: MediaStream, isAgent: boolean) => {
setIncomeStream((oldState) => {
if (oldState === null) return [stream];
if (!oldState.find((existingStream) => existingStream.id === stream.id)) {
if (oldState === null) return [{ stream, isAgent }];
if (!oldState.find((existingStream) => existingStream.stream.id === stream.id)) {
audioContextManager.mergeAudioStreams(stream);
return [...oldState, stream];
return [...oldState, { stream, isAgent }];
}
return oldState;
});
};
function call(additionalAgentIds?: string[]) {
const removeIncomeStream = (stream: MediaStream) => {
setIncomeStream((prevState) => {
if (!prevState) return [];
return prevState.filter((existingStream) => existingStream.stream.id !== stream.id);
});
};
function call() {
RequestLocalStream()
.then((lStream) => {
setLocalStream(lStream);
@ -152,15 +159,16 @@ function AssistActions({
addIncomeStream,
() => {
player.assistManager.ping(AssistActionsPing.call.end, agentId)
lStream.stop.bind(lStream);
lStream.stop.apply(lStream);
removeIncomeStream(lStream.stream);
},
onReject,
onError
);
setCallObject(callPeer());
if (additionalAgentIds) {
callPeer(additionalAgentIds);
}
// if (additionalAgentIds) {
// callPeer(additionalAgentIds);
// }
})
.catch(onError);
}

25
frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx
View file

@ -5,9 +5,18 @@ interface Props {
muted?: boolean;
height?: number | string;
setRemoteEnabled?: (isEnabled: boolean) => void;
local?: boolean;
isAgent?: boolean;
}
function VideoContainer({ stream, muted = false, height = 280, setRemoteEnabled }: Props) {
function VideoContainer({
stream,
muted = false,
height = 280,
setRemoteEnabled,
local,
isAgent,
}: Props) {
const ref = useRef<HTMLVideoElement>(null);
const [isEnabled, setEnabled] = React.useState(false);
@ -15,14 +24,14 @@ function VideoContainer({ stream, muted = false, height = 280, setRemoteEnabled
if (ref.current) {
ref.current.srcObject = stream;
}
}, [ref.current, stream, stream.getVideoTracks()[0]?.getSettings().width]);
}, [ref.current, stream, stream?.getVideoTracks()[0]?.getSettings().width]);
useEffect(() => {
if (!stream) {
return;
}
const iid = setInterval(() => {
const track = stream.getVideoTracks()[0]
const track = stream.getVideoTracks()[0];
const settings = track?.getSettings();
const isDummyVideoTrack = settings
? settings.width === 2 ||
@ -47,9 +56,19 @@ function VideoContainer({ stream, muted = false, height = 280, setRemoteEnabled
width: isEnabled ? undefined : '0px!important',
height: isEnabled ? undefined : '0px!important',
border: '1px solid grey',
transform: local ? 'scaleX(-1)' : undefined,
}}
>
<video autoPlay ref={ref} muted={muted} style={{ height: height }} />
{isAgent ? (
<div
style={{
position: 'absolute',
}}
>
Agent
</div>
) : null}
</div>
);
}

2
frontend/app/components/Session/Player/LivePlayer/LivePlayerBlockHeader.tsx
View file

@ -87,7 +87,7 @@ function LivePlayerBlockHeader({
</div>
)}
<AssistActions userId={userId} isCallActive={isCallActive} agentIds={agentIds} />
<AssistActions userId={userId} isCallActive={isCallActive} agentIds={agentIds ?? []} />
</div>
</div>
</div>

1
frontend/app/player/web/WebLivePlayer.ts
View file

@ -39,6 +39,7 @@ export default class WebLivePlayer extends WebPlayer {
config,
wpState,
(id) => this.messageManager.getNode(id),
agentId,
uiErrorHandler
);
this.assistManager.connect(session.agentToken!, agentId, projectId);

25
frontend/app/player/web/assist/AssistManager.ts
View file

@ -67,6 +67,7 @@ export default class AssistManager {
...RemoteControl.INITIAL_STATE,
...ScreenRecording.INITIAL_STATE,
};
private agentIds: string[] = [];
// TODO: Session type
constructor(
@ -77,9 +78,10 @@ export default class AssistManager {
private config: RTCIceServer[] | null,
private store: Store<typeof AssistManager.INITIAL_STATE>,
private getNode: MessageManager['getNode'],
public readonly agentId: number,
public readonly uiErrorHandler?: {
error: (msg: string) => void;
}
},
) {}
public getAssistVersion = () => this.assistVersion;
@ -192,6 +194,12 @@ export default class AssistManager {
}),
},
}));
// socket.onAny((event, ...args) => {
// logger.log(`📩 Socket: ${event}`, args);
// });
socket.on('connect', () => {
waitingForMessages = true;
// TODO: reconnect happens frequently on bad network
@ -268,6 +276,10 @@ export default class AssistManager {
}
}
}
if (data.agentIds) {
const filteredAgentIds = this.agentIds.filter((id: string) => id.split('-')[3] !== agentId.toString());
this.agentIds = filteredAgentIds;
}
});
socket.on('SESSION_DISCONNECTED', (e) => {
waitingForMessages = true;
@ -288,7 +300,12 @@ export default class AssistManager {
socket,
this.config,
this.peerID,
this.getAssistVersion
this.getAssistVersion,
{
...this.session.agentInfo,
id: agentId,
},
this.agentIds,
);
this.remoteControl = new RemoteControl(
this.store,
@ -309,7 +326,7 @@ export default class AssistManager {
this.canvasReceiver = new CanvasReceiver(this.peerID, this.config, this.getNode, {
...this.session.agentInfo,
id: agentId,
});
}, socket);
document.addEventListener('visibilitychange', this.onVisChange);
});
@ -318,7 +335,7 @@ export default class AssistManager {
/**
* Sends event ping to stats service
* */
public ping(event: StatsEvent, id: number) {
public ping(event: StatsEvent, id: string) {
this.socket?.emit(event, id);
}

463
frontend/app/player/web/assist/Call.ts
View file

@ -1,10 +1,8 @@
import type Peer from 'peerjs';
import type { MediaConnection } from 'peerjs';
import type { LocalStream } from './LocalStream';
import type { Socket } from './types';
import type { Store } from '../../common/types';
import { userStore } from "App/mstore";
import logger from '@/logger';
export enum CallingState {
NoCall,
@ -19,28 +17,59 @@ export interface State {
currentTab?: string;
}
const WEBRTC_CALL_AGENT_EVENT_TYPES = {
OFFER: 'offer',
ANSWER: 'answer',
ICE_CANDIDATE: 'ice-candidate',
}
export default class Call {
private assistVersion = 1;
static readonly INITIAL_STATE: Readonly<State> = {
calling: CallingState.NoCall,
};
private _peer: Peer | null = null;
private connectionAttempts: number = 0;
private callConnection: MediaConnection[] = [];
private connections: Record<string, RTCPeerConnection> = {};
private connectAttempts = 0;
private videoStreams: Record<string, MediaStreamTrack> = {};
private callID: string;
private agentInCallIds: string[] = [];
constructor(
private store: Store<State & { tabs: Set<string> }>,
private socket: Socket,
private config: RTCIceServer[] | null,
private peerID: string,
private getAssistVersion: () => number
private getAssistVersion: () => number,
private agent: Record<string, any>,
private agentIds: string[],
) {
socket.on('WEBRTC_AGENT_CALL', (data) => {
switch (data.type) {
case WEBRTC_CALL_AGENT_EVENT_TYPES.OFFER:
this.handleOffer(data, true);
break;
case WEBRTC_CALL_AGENT_EVENT_TYPES.ICE_CANDIDATE:
this.handleIceCandidate(data);
break;
case WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER:
this.handleAnswer(data, true);
default:
break;
}
})
socket.on('UPDATE_SESSION', (data: { data: { agentIds: string[] }}) => {
this.callAgentsInSession({ agentIds: data.data.agentIds });
});
socket.on('call_end', () => {
this.onRemoteCallEnd()
});
socket.on('videofeed', ({ streamId, enabled }) => {
socket.on('videofeed', (data: { data: { streamId: string; enabled: boolean }}) => {
const { streamId, enabled } = data.data;
if (this.videoStreams[streamId]) {
this.videoStreams[streamId].enabled = enabled;
}
@ -56,14 +85,13 @@ export default class Call {
});
socket.on('messages_gz', () => {
if (reconnecting) {
// 'messages' come frequently, so it is better to have Reconnecting
// When the connection is restored, we initiate a re-creation of the connection
this._callSessionPeer();
reconnecting = false;
}
})
socket.on('messages', () => {
if (reconnecting) {
// 'messages' come frequently, so it is better to have Reconnecting
this._callSessionPeer();
reconnecting = false;
}
@ -71,111 +99,239 @@ export default class Call {
socket.on('disconnect', () => {
this.store.update({ calling: CallingState.NoCall });
});
socket.on('webrtc_call_offer', (data: { data: { from: string, offer: RTCSessionDescriptionInit } }) => {
this.handleOffer(data.data);
});
socket.on('webrtc_call_answer', (data: { data: { from: string, answer: RTCSessionDescriptionInit } }) => {
this.handleAnswer(data.data);
});
socket.on('webrtc_call_ice_candidate', (data: { data: { from: string, candidate: RTCIceCandidateInit } }) => {
this.handleIceCandidate({ candidate: data.data.candidate, from: data.data.from });
});
this.assistVersion = this.getAssistVersion();
}
private getPeer(): Promise<Peer> {
if (this._peer && !this._peer.disconnected) {
return Promise.resolve(this._peer);
// CREATE A LOCAL PEER
private async createPeerConnection({ remotePeerId, localPeerId, isAgent }: { remotePeerId: string, isAgent?: boolean, localPeerId?: string }): Promise<RTCPeerConnection> {
// create pc with ice config
const pc = new RTCPeerConnection({
iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
});
// If there is a local stream, add its tracks to the connection
if (this.callArgs && this.callArgs.localStream && this.callArgs.localStream.stream) {
this.callArgs.localStream.stream.getTracks().forEach((track) => {
pc.addTrack(track, this.callArgs!.localStream.stream);
});
}
// @ts-ignore
const urlObject = new URL(window.env.API_EDP || window.location.origin);
// @ts-ignore TODO: set module in ts settings
return import('peerjs').then(({ default: Peer }) => {
if (this.cleaned) {
return Promise.reject('Already cleaned');
// when ice is ready we send it
pc.onicecandidate = (event) => {
if (event.candidate) {
if (isAgent) {
this.socket.emit('WEBRTC_AGENT_CALL', { from: localPeerId, candidate: event.candidate, toAgentId: getSocketIdByCallId(remotePeerId), type: WEBRTC_CALL_AGENT_EVENT_TYPES.ICE_CANDIDATE });
} else {
this.socket.emit('webrtc_call_ice_candidate', { from: remotePeerId, candidate: event.candidate });
}
} else {
logger.log("ICE candidate gathering complete");
}
const peerOpts: Peer.PeerJSOption = {
host: urlObject.hostname,
path: '/assist',
port:
urlObject.port === ''
? location.protocol === 'https:'
? 443
: 80
: parseInt(urlObject.port),
};
if (this.config) {
peerOpts['config'] = {
iceServers: this.config,
//@ts-ignore
sdpSemantics: 'unified-plan',
iceTransportPolicy: 'all',
};
}
const peer = (this._peer = new Peer(peerOpts));
peer.on('call', (call) => {
console.log('getting call from', call.peer);
call.answer(this.callArgs?.localStream.stream);
this.callConnection.push(call);
this.callArgs?.localStream.onVideoTrack((vTrack) => {
const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video');
// when we receive a remote track, we write it to videoStreams[peerId]
pc.ontrack = (event) => {
const stream = event.streams[0];
if (stream && !this.videoStreams[remotePeerId]) {
const clonnedStream = stream.clone();
this.videoStreams[remotePeerId] = clonnedStream.getVideoTracks()[0];
if (this.store.get().calling !== CallingState.OnCall) {
this.store.update({ calling: CallingState.OnCall });
}
if (this.callArgs) {
this.callArgs.onStream(stream, remotePeerId !== this.callID && isAgentId(remotePeerId));
}
}
};
// If the connection is lost, we end the call
pc.onconnectionstatechange = () => {
if (pc.connectionState === "disconnected" || pc.connectionState === "failed") {
this.onRemoteCallEnd();
}
};
// Handle track replacement when local video changes
if (this.callArgs && this.callArgs.localStream) {
this.callArgs.localStream.onVideoTrack((vTrack: MediaStreamTrack) => {
const sender = pc.getSenders().find((s) => s.track?.kind === 'video');
if (!sender) {
console.warn('No video sender found');
logger.warn('No video sender found');
return;
}
sender.replaceTrack(vTrack);
});
call.on('stream', (stream) => {
this.videoStreams[call.peer] = stream.getVideoTracks()[0];
this.callArgs && this.callArgs.onStream(stream);
});
call.on('close', this.onRemoteCallEnd);
call.on('error', (e) => {
console.error('PeerJS error (on call):', e);
this.initiateCallEnd();
this.callArgs && this.callArgs.onError && this.callArgs.onError();
});
});
peer.on('error', (e) => {
if (e.type === 'disconnected') {
return peer.reconnect();
} else if (e.type !== 'peer-unavailable') {
console.error(`PeerJS error (on peer). Type ${e.type}`, e);
}
});
return new Promise((resolve) => {
peer.on('open', () => resolve(peer));
});
});
}
return pc;
}
// ESTABLISHING A CONNECTION
private async _peerConnection({ remotePeerId, isAgent, socketId, localPeerId }: { remotePeerId: string, isAgent?: boolean, socketId?: string, localPeerId?: string }) {
try {
// Create RTCPeerConnection with client
const pc = await this.createPeerConnection({ remotePeerId, localPeerId, isAgent });
this.connections[remotePeerId] = pc;
// Create an SDP offer
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
// Sending offer
if (isAgent) {
this.socket.emit('WEBRTC_AGENT_CALL', { from: localPeerId, offer, toAgentId: socketId, type: WEBRTC_CALL_AGENT_EVENT_TYPES.OFFER });
} else {
this.socket.emit('webrtc_call_offer', { from: remotePeerId, offer });
}
this.connectAttempts = 0;
} catch (e: any) {
logger.error(e);
// Trying to reconnect
const tryReconnect = async (error: any) => {
if (error.type === 'peer-unavailable' && this.connectAttempts < 5) {
this.connectAttempts++;
logger.log('reconnecting', this.connectAttempts);
await new Promise((resolve) => setTimeout(resolve, 250));
await this._peerConnection({ remotePeerId });
} else {
logger.log('error', this.connectAttempts);
this.callArgs?.onError?.('Could not establish a connection with the peer after 5 attempts');
}
};
await tryReconnect(e);
}
}
// Process the received offer to answer
private async handleOffer(data: { from: string, offer: RTCSessionDescriptionInit }, isAgent?: boolean) {
// set to remotePeerId data.from
logger.log("RECEIVED OFFER", data);
const fromCallId = data.from;
let pc = this.connections[fromCallId];
if (!pc) {
if (isAgent) {
this.connections[fromCallId] = await this.createPeerConnection({ remotePeerId: fromCallId, isAgent, localPeerId: this.callID });
pc = this.connections[fromCallId];
} else {
logger.error("No connection found for remote peer", fromCallId);
return;
}
}
try {
// if the connection is not established yet, then set remoteDescription to peer
if (!pc.localDescription) {
await pc.setRemoteDescription(new RTCSessionDescription(data.offer));
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
if (isAgent) {
this.socket.emit('WEBRTC_AGENT_CALL', { from: this.callID, answer, toAgentId: getSocketIdByCallId(fromCallId), type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER });
} else {
this.socket.emit('webrtc_call_answer', { from: fromCallId, answer });
}
} else {
logger.warn("Skipping setRemoteDescription: Already in stable state");
}
} catch (e) {
logger.error("Error setting remote description from answer", e);
this.callArgs?.onError?.(e);
}
}
// Process the received answer to offer
private async handleAnswer(data: { from: string, answer: RTCSessionDescriptionInit }, isAgent?: boolean) {
// set to remotePeerId data.from
logger.log("RECEIVED ANSWER", data);
if (this.agentInCallIds.includes(data.from) && !isAgent) {
return;
}
const callId = data.from;
const pc = this.connections[callId];
if (!pc) {
logger.error("No connection found for remote peer", callId, this.connections);
return;
}
try {
// if the connection is not established yet, then set remoteDescription to peer
if (pc.signalingState !== "stable") {
await pc.setRemoteDescription(new RTCSessionDescription(data.answer));
} else {
logger.warn("Skipping setRemoteDescription: Already in stable state");
}
} catch (e) {
logger.error("Error setting remote description from answer", e);
this.callArgs?.onError?.(e);
}
}
// process the received iceCandidate
private async handleIceCandidate(data: { from: string, candidate: RTCIceCandidateInit }) {
const callId = data.from;
const pc = this.connections[callId];
if (!pc) return;
// if there are ice candidates then add candidate to peer
if (data.candidate && (data.candidate.sdpMid || data.candidate.sdpMLineIndex !== null)) {
try {
await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
} catch (e) {
logger.error("Error adding ICE candidate", e);
}
} else {
logger.warn("Invalid ICE candidate skipped:", data.candidate);
}
}
// handle call ends
private handleCallEnd() {
if (this.store.get().calling !== CallingState.NoCall) this.callArgs && this.callArgs.onCallEnd();
// If the call is not completed, then call onCallEnd
if (this.store.get().calling !== CallingState.NoCall) {
this.callArgs && this.callArgs.onCallEnd();
}
// change state to NoCall
this.store.update({ calling: CallingState.NoCall });
this.callConnection[0] && this.callConnection[0].close();
// Close all created RTCPeerConnection
Object.values(this.connections).forEach((pc) => pc.close());
this.callArgs?.onCallEnd();
// Clear connections
this.connections = {};
this.callArgs = null;
this.videoStreams = {};
this.callArgs = null;
// TODO: We have it separated, right? (check)
//this.toggleAnnotation(false)
}
// Call completion event handler by signal
private onRemoteCallEnd = () => {
if ([CallingState.Requesting, CallingState.Connecting].includes(this.store.get().calling)) {
// If the call has not started yet, then call onReject
this.callArgs && this.callArgs.onReject();
this.callConnection[0] && this.callConnection[0].close();
// Close all connections and reset callArgs
Object.values(this.connections).forEach((pc) => pc.close());
this.connections = {};
this.callArgs?.onCallEnd();
this.store.update({ calling: CallingState.NoCall });
this.callArgs = null;
} else {
// Call the full call completion handler
this.handleCallEnd();
}
};
// Ends the call and sends the call_end signal
initiateCallEnd = async () => {
const userName = userStore.account.name;
this.emitData('call_end', userName);
this.emitData('call_end', this.callID);
this.handleCallEnd();
// TODO: We have it separated, right? (check)
// const remoteControl = this.store.get().remoteControl
// if (remoteControl === RemoteControlStatus.Enabled) {
// this.socket.emit("release_control")
// this.toggleRemoteControl(false)
// }
};
private emitData = (event: string, data?: any) => {
@ -188,7 +344,7 @@ export default class Call {
private callArgs: {
localStream: LocalStream;
onStream: (s: MediaStream) => void;
onStream: (s: MediaStream, isAgent: boolean) => void;
onCallEnd: () => void;
onReject: () => void;
onError?: (arg?: any) => void;
@ -196,7 +352,7 @@ export default class Call {
setCallArgs(
localStream: LocalStream,
onStream: (s: MediaStream) => void,
onStream: (s: MediaStream, isAgent: boolean) => void,
onCallEnd: () => void,
onReject: () => void,
onError?: (e?: any) => void
@ -210,117 +366,88 @@ export default class Call {
};
}
call(thirdPartyPeers?: string[]): { end: () => void } {
if (thirdPartyPeers && thirdPartyPeers.length > 0) {
this.addPeerCall(thirdPartyPeers);
} else {
// Initiates a call
call(): { end: () => void } {
this._callSessionPeer();
}
// this.callAgentsInSession({ agentIds: this.agentInCallIds });
return {
end: this.initiateCallEnd,
};
}
// Notify peers of local video state change
toggleVideoLocalStream(enabled: boolean) {
this.getPeer().then((peer) => {
this.emitData('videofeed', { streamId: peer.id, enabled });
});
this.emitData('videofeed', { streamId: this.callID, enabled });
}
/** Connecting to the other agents that are already
* in the call with the user
*/
addPeerCall(thirdPartyPeers: string[]) {
thirdPartyPeers.forEach((peer) => this._peerConnection(peer));
}
/** Connecting to the app user */
// Calls the method to create a connection with a peer
private _callSessionPeer() {
if (![CallingState.NoCall, CallingState.Reconnecting].includes(this.store.get().calling)) {
return;
}
this.store.update({ calling: CallingState.Connecting });
const tab = this.store.get().currentTab;
if (!this.store.get().currentTab) {
console.warn('No tab data to connect to peer');
if (!tab) {
logger.warn('No tab data to connect to peer');
}
const peerId =
this.getAssistVersion() === 1
? this.peerID
: `${this.peerID}-${tab || Object.keys(this.store.get().tabs)[0]}`;
// Generate a peer identifier depending on the assist version
this.callID = this.getCallId();
const userName = userStore.account.name;
this.emitData('_agent_name', userName);
void this._peerConnection(peerId);
void this._peerConnection({ remotePeerId: this.callID });
}
connectAttempts = 0;
private async _peerConnection(remotePeerId: string) {
try {
const peer = await this.getPeer();
// let canCall = false
const tryReconnect = async (e: any) => {
peer.off('error', tryReconnect)
console.log(e.type, this.connectAttempts);
if (e.type === 'peer-unavailable' && this.connectAttempts < 5) {
this.connectAttempts++;
console.log('reconnecting', this.connectAttempts);
await new Promise((resolve) => setTimeout(resolve, 250));
await this._peerConnection(remotePeerId);
} else {
console.log('error', this.connectAttempts);
this.callArgs?.onError?.('Could not establish a connection with the peer after 5 attempts');
}
}
const call = peer.call(remotePeerId, this.callArgs!.localStream.stream);
peer.on('error', tryReconnect);
peer.on('connection', () => {
this.callConnection.push(call);
this.connectAttempts = 0;
this.callArgs?.localStream.onVideoTrack((vTrack) => {
const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video');
if (!sender) {
console.warn('No video sender found');
return;
}
sender.replaceTrack(vTrack);
private callAgentsInSession({ agentIds }: { agentIds: string[] }) {
if (agentIds) {
const filteredAgentIds = agentIds.filter((id: string) => id.split('-')[3] !== this.agent.id.toString());
const newIds = filteredAgentIds.filter((id: string) => !this.agentInCallIds.includes(id));
const removedIds = this.agentInCallIds.filter((id: string) => !filteredAgentIds.includes(id));
removedIds.forEach((id: string) => this.agentDisconnected(id));
if (this.store.get().calling === CallingState.OnCall) {
newIds.forEach((id: string) => {
const socketId = getSocketIdByCallId(id);
this._peerConnection({ remotePeerId: id, isAgent: true, socketId, localPeerId: this.callID });
});
})
}
call.on('stream', (stream) => {
this.store.get().calling !== CallingState.OnCall &&
this.store.update({ calling: CallingState.OnCall });
this.videoStreams[call.peer] = stream.getVideoTracks()[0];
this.callArgs && this.callArgs.onStream(stream);
});
call.on('close', this.onRemoteCallEnd);
call.on('error', (e) => {
console.error('PeerJS error (on call):', e);
this.initiateCallEnd();
this.callArgs && this.callArgs.onError && this.callArgs.onError();
});
} catch (e) {
console.error(e);
this.agentInCallIds = filteredAgentIds;
}
}
private cleaned: boolean = false;
private getCallId() {
const tab = this.store.get().currentTab;
if (!tab) {
logger.warn('No tab data to connect to peer');
}
// Generate a peer identifier depending on the assist version
return `${this.peerID}-${tab || Array.from(this.store.get().tabs)[0]}-${this.agent.id}-${this.socket.id}-agent`;
}
agentDisconnected(agentId: string) {
this.connections[agentId]?.close();
delete this.connections[agentId];
}
// Method for clearing resources
clean() {
this.cleaned = true; // sometimes cleaned before modules loaded
void this.initiateCallEnd();
if (this._peer) {
console.log('destroying peer...');
const peer = this._peer; // otherwise it calls reconnection on data chan close
this._peer = null;
peer.disconnect();
peer.destroy();
}
Object.values(this.connections).forEach((pc) => pc.close());
this.connections = {};
this.callArgs?.onCallEnd();
}
}
function isAgentId(id: string): boolean {
return id.endsWith('_agent');
}
function getSocketIdByCallId(callId?: string): string | undefined {
const socketIdRegex = /-\d{2}-(.*?)\-agent/;
const match = callId?.match(socketIdRegex);
if (match) {
return match[1];
}
}

124
frontend/app/player/web/assist/CanvasReceiver.ts
View file

@ -1,6 +1,6 @@
import Peer from 'peerjs';
import { VElement } from 'Player/web/managers/DOM/VirtualDOM';
import MessageManager from 'Player/web/MessageManager';
import { Socket } from 'socket.io-client';
let frameCounter = 0;
@ -18,70 +18,102 @@ function draw(
export default class CanvasReceiver {
private streams: Map<string, MediaStream> = new Map();
private peer: Peer | null = null;
// Store RTCPeerConnection for each remote peer
private connections: Map<string, RTCPeerConnection> = new Map();
private cId: string;
// sendSignal – for sending signals (offer/answer/ICE)
constructor(
private readonly peerIdPrefix: string,
private readonly config: RTCIceServer[] | null,
private readonly getNode: MessageManager['getNode'],
private readonly agentInfo: Record<string, any>
private readonly agentInfo: Record<string, any>,
private readonly socket: Socket,
) {
// @ts-ignore
const urlObject = new URL(window.env.API_EDP || window.location.origin);
const peerOpts: Peer.PeerJSOption = {
host: urlObject.hostname,
path: '/assist',
port:
urlObject.port === ''
? location.protocol === 'https:'
? 443
: 80
: parseInt(urlObject.port),
};
if (this.config) {
peerOpts['config'] = {
iceServers: this.config,
//@ts-ignore
sdpSemantics: 'unified-plan',
iceTransportPolicy: 'all',
};
// Form an id like in PeerJS
this.cId = `${this.peerIdPrefix}-${this.agentInfo.id}-canvas`;
this.socket.on('webrtc_canvas_offer', (data: { data: { offer: RTCSessionDescriptionInit, id: string }}) => {
const { offer, id } = data.data;
if (checkId(id, this.cId)) {
this.handleOffer(offer, id);
}
const id = `${this.peerIdPrefix}-${this.agentInfo.id}-canvas`;
const canvasPeer = new Peer(id, peerOpts);
this.peer = canvasPeer;
canvasPeer.on('error', (err) => console.error('canvas peer error', err));
canvasPeer.on('call', (call) => {
call.answer();
const canvasId = call.peer.split('-')[2];
call.on('stream', (stream) => {
});
this.socket.on('webrtc_canvas_ice_candidate', (data: { data: { candidate: RTCIceCandidateInit, id: string }}) => {
const {candidate, id } = data.data;
if (checkId(id, this.cId)) {
this.handleCandidate(candidate, id);
}
});
this.socket.on('webrtc_canvas_restart', () => {
this.clear();
});
}
async handleOffer(offer: RTCSessionDescriptionInit, id: string): Promise<void> {
const pc = new RTCPeerConnection({
iceServers: this.config ? this.config : [{ urls: "stun:stun.l.google.com:19302" }],
});
// Save the connection
this.connections.set(id, pc);
pc.onicecandidate = (event) => {
if (event.candidate) {
this.socket.emit('webrtc_canvas_ice_candidate', ({ candidate: event.candidate, id }));
}
};
pc.ontrack = (event) => {
const stream = event.streams[0];
if (stream) {
// Detect canvasId from remote peer id
const canvasId = id.split('-')[4];
this.streams.set(canvasId, stream);
setTimeout(() => {
const node = this.getNode(parseInt(canvasId, 10));
const videoEl = spawnVideo(
this.streams.get(canvasId)?.clone() as MediaStream,
node as VElement
);
const videoEl = spawnVideo(stream.clone() as MediaStream, node as VElement);
if (node) {
draw(
videoEl,
node.node as HTMLCanvasElement,
(node.node as HTMLCanvasElement).getContext('2d') as CanvasRenderingContext2D
);
} else {
console.log('NODE', canvasId, 'IS NOT FOUND');
}
}, 250);
});
call.on('error', (err) => console.error('canvas call error', err));
});
}
};
await pc.setRemoteDescription(new RTCSessionDescription(offer));
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
this.socket.emit('webrtc_canvas_answer', { answer: answer, id });
}
async handleCandidate(candidate: RTCIceCandidateInit, id: string): Promise<void> {
const pc = this.connections.get(id);
if (pc) {
try {
await pc.addIceCandidate(new RTCIceCandidate(candidate));
} catch (e) {
console.error('Error adding ICE candidate', e);
}
}
}
clear() {
if (this.peer) {
// otherwise it calls reconnection on data chan close
const peer = this.peer;
this.peer = null;
peer.disconnect();
peer.destroy();
}
this.connections.forEach((pc) => {
pc.close();
});
this.connections.clear();
this.streams.clear();
}
}
@ -157,6 +189,10 @@ function spawnDebugVideo(stream: MediaStream, node: VElement) {
});
}
function checkId(id: string, cId: string): boolean {
return id.includes(cId);
}
/** simple peer example
* // @ts-ignore
* const peer = new SLPeer({ initiator: false })

1
frontend/package.json
View file

@ -51,7 +51,6 @@
"mobx": "^6.13.3",
"mobx-persist-store": "^1.1.5",
"mobx-react-lite": "^4.0.7",
"peerjs": "1.3.2",
"prismjs": "^1.29.0",
"rc-time-picker": "^3.7.3",
"react": "^18.2.0",

3375
frontend/yarn.lock
View file

File diff suppressed because it is too large Load diff

BIN
tracker/tracker-assist/.yarn/install-state.gz
View file

Binary file not shown.

405
tracker/tracker-assist/layout/index-chat.html
View file

@ -1,6 +1,7 @@
<!doctype html>
<html lang="en">
<head>
<head>
<!-- Required meta tags -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
@ -13,26 +14,29 @@
margin: 0;
padding: 0;
}
.text-uppercase {
.text-uppercase {
text-transform: uppercase;
}
.connecting-message {
}
.connecting-message {
/* margin-top: 50%; */
font-size: 20px;
color: #aaa;
text-align: center;
display: none;
font-family: sans-serif;
}
}
.status-connecting .connecting-message {
.status-connecting .connecting-message {
/* display: block; */
}
.status-connecting .card {
/* display: none; */
}
}
.card{
.status-connecting .card {
/* display: none; */
}
.card {
font: 14px 'Roboto', sans-serif;
/* min-width: 324px; */
width: 300px;
@ -41,31 +45,31 @@
/* box-shadow: 0 0 10px #aaa; */
border: solid 4px rgba(0, 0, 0, 0.2);
border-radius: .5rem;
}
}
.card-footers {
.card-footers {
display: flex;
border-bottom: solid thin #CCC;
padding: 5px 5px;
justify-content: space-between;
}
}
.card-footers .assist-controls {
.card-footers .assist-controls {
display: flex;
align-items: center;
}
}
.btn-danger {
.btn-danger {
background-color: #CC0000 !important;
color: white;
}
}
.btn-danger:hover {
.btn-danger:hover {
background-color: #FF0000 !important;
color: white;
}
}
.btn {
.btn {
padding: 5px 8px;
font-size: 14px;
border-radius: .5rem;
@ -74,54 +78,61 @@
cursor: pointer;
display: flex;
align-items: center;
}
}
.btn span {
.btn span {
margin-left: 10px;
}
}
.btn:hover {
.btn:hover {
filter: brightness(0.9);
}
}
.card .card-header{
.card .card-header {
cursor: move;
padding: 14px 18px;
display: flex;
justify-content: space-between;
border-bottom: solid thin #ccc;
}
}
#agent-name, #duration{
cursor:default;
}
#agent-name,
#duration {
cursor: default;
}
#video-container {
#video-container {
background-color: rgb(90, 90, 90);
position: relative;
overflow: hidden;
/* width: 300px; */
}
}
#video-container video {
#video-container video {
width: 100% !important;
height: auto;
object-fit: cover;
}
#local-stream, #remote-stream {
/* display:none; */ /* TODO uncomment this line */
}
#video-container.remote #remote-stream {
display: block;
}
#video-container.local {
min-height: 100px;
}
#video-container.local #local-stream {
display: block;
}
}
#local-stream{
#local-stream,
#remote-stream {
/* display:none; */
/* TODO uncomment this line */
}
#video-container.remote #remote-stream {
display: block;
}
#video-container.local {
min-height: 100px;
}
#video-container.local #local-stream {
display: block;
}
#local-stream {
width: 35%;
/* top: 50%; */
/* left: 70%; */
@ -129,153 +140,183 @@
z-index: 99;
bottom: 5px;
right: 5px;
border: thin solid rgba(255,255,255, .3);
border: thin solid rgba(255, 255, 255, .3);
overflow: hidden;
}
}
#audio-btn {
#audio-btn {
margin-right: 10px;
}
#audio-btn .bi-mic {
}
#audio-btn .bi-mic {
fill: #CC0000;
}
#audio-btn .bi-mic-mute {
display:none;
}
#audio-btn:after {
}
#audio-btn .bi-mic-mute {
display: none;
}
#audio-btn:after {
/* text-transform: capitalize; */
color: #CC0000;
content: 'Mute';
padding-left: 5px;
}
#audio-btn.muted .bi-mic-mute {
}
#audio-btn.muted .bi-mic-mute {
display: inline-block;
}
#audio-btn.muted .bi-mic {
display:none;
}
#audio-btn.muted:after {
}
#audio-btn.muted .bi-mic {
display: none;
}
#audio-btn.muted:after {
content: 'Unmute';
padding-left: 5px;
}
}
#video-btn .bi-camera-video {
#video-btn .bi-camera-video {
fill: #CC0000;
}
#video-btn .bi-camera-video-off {
display:none;
}
#video-btn:after {
}
#video-btn .bi-camera-video-off {
display: none;
}
#video-btn:after {
/* text-transform: capitalize; */
color: #CC0000;
content: 'Stop Video';
padding-left: 5px;
}
#video-btn.off:after {
}
#video-btn.off:after {
content: 'Start Video';
padding-left: 5px;
}
#video-btn.off .bi-camera-video-off {
display: inline-block;
}
#video-btn.off .bi-camera-video {
display:none;
}
}
/* CHART */
#chat-card {
#video-btn.off .bi-camera-video-off {
display: inline-block;
}
#video-btn.off .bi-camera-video {
display: none;
}
/* CHART */
#chat-card {
display: flex;
flex-direction: column;
font-size: 14px;
background-color: white;
}
}
#chat-card .chat-messages { display: none; }
#chat-card .chat-input { display: none; }
#chat-card .chat-header .arrow-state { transform: rotate(180deg); }
#chat-card.active .chat-messages { display: flex; }
#chat-card.active .chat-input { display: flex; }
#chat-card.active .chat-header .arrow-state { transform: rotate(0deg); }
#chat-card .chat-messages {
display: none;
}
#chat-card .chat-header {
#chat-card .chat-input {
display: none;
}
#chat-card .chat-header .arrow-state {
transform: rotate(180deg);
}
#chat-card.active .chat-messages {
display: flex;
}
#chat-card.active .chat-input {
display: flex;
}
#chat-card.active .chat-header .arrow-state {
transform: rotate(0deg);
}
#chat-card .chat-header {
border-bottom: solid thin #ccc;
padding: 8px 16px;
display: flex;
justify-content: space-between;
cursor: pointer;
}
}
#chat-card .chat-header .chat-title {
#chat-card .chat-header .chat-title {
display: flex;
align-items: center;
}
}
#chat-card .chat-header .chat-title span {
#chat-card .chat-header .chat-title span {
margin-left: 6px;
}
}
#chat-card .chat-messages {
#chat-card .chat-messages {
padding: 8px 16px;
overflow-y: auto;
height: 250px;
overflow-y: auto;
flex-direction: column;
justify-content: flex-end;
}
}
#chat-card .message-text {
#chat-card .message-text {
padding: 8px 16px;
border-radius: 20px;
color: #666666;
margin-bottom: 2px;
}
}
#chat-card .message .message-text {
#chat-card .message .message-text {
/* max-width: 70%; */
width: fit-content;
}
#chat-card .message {
margin-bottom: 15px;
}
}
#chat-card .chat-messages .message.left .message-text {
#chat-card .message {
margin-bottom: 15px;
}
#chat-card .chat-messages .message.left .message-text {
text-align: left;
background: #D7E2E2;
border-radius: 0px 30px 30px 30px;
}
}
#chat-card .message .message-user {
#chat-card .message .message-user {
font-size: 12px;
font-weight: bold;
color: #999999;
}
#chat-card .message .message-time {
}
#chat-card .message .message-time {
font-size: 12px;
color: #999999;
margin-left: 4px;
}
}
#chat-card .chat-messages .message.right {
#chat-card .chat-messages .message.right {
margin-left: auto;
text-align: right;
}
}
#chat-card .chat-messages .message.right .message-text {
#chat-card .chat-messages .message.right .message-text {
background: #E4E4E4;
box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.15);
border-radius: 30px 30px 0px 30px;
}
}
#chat-card .chat-input {
#chat-card .chat-input {
margin: 10px;
border-radius: .5rem;
box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.15);
background-color: #DDDDDD;
position: relative;
}
}
#chat-card .chat-input .input {
#chat-card .chat-input .input {
width: 100%;
border: none;
border-radius: 0px;
@ -283,9 +324,9 @@
font-size: 16px;
color: #333;
background-color: transparent;
}
}
.send-btn {
.send-btn {
width: 26px;
height: 26px;
background-color: #AAA;
@ -299,18 +340,21 @@
justify-content: center;
margin: auto;
cursor: pointer;
}
.send-btn:hover {
background-color: #999;
}
.send-btn svg {
fill: #DDDDDD;
}
}
.confirm-window .title {
.send-btn:hover {
background-color: #999;
}
.send-btn svg {
fill: #DDDDDD;
}
.confirm-window .title {
margin-bottom: 10px;
}
.confirm-window {
}
.confirm-window {
font: 14px 'Roboto', sans-serif;
padding: 20px;
background-color: #F3F3F3;
@ -319,42 +363,43 @@
width: fit-content;
color: #666666;
display: none;
}
.confirm-window .actions {
}
.confirm-window .actions {
background-color: white;
padding: 10px;
display: flex;
box-shadow: 0px 0px 3.99778px 1.99889px rgba(0, 0, 0, 0.1);
border-radius: 6px;
}
}
.btn-lg {
.btn-lg {
font-size: 14px;
padding: 10px 14px;
}
}
.btn-success {
.btn-success {
background: rgba(0, 167, 47, 1);
color: white;
}
}
/* .btn-error:hover,
/* .btn-error:hover,
.btn-success:hover {
filter: brightness(0.9);
} */
.btn-error {
.btn-error {
background: #FFE9E9;
/* border-color: #d43f3a; */
color: #CC0000;
}
}
</style>
</head>
</head>
<body data-openreplay-hidden>
<body data-openreplay-hidden>
<div id="remote-control-confirm" class="confirm-window">
<div class="title">The agent is requesting remote control</div>
<div class="actions">
@ -367,8 +412,10 @@
<div class="title">Answer the call so the agent can assist.</div>
<div class="actions">
<button class="text-uppercase btn btn-lg btn-success" style="margin-right: 10px">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-telephone" viewBox="0 0 16 16">
<path d="M3.654 1.328a.678.678 0 0 0-1.015-.063L1.605 2.3c-.483.484-.661 1.169-.45 1.77a17.568 17.568 0 0 0 4.168 6.608 17.569 17.569 0 0 0 6.608 4.168c.601.211 1.286.033 1.77-.45l1.034-1.034a.678.678 0 0 0-.063-1.015l-2.307-1.794a.678.678 0 0 0-.58-.122l-2.19.547a1.745 1.745 0 0 1-1.657-.459L5.482 8.062a1.745 1.745 0 0 1-.46-1.657l.548-2.19a.678.678 0 0 0-.122-.58L3.654 1.328zM1.884.511a1.745 1.745 0 0 1 2.612.163L6.29 2.98c.329.423.445.974.315 1.494l-.547 2.19a.678.678 0 0 0 .178.643l2.457 2.457a.678.678 0 0 0 .644.178l2.189-.547a1.745 1.745 0 0 1 1.494.315l2.306 1.794c.829.645.905 1.87.163 2.611l-1.034 1.034c-.74.74-1.846 1.065-2.877.702a18.634 18.634 0 0 1-7.01-4.42 18.634 18.634 0 0 1-4.42-7.009c-.362-1.03-.037-2.137.703-2.877L1.885.511z"/>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-telephone"
viewBox="0 0 16 16">
<path
d="M3.654 1.328a.678.678 0 0 0-1.015-.063L1.605 2.3c-.483.484-.661 1.169-.45 1.77a17.568 17.568 0 0 0 4.168 6.608 17.569 17.569 0 0 0 6.608 4.168c.601.211 1.286.033 1.77-.45l1.034-1.034a.678.678 0 0 0-.063-1.015l-2.307-1.794a.678.678 0 0 0-.58-.122l-2.19.547a1.745 1.745 0 0 1-1.657-.459L5.482 8.062a1.745 1.745 0 0 1-.46-1.657l.548-2.19a.678.678 0 0 0-.122-.58L3.654 1.328zM1.884.511a1.745 1.745 0 0 1 2.612.163L6.29 2.98c.329.423.445.974.315 1.494l-.547 2.19a.678.678 0 0 0 .178.643l2.457 2.457a.678.678 0 0 0 .644.178l2.189-.547a1.745 1.745 0 0 1 1.494.315l2.306 1.794c.829.645.905 1.87.163 2.611l-1.034 1.034c-.74.74-1.846 1.065-2.877.702a18.634 18.634 0 0 1-7.01-4.42 18.634 18.634 0 0 1-4.42-7.009c-.362-1.03-.037-2.137.703-2.877L1.885.511z" />
</svg>
<span>Answer</span>
</button>
@ -382,17 +429,18 @@
<div class="user-info">
<span>Call with</span>
<!-- User Name -->
<span id="agent-name" class="person-name fw-light" >Support Agent</span>
<span id="agent-name" class="person-name fw-light">Support Agent</span>
</div>
<div class="call-duration">
<!--Call Duration. -->
<span id="duration" class="card-subtitle mb-2 text-muted fw-light" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Duration">00:00</span>
<span id="duration" class="card-subtitle mb-2 text-muted fw-light" data-bs-toggle="tooltip"
data-bs-placement="bottom" title="Duration">00:00</span>
</div>
</div>
<div id="video-container" class="card-body bg-dark p-0 d-flex align-items-center position-relative">
<div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow">
<div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow scale-x-[-1]">
<!-- <p class="text-white m-auto text-center">Starting video...</p> -->
<video id="video-local" autoplay muted></video>
<video id="video-local" autoplay muted class="scale-x-[-1]"></video>
</div>
<div id="remote-stream" class="ratio ratio-4x3 m-0 p-0">
@ -404,56 +452,60 @@
<div class="card-footers">
<div class="assist-controls">
<!-- Add class .muted to #audio-btn when user mutes audio -->
<button
href="#"
id="audio-btn"
class="btn btn-light btn-sm text-uppercase me-2"
>
<button href="#" id="audio-btn" class="btn btn-light btn-sm text-uppercase me-2">
<i>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" class="bi bi-mic" viewBox="0 0 16 16">
<path d="M3.5 6.5A.5.5 0 0 1 4 7v1a4 4 0 0 0 8 0V7a.5.5 0 0 1 1 0v1a5 5 0 0 1-4.5 4.975V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 .5-.5z"/>
<path d="M10 8a2 2 0 1 1-4 0V3a2 2 0 1 1 4 0v5zM8 0a3 3 0 0 0-3 3v5a3 3 0 0 0 6 0V3a3 3 0 0 0-3-3z"/>
<path
d="M3.5 6.5A.5.5 0 0 1 4 7v1a4 4 0 0 0 8 0V7a.5.5 0 0 1 1 0v1a5 5 0 0 1-4.5 4.975V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 .5-.5z" />
<path d="M10 8a2 2 0 1 1-4 0V3a2 2 0 1 1 4 0v5zM8 0a3 3 0 0 0-3 3v5a3 3 0 0 0 6 0V3a3 3 0 0 0-3-3z" />
</svg>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" class="bi bi-mic-mute" viewBox="0 0 16 16">
<path d="M13 8c0 .564-.094 1.107-.266 1.613l-.814-.814A4.02 4.02 0 0 0 12 8V7a.5.5 0 0 1 1 0v1zm-5 4c.818 0 1.578-.245 2.212-.667l.718.719a4.973 4.973 0 0 1-2.43.923V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 1 0v1a4 4 0 0 0 4 4zm3-9v4.879l-1-1V3a2 2 0 0 0-3.997-.118l-.845-.845A3.001 3.001 0 0 1 11 3z"/>
<path d="m9.486 10.607-.748-.748A2 2 0 0 1 6 8v-.878l-1-1V8a3 3 0 0 0 4.486 2.607zm-7.84-9.253 12 12 .708-.708-12-12-.708.708z"/>
<path
d="M13 8c0 .564-.094 1.107-.266 1.613l-.814-.814A4.02 4.02 0 0 0 12 8V7a.5.5 0 0 1 1 0v1zm-5 4c.818 0 1.578-.245 2.212-.667l.718.719a4.973 4.973 0 0 1-2.43.923V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 1 0v1a4 4 0 0 0 4 4zm3-9v4.879l-1-1V3a2 2 0 0 0-3.997-.118l-.845-.845A3.001 3.001 0 0 1 11 3z" />
<path
d="m9.486 10.607-.748-.748A2 2 0 0 1 6 8v-.878l-1-1V8a3 3 0 0 0 4.486 2.607zm-7.84-9.253 12 12 .708-.708-12-12-.708.708z" />
</svg>
</i>
</button>
<!--Add class .off to #video-btn when user stops video -->
<button
href="#"
id="video-btn"
class="btn btn-light btn-sm text-uppercase ms-2"
>
<button href="#" id="video-btn" class="btn btn-light btn-sm text-uppercase ms-2">
<i>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-camera-video" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M0 5a2 2 0 0 1 2-2h7.5a2 2 0 0 1 1.983 1.738l3.11-1.382A1 1 0 0 1 16 4.269v7.462a1 1 0 0 1-1.406.913l-3.111-1.382A2 2 0 0 1 9.5 13H2a2 2 0 0 1-2-2V5zm11.5 5.175 3.5 1.556V4.269l-3.5 1.556v4.35zM2 4a1 1 0 0 0-1 1v6a1 1 0 0 0 1 1h7.5a1 1 0 0 0 1-1V5a1 1 0 0 0-1-1H2z"/>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
class="bi bi-camera-video" viewBox="0 0 16 16">
<path fill-rule="evenodd"
d="M0 5a2 2 0 0 1 2-2h7.5a2 2 0 0 1 1.983 1.738l3.11-1.382A1 1 0 0 1 16 4.269v7.462a1 1 0 0 1-1.406.913l-3.111-1.382A2 2 0 0 1 9.5 13H2a2 2 0 0 1-2-2V5zm11.5 5.175 3.5 1.556V4.269l-3.5 1.556v4.35zM2 4a1 1 0 0 0-1 1v6a1 1 0 0 0 1 1h7.5a1 1 0 0 0 1-1V5a1 1 0 0 0-1-1H2z" />
</svg>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-camera-video-off" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M10.961 12.365a1.99 1.99 0 0 0 .522-1.103l3.11 1.382A1 1 0 0 0 16 11.731V4.269a1 1 0 0 0-1.406-.913l-3.111 1.382A2 2 0 0 0 9.5 3H4.272l.714 1H9.5a1 1 0 0 1 1 1v6a1 1 0 0 1-.144.518l.605.847zM1.428 4.18A.999.999 0 0 0 1 5v6a1 1 0 0 0 1 1h5.014l.714 1H2a2 2 0 0 1-2-2V5c0-.675.334-1.272.847-1.634l.58.814zM15 11.73l-3.5-1.555v-4.35L15 4.269v7.462zm-4.407 3.56-10-14 .814-.58 10 14-.814.58z"/>
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
class="bi bi-camera-video-off" viewBox="0 0 16 16">
<path fill-rule="evenodd"
d="M10.961 12.365a1.99 1.99 0 0 0 .522-1.103l3.11 1.382A1 1 0 0 0 16 11.731V4.269a1 1 0 0 0-1.406-.913l-3.111 1.382A2 2 0 0 0 9.5 3H4.272l.714 1H9.5a1 1 0 0 1 1 1v6a1 1 0 0 1-.144.518l.605.847zM1.428 4.18A.999.999 0 0 0 1 5v6a1 1 0 0 0 1 1h5.014l.714 1H2a2 2 0 0 1-2-2V5c0-.675.334-1.272.847-1.634l.58.814zM15 11.73l-3.5-1.555v-4.35L15 4.269v7.462zm-4.407 3.56-10-14 .814-.58 10 14-.814.58z" />
</svg>
</i>
</button>
</div>
<button id="end-call-btn" href="#" class="btn btn-danger btn-sm text-uppercase" style="margin-right: 8px;">End</button>
<button id="end-call-btn" href="#" class="btn btn-danger btn-sm text-uppercase"
style="margin-right: 8px;">End</button>
</div>
<!-- CHAT - add .active class to show the messages and input -->
<div id="chat-card" class="active">
<div class="chat-header">
<div class="chat-title">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="bi bi-chat" viewBox="0 0 16 16">
<path d="M2.678 11.894a1 1 0 0 1 .287.801 10.97 10.97 0 0 1-.398 2c1.395-.323 2.247-.697 2.634-.893a1 1 0 0 1 .71-.074A8.06 8.06 0 0 0 8 14c3.996 0 7-2.807 7-6 0-3.192-3.004-6-7-6S1 4.808 1 8c0 1.468.617 2.83 1.678 3.894zm-.493 3.905a21.682 21.682 0 0 1-.713.129c-.2.032-.352-.176-.273-.362a9.68 9.68 0 0 0 .244-.637l.003-.01c.248-.72.45-1.548.524-2.319C.743 11.37 0 9.76 0 8c0-3.866 3.582-7 8-7s8 3.134 8 7-3.582 7-8 7a9.06 9.06 0 0 1-2.347-.306c-.52.263-1.639.742-3.468 1.105z"/>
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="bi bi-chat"
viewBox="0 0 16 16">
<path
d="M2.678 11.894a1 1 0 0 1 .287.801 10.97 10.97 0 0 1-.398 2c1.395-.323 2.247-.697 2.634-.893a1 1 0 0 1 .71-.074A8.06 8.06 0 0 0 8 14c3.996 0 7-2.807 7-6 0-3.192-3.004-6-7-6S1 4.808 1 8c0 1.468.617 2.83 1.678 3.894zm-.493 3.905a21.682 21.682 0 0 1-.713.129c-.2.032-.352-.176-.273-.362a9.68 9.68 0 0 0 .244-.637l.003-.01c.248-.72.45-1.548.524-2.319C.743 11.37 0 9.76 0 8c0-3.866 3.582-7 8-7s8 3.134 8 7-3.582 7-8 7a9.06 9.06 0 0 1-2.347-.306c-.52.263-1.639.742-3.468 1.105z" />
</svg>
<span>Chat</span>
</div>
<div>
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" class="bi bi-chevron-up arrow-state" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M7.646 4.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1-.708.708L8 5.707l-5.646 5.647a.5.5 0 0 1-.708-.708l6-6z"/>
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" class="bi bi-chevron-up arrow-state"
viewBox="0 0 16 16">
<path fill-rule="evenodd"
d="M7.646 4.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1-.708.708L8 5.707l-5.646 5.647a.5.5 0 0 1-.708-.708l6-6z" />
</svg>
</div>
</div>
@ -478,13 +530,16 @@
<div class="chat-input">
<input type="text" class="input" placeholder="Type a message...">
<div class="send-btn">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" class="bi bi-arrow-right-short" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z"/>
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" class="bi bi-arrow-right-short"
viewBox="0 0 16 16">
<path fill-rule="evenodd"
d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" />
</svg>
</div>
</div>
</div>
</div>
</section>
</body>
</html>
</body>
</html>

5
tracker/tracker-assist/layout/index.html
View file

@ -152,8 +152,9 @@
</div>
</div>
<div id="video-container" class="card-body bg-dark p-0 d-flex align-items-center position-relative">
<div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow">
<video id="video-local" autoplay muted></video>
<div id="local-stream" class="ratio ratio-4x3 rounded m-0 p-0 shadow scale-x-[-1]">
<!-- fix horizontal mirroring -->
<video id="video-local" autoplay muted class="scale-x-[-1]"></video>
</div>
<div id="remote-stream" class="ratio ratio-4x3 m-0 p-0">

1
tracker/tracker-assist/package.json
View file

@ -30,7 +30,6 @@
"dependencies": {
"csstype": "^3.0.10",
"fflate": "^0.8.2",
"peerjs": "1.5.4",
"socket.io-client": "^4.8.1"
},
"peerDependencies": {

453
tracker/tracker-assist/src/Assist.ts
View file

@ -1,25 +1,21 @@
/* eslint-disable @typescript-eslint/no-empty-function */
import type { Socket, } from 'socket.io-client'
import { connect, } from 'socket.io-client'
import Peer, { MediaConnection, } from 'peerjs'
import type { Properties, } from 'csstype'
import { App, } from '@openreplay/tracker'
import type { Socket } from 'socket.io-client'
import { connect } from 'socket.io-client'
import type { Properties } from 'csstype'
import { App } from '@openreplay/tracker'
import RequestLocalStream, { LocalStream, } from './LocalStream.js'
import {hasTag,} from './guards.js'
import RemoteControl, { RCStatus, } from './RemoteControl.js'
import RequestLocalStream, { LocalStream } from './LocalStream.js'
import { hasTag } from './guards.js'
import RemoteControl, { RCStatus } from './RemoteControl.js'
import CallWindow from './CallWindow.js'
import AnnotationCanvas from './AnnotationCanvas.js'
import ConfirmWindow from './ConfirmWindow/ConfirmWindow.js'
import { callConfirmDefault, } from './ConfirmWindow/defaults.js'
import type { Options as ConfirmOptions, } from './ConfirmWindow/defaults.js'
import { callConfirmDefault } from './ConfirmWindow/defaults.js'
import type { Options as ConfirmOptions } from './ConfirmWindow/defaults.js'
import ScreenRecordingState from './ScreenRecordingState.js'
import { pkgVersion, } from './version.js'
import { pkgVersion } from './version.js'
import Canvas from './Canvas.js'
import { gzip, } from 'fflate'
// TODO: fully specified strict check with no-any (everywhere)
// @ts-ignore
const safeCastedPeer = Peer.default || Peer
import { gzip } from 'fflate'
type StartEndCallback = (agentInfo?: Record<string, any>) => ((() => any) | void)
@ -52,26 +48,23 @@ export interface Options {
confirmStyle?: Properties;
config: RTCConfiguration;
serverURL: string
serverURL: string;
callUITemplate?: string;
compressionEnabled: boolean;
/**
* Minimum amount of messages in a batch to trigger compression run
* @default 5000
* */
compressionMinBatchSize: number
*/
compressionMinBatchSize: number;
}
enum CallingState {
Requesting,
True,
False,
}
// TODO typing????
type OptionalCallback = (()=>Record<string, unknown>) | void
type OptionalCallback = (() => Record<string, unknown>) | void
type Agent = {
onDisconnect?: OptionalCallback,
onControlReleased?: OptionalCallback,
@ -84,8 +77,8 @@ export default class Assist {
readonly version = pkgVersion
private socket: Socket | null = null
private peer: Peer | null = null
private canvasPeers: Record<number, Peer | null> = {}
private calls: Map<string, RTCPeerConnection> = new Map();
private canvasPeers: { [id: number]: RTCPeerConnection | null } = {}
private canvasNodeCheckers: Map<number, any> = new Map()
private assistDemandedRestart = false
private callingState: CallingState = CallingState.False
@ -107,9 +100,9 @@ export default class Assist {
session_control_peer_key: '__openreplay_control_peer',
config: null,
serverURL: null,
onCallStart: ()=>{},
onAgentConnect: ()=>{},
onRemoteControlStart: ()=>{},
onCallStart: () => { },
onAgentConnect: () => { },
onRemoteControlStart: () => { },
callConfirm: {},
controlConfirm: {}, // TODO: clear options passing/merging/overwriting
recordingConfirm: {},
@ -198,17 +191,17 @@ export default class Assist {
private readonly setCallingState = (newState: CallingState): void => {
this.callingState = newState
}
private getHost():string{
private getHost(): string {
if (this.options.socketHost) {
return this.options.socketHost
}
if (this.options.serverURL){
if (this.options.serverURL) {
return new URL(this.options.serverURL).host
}
return this.app.getHost()
}
private getBasePrefixUrl(): string{
if (this.options.serverURL){
private getBasePrefixUrl(): string {
if (this.options.serverURL) {
return new URL(this.options.serverURL).pathname
}
return ''
@ -232,7 +225,7 @@ export default class Assist {
// SocketIO
const socket = this.socket = connect(this.getHost(), {
path: this.getBasePrefixUrl()+'/ws-assist/socket',
path: this.getBasePrefixUrl() + '/ws-assist/socket',
query: {
'peerId': peerID,
'identity': 'session',
@ -257,14 +250,19 @@ export default class Assist {
if (args[0] === 'messages' || args[0] === 'UPDATE_SESSION') {
return
}
if (args[0] !== 'webrtc_call_ice_candidate') {
app.debug.log('Socket:', ...args)
};
socket.on('close', (e) => {
app.debug.warn('Socket closed:', e);
})
})
const onGrand = (id: string) => {
if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
}
if (this.remoteControl){
if (this.remoteControl) {
callUI?.showRemoteControl(this.remoteControl.releaseControl)
}
this.agents[id] = { ...this.agents[id], onControlReleased: this.options.onRemoteControlStart(this.agents[id]?.agentInfo), }
@ -274,7 +272,6 @@ export default class Assist {
return callingAgents.get(id)
}
const onRelease = (id?: string | null, isDenied?: boolean) => {
{
if (id) {
const cb = this.agents[id].onControlReleased
delete this.agents[id].onControlReleased
@ -295,7 +292,6 @@ export default class Assist {
this.options.onRemoteControlDeny?.(info || {})
}
}
}
this.remoteControl = new RemoteControl(
this.options,
@ -347,6 +343,7 @@ export default class Assist {
socket.on('stopAnnotation', (id, event) => processEvent(id, event, annot?.stop))
socket.on('NEW_AGENT', (id: string, info: AgentInfo) => {
this.cleanCanvasConnections();
this.agents[id] = {
onDisconnect: this.options.onAgentConnect?.(info),
agentInfo: info, // TODO ?
@ -369,8 +366,10 @@ export default class Assist {
})
}
})
socket.on('AGENTS_CONNECTED', (ids: string[]) => {
ids.forEach(id =>{
this.cleanCanvasConnections();
ids.forEach(id => {
const agentInfo = this.agents[id]?.agentInfo
this.agents[id] = {
agentInfo,
@ -400,37 +399,65 @@ export default class Assist {
this.agents[id]?.onDisconnect?.()
delete this.agents[id]
Object.values(this.calls).forEach(pc => pc.close())
this.calls.clear();
recordingState.stopAgentRecording(id)
endAgentCall(id)
endAgentCall({ socketId: id })
})
socket.on('NO_AGENT', () => {
Object.values(this.agents).forEach(a => a.onDisconnect?.())
this.cleanCanvasConnections();
this.agents = {}
if (recordingState.isActive) recordingState.stopRecording()
})
socket.on('call_end', (id) => {
if (!callingAgents.has(id)) {
app.debug.warn('Received call_end from unknown agent', id)
socket.on('call_end', (socketId, { data: callId }) => {
if (!callingAgents.has(socketId)) {
app.debug.warn('Received call_end from unknown agent', socketId)
return
}
endAgentCall(id)
endAgentCall({ socketId, callId })
})
socket.on('_agent_name', (id, info) => {
if (app.getTabId() !== info.meta.tabId) return
const name = info.data
callingAgents.set(id, name)
if (!this.peer) {
setupPeer()
}
updateCallerNames()
})
socket.on('webrtc_canvas_answer', async (_, data: { answer, id }) => {
const pc = this.canvasPeers[data.id];
if (pc) {
try {
await pc.setRemoteDescription(new RTCSessionDescription(data.answer));
} catch (e) {
app.debug.error('Error adding ICE candidate', e);
}
}
})
socket.on('webrtc_canvas_ice_candidate', async (_, data: { candidate, id }) => {
const pc = this.canvasPeers[data.id];
if (pc) {
try {
await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
} catch (e) {
app.debug.error('Error adding ICE candidate', e);
}
}
})
// If a videofeed arrives, then we show the video in the ui
socket.on('videofeed', (_, info) => {
if (app.getTabId() !== info.meta.tabId) return
const feedState = info.data
callUI?.toggleVideoStream(feedState)
})
socket.on('request_recording', (id, info) => {
if (app.getTabId() !== info.meta.tabId) return
const agentData = info.data
@ -448,29 +475,52 @@ export default class Assist {
}
})
socket.on('webrtc_call_offer', async (_, data: { from: string, offer: RTCSessionDescriptionInit }) => {
if (!this.calls.has(data.from)) {
await handleIncomingCallOffer(data.from, data.offer);
}
});
socket.on('webrtc_call_ice_candidate', async (data: { from: string, candidate: RTCIceCandidateInit }) => {
const pc = this.calls[data.from];
if (pc) {
try {
await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
} catch (e) {
app.debug.error('Error adding ICE candidate', e);
}
}
});
const callingAgents: Map<string, string> = new Map() // !! uses socket.io ID
// TODO: merge peerId & socket.io id (simplest way - send peerId with the name)
const calls: Record<string, MediaConnection> = {} // !! uses peerJS ID
const lStreams: Record<string, LocalStream> = {}
function updateCallerNames() {
callUI?.setAssistentName(callingAgents)
}
function endAgentCall(id: string) {
callingAgents.delete(id)
function endAgentCall({ socketId, callId }: { socketId: string, callId?: string }) {
callingAgents.delete(socketId)
if (callingAgents.size === 0) {
handleCallEnd()
} else {
updateCallerNames()
//TODO: close() specific call and corresponding lStreams (after connecting peerId & socket.io id)
if (callId) {
handleCallEndWithAgent(callId)
}
}
const handleCallEnd = () => { // Complete stop and clear all calls
// Streams
Object.values(calls).forEach(call => call.close())
Object.keys(calls).forEach(peerId => {
delete calls[peerId]
})
}
const handleCallEndWithAgent = (id: string) => {
this.calls.get(id)?.close()
this.calls.delete(id)
}
// call end handling
const handleCallEnd = () => {
Object.values(this.calls).forEach(pc => pc.close())
this.calls.clear();
Object.values(lStreams).forEach((stream) => { stream.stop() })
Object.keys(lStreams).forEach((peerId: string) => { delete lStreams[peerId] })
// UI
@ -484,7 +534,7 @@ export default class Assist {
callUI?.hideControls()
}
this.emit('UPDATE_SESSION', { agentIds: [], isCallActive: false, })
this.emit('UPDATE_SESSION', { agentIds: [], isCallActive: false })
this.setCallingState(CallingState.False)
sessionStorage.removeItem(this.options.session_calling_peer_key)
@ -498,175 +548,192 @@ export default class Assist {
}
}
// PeerJS call (todo: use native WebRTC)
const peerOptions = {
host: this.getHost(),
path: this.getBasePrefixUrl()+'/assist',
port: location.protocol === 'http:' && this.noSecureMode ? 80 : 443,
debug: 2, //appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs.
}
const setupPeer = () => {
if (this.options.config) {
peerOptions['config'] = this.options.config
}
const peer = new safeCastedPeer(peerID, peerOptions) as Peer
this.peer = peer
let peerReconnectAttempts = 0
// @ts-ignore (peerjs typing)
peer.on('error', e => app.debug.warn('Peer error: ', e.type, e))
peer.on('disconnected', () => {
if (peerReconnectAttempts < 30) {
this.peerReconnectTimeout = setTimeout(() => {
if (this.app.active() && !peer.destroyed) {
peer.reconnect()
}
}, Math.min(peerReconnectAttempts, 8) * 2 * 1000)
peerReconnectAttempts += 1
}
})
const requestCallConfirm = () => {
if (callConfirmAnswer) { // Already asking
return callConfirmAnswer
}
callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || {
text: this.options.confirmText,
style: this.options.confirmStyle,
})) // TODO: reuse ?
return callConfirmAnswer = callConfirmWindow.mount().then(answer => {
closeCallConfirmWindow()
return answer
})
}
const initiateCallEnd = () => {
this.emit('call_end')
handleCallEnd()
}
const updateVideoFeed = ({ enabled, }) => this.emit('videofeed', { streamId: this.peer?.id, enabled, })
peer.on('call', (call) => {
app.debug.log('Incoming call from', call.peer)
const handleIncomingCallOffer = async (from: string, offer: RTCSessionDescriptionInit) => {
app.debug.log('handleIncomingCallOffer', from)
let confirmAnswer: Promise<boolean>
const callingPeerIds = JSON.parse(sessionStorage.getItem(this.options.session_calling_peer_key) || '[]')
if (callingPeerIds.includes(call.peer) || this.callingState === CallingState.True) {
// if the caller is already in the list, then we immediately accept the call without ui
if (callingPeerIds.includes(from) || this.callingState === CallingState.True) {
confirmAnswer = Promise.resolve(true)
} else {
// set the state to wait for confirmation
this.setCallingState(CallingState.Requesting)
// call the call confirmation window
confirmAnswer = requestCallConfirm()
this.playNotificationSound() // For every new agent during confirmation here
// sound notification of a call
this.playNotificationSound()
// TODO: only one (latest) timeout
// after 30 seconds we drop the call
setTimeout(() => {
if (this.callingState !== CallingState.Requesting) { return }
initiateCallEnd()
}, 30000)
}
confirmAnswer.then(async agreed => {
try {
// waiting for a decision on accepting the challenge
const agreed = await confirmAnswer
// if rejected, then terminate the call
if (!agreed) {
initiateCallEnd()
this.options.onCallDeny?.()
return
}
// Request local stream for the new connection
try {
// lStreams are reusable so fare we don't delete them in the `endAgentCall`
if (!lStreams[call.peer]) {
app.debug.log('starting new stream for', call.peer)
lStreams[call.peer] = await RequestLocalStream()
}
calls[call.peer] = call
} catch (e) {
app.debug.error('Audio media device request error:', e)
initiateCallEnd()
return
}
if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
callUI.setVideoToggleCallback(updateVideoFeed)
callUI.setVideoToggleCallback((args: { enabled: boolean }) =>
this.emit('videofeed', { streamId: from, enabled: args.enabled })
);
}
// show buttons in the call window
callUI.showControls(initiateCallEnd)
if (!annot) {
annot = new AnnotationCanvas()
annot.mount()
}
// have to be updated
callUI.setLocalStreams(Object.values(lStreams))
call.on('error', e => {
app.debug.warn('Call error:', e)
initiateCallEnd()
})
call.on('stream', (rStream) => {
callUI?.addRemoteStream(rStream, call.peer)
const onInteraction = () => { // do only if document.hidden ?
callUI?.playRemote()
document.removeEventListener('click', onInteraction)
// callUI.setLocalStreams(Object.values(lStreams))
try {
// if there are no local streams in lStrems then we set
if (!lStreams[from]) {
app.debug.log('starting new stream for', from)
// request a local stream, and set it to lStreams
lStreams[from] = await RequestLocalStream()
}
document.addEventListener('click', onInteraction)
})
// we pass the received tracks to Call ui
callUI.setLocalStreams(Object.values(lStreams))
} catch (e) {
app.debug.error('Error requesting local stream', e);
// if something didn't work out, we terminate the call
initiateCallEnd();
return;
}
// create a new RTCPeerConnection with ice server config
const pc = new RTCPeerConnection({
iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
});
// remote video on/off/camera change
lStreams[call.peer].onVideoTrack(vTrack => {
const sender = call.peerConnection.getSenders().find(s => s.track?.kind === 'video')
// get all local tracks and add them to RTCPeerConnection
lStreams[from].stream.getTracks().forEach(track => {
pc.addTrack(track, lStreams[from].stream);
});
// When we receive local ice candidates, we emit them via socket
pc.onicecandidate = (event) => {
if (event.candidate) {
socket.emit('webrtc_call_ice_candidate', { from, candidate: event.candidate });
}
};
// when we get a remote stream, add it to call ui
pc.ontrack = (event) => {
const rStream = event.streams[0];
if (rStream && callUI) {
callUI.addRemoteStream(rStream, from);
const onInteraction = () => {
callUI?.playRemote();
document.removeEventListener('click', onInteraction);
};
document.addEventListener('click', onInteraction);
}
};
// Keep connection with the caller
this.calls.set(from, pc);
// set remote description on incoming request
await pc.setRemoteDescription(new RTCSessionDescription(offer));
// create a response to the incoming request
const answer = await pc.createAnswer();
// set answer as local description
await pc.setLocalDescription(answer);
// set the response as local
socket.emit('webrtc_call_answer', { from, answer });
// If the state changes to an error, we terminate the call
// pc.onconnectionstatechange = () => {
// if (pc.connectionState === 'disconnected' || pc.connectionState === 'failed') {
// initiateCallEnd();
// }
// };
// Update track when local video changes
lStreams[from].onVideoTrack(vTrack => {
const sender = pc.getSenders().find(s => s.track?.kind === 'video');
if (!sender) {
app.debug.warn('No video sender found')
return
}
app.debug.log('sender found:', sender)
void sender.replaceTrack(vTrack)
sender.replaceTrack(vTrack)
})
call.answer(lStreams[call.peer].stream)
// if the user closed the tab or switched, then we end the call
document.addEventListener('visibilitychange', () => {
initiateCallEnd()
})
// when everything is set, we change the state to true
this.setCallingState(CallingState.True)
if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() }
const callingPeerIds = Object.keys(calls)
sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIds))
this.emit('UPDATE_SESSION', { agentIds: callingPeerIds, isCallActive: true, })
}).catch(reason => { // in case of Confirm.remove() without user answer (not an error)
app.debug.log(reason)
})
})
const callingPeerIdsNow = Array.from(this.calls.keys())
// in session storage we write down everyone with whom the call is established
sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIdsNow))
this.emit('UPDATE_SESSION', { agentIds: callingPeerIdsNow, isCallActive: true })
} catch (reason) {
app.debug.log(reason);
}
};
// Functions for requesting confirmation, ending a call, notifying, etc.
const requestCallConfirm = () => {
if (callConfirmAnswer) { // If confirmation has already been requested
return callConfirmAnswer;
}
callConfirmWindow = new ConfirmWindow(callConfirmDefault(this.options.callConfirm || {
text: this.options.confirmText,
style: this.options.confirmStyle,
}));
return callConfirmAnswer = callConfirmWindow.mount().then(answer => {
closeCallConfirmWindow();
return answer;
});
};
const startCanvasStream = (stream: MediaStream, id: number) => {
const canvasPID = `${app.getProjectKey()}-${sessionId}-${id}`
if (!this.canvasPeers[id]) {
this.canvasPeers[id] = new safeCastedPeer(canvasPID, peerOptions) as Peer
}
this.canvasPeers[id]?.on('error', (e) => app.debug.error(e))
const initiateCallEnd = () => {
this.emit('call_end');
handleCallEnd();
};
const startCanvasStream = async (stream: MediaStream, id: number) => {
for (const agent of Object.values(this.agents)) {
if (!agent.agentInfo) return;
const uniqueId = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas-${id}`;
if (!this.canvasPeers[uniqueId]) {
this.canvasPeers[uniqueId] = new RTCPeerConnection({
iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
});
this.setupPeerListeners(uniqueId);
stream.getTracks().forEach((track) => {
this.canvasPeers[uniqueId]?.addTrack(track, stream);
});
// Create SDP offer
const offer = await this.canvasPeers[uniqueId].createOffer();
await this.canvasPeers[uniqueId].setLocalDescription(offer);
// Send offer via signaling server
socket.emit('webrtc_canvas_offer', { offer, id: uniqueId });
Object.values(this.agents).forEach(agent => {
if (agent.agentInfo) {
const target = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas`
const connection = this.canvasPeers[id]?.connect(target)
connection?.on('open', () => {
if (agent.agentInfo) {
const call = this.canvasPeers[id]?.call(target, stream.clone())
call?.on('error', app.debug.error)
}
})
connection?.on('error', (e) => app.debug.error(e))
} else {
app.debug.error('Assist: cant establish canvas peer to agent, no agent info')
}
})
}
app.nodes.attachNodeCallback((node) => {
const id = app.nodes.getID(node)
if (id && hasTag(node, 'canvas')) {
if (id && hasTag(node, 'canvas') && !app.sanitizer.isHidden(id)) {
app.debug.log(`Creating stream for canvas ${id}`)
const canvasHandler = new Canvas(
node as unknown as HTMLCanvasElement,
@ -686,14 +753,30 @@ export default class Assist {
if (!isPresent) {
canvasHandler.stop()
this.canvasMap.delete(id)
this.canvasPeers[id]?.destroy()
if (this.canvasPeers[id]) {
this.canvasPeers[id]?.close()
this.canvasPeers[id] = null
}
clearInterval(int)
}
}, 5000)
this.canvasNodeCheckers.set(id, int)
}
})
});
}
private setupPeerListeners(id: string) {
const peer = this.canvasPeers[id];
if (!peer) return;
// ICE candidates
peer.onicecandidate = (event) => {
if (event.candidate && this.socket) {
this.socket.emit('webrtc_canvas_ice_candidate', {
candidate: event.candidate,
id,
});
}
};
}
private playNotificationSound() {
@ -706,26 +789,32 @@ export default class Assist {
}
}
// clear all data
private clean() {
// sometimes means new agent connected, so we keep id for control
this.remoteControl?.releaseControl(false, true)
this.remoteControl?.releaseControl(false, true);
if (this.peerReconnectTimeout) {
clearTimeout(this.peerReconnectTimeout)
this.peerReconnectTimeout = null
}
if (this.peer) {
this.peer.destroy()
this.app.debug.log('Peer destroyed')
}
this.cleanCanvasConnections();
Object.values(this.calls).forEach(pc => pc.close())
this.calls.clear();
if (this.socket) {
this.socket.disconnect()
this.app.debug.log('Socket disconnected')
}
this.canvasMap.clear()
this.canvasPeers = []
this.canvasPeers = {}
this.canvasNodeCheckers.forEach((int) => clearInterval(int))
this.canvasNodeCheckers.clear()
}
private cleanCanvasConnections() {
Object.values(this.canvasPeers).forEach(pc => pc?.close())
this.canvasPeers = {}
this.socket?.emit('webrtc_canvas_restart')
}
}
/** simple peers impl

3
tracker/tracker-assist/src/CallWindow.ts
View file

@ -12,6 +12,7 @@ export default class CallWindow {
private videoBtn: HTMLElement | null = null
private endCallBtn: HTMLElement | null = null
private agentNameElem: HTMLElement | null = null
private remoteStreamVideoContainerSample: HTMLElement | null = null
private videoContainer: HTMLElement | null = null
private vPlaceholder: HTMLElement | null = null
private remoteControlContainer: HTMLElement | null = null
@ -62,7 +63,6 @@ export default class CallWindow {
this.adjustIframeSize()
iframe.onload = null
}
// ?
text = text.replace(/href="css/g, `href="${baseHref}/css`)
doc.open()
@ -71,6 +71,7 @@ export default class CallWindow {
this.vLocal = doc.getElementById('video-local') as HTMLVideoElement | null
this.vRemote = doc.getElementById('video-remote') as HTMLVideoElement | null
this.videoContainer = doc.getElementById('video-container')
this.audioBtn = doc.getElementById('audio-btn')

10418
tracker/tracker-redux/.pnp.cjs generated Executable file
View file

File diff suppressed because one or more lines are too long

2126
tracker/tracker-redux/.pnp.loader.mjs generated Normal file
View file

File diff suppressed because it is too large Load diff

BIN
tracker/tracker-redux/.yarn/install-state.gz Normal file
View file

Binary file not shown.

12
tracker/tracker-redux/package.json
View file

@ -28,14 +28,16 @@
"redux": "^4.0.0"
},
"devDependencies": {
"@babel/core": "^7.26.8",
"@openreplay/tracker": "file:../tracker",
"prettier": "^1.18.2",
"replace-in-files-cli": "^1.0.0",
"typescript": "^4.6.0-dev.20211126",
"@rollup/plugin-babel": "^6.0.4",
"@rollup/plugin-node-resolve": "^15.2.3",
"prettier": "^1.18.2",
"replace-in-files": "^3.0.0",
"replace-in-files-cli": "^1.0.0",
"rollup": "^4.14.0",
"rollup-plugin-terser": "^7.0.2"
}
"rollup-plugin-terser": "^7.0.2",
"typescript": "^4.6.0-dev.20211126"
},
"packageManager": "yarn@4.6.0+sha512.5383cc12567a95f1d668fbe762dfe0075c595b4bfff433be478dbbe24e05251a8e8c3eb992a986667c1d53b6c3a9c85b8398c35a960587fbd9fa3a0915406728"
}