openreplay/tracker/tracker-assist/src/Assist.ts
Andrey Babushkin 055ff8f64a
Assist remote canvas control (#3287)
* refactor(searchStore): reformat filterMap function parameters (#3166)

- Reformat the parameters of the filterMap function for better readability.
- Comment out the fetchSessions call in clearSearch method to avoid unnecessary session fetch.

* Increment frontend chart version (#3167)

Co-authored-by: GitHub Action <action@github.com>

* refactor(chalice): cleaned code
fix(chalice): fixed session-search-pg sortKey issue
fix(chalice): fixed CH-query-formatter to handle special chars
fix(chalice): fixed /ids response

* feat(auth): implement withCaptcha HOC for consistent reCAPTCHA (#3177)

* feat(auth): implement withCaptcha HOC for consistent reCAPTCHA

This commit refactors the reCAPTCHA implementation across the application
by introducing a Higher Order Component (withCaptcha) that encapsulates
captcha verification logic. The changes:

- Create a reusable withCaptcha HOC in withRecaptcha.tsx
- Refactor Login, ResetPasswordRequest, and CreatePassword components
- Extract SSOLogin into a separate component
- Improve error handling and user feedback
- Standardize loading and verification states across forms
- Make captcha implementation more maintainable and consistent

* feat(auth): support msaas edition for enterprise features

Add msaas to the isEnterprise check alongside ee edition to properly
display enterprise features. Use userStore.isEnterprise in SSOLogin
component instead of directly checking authDetails.edition for
consistent
enterprise status detection.

* Increment frontend chart version (#3179)

Co-authored-by: GitHub Action <action@github.com>

* feat(assist): improved caching mechanism for cluster mode (#3180)

* Increment assist chart version (#3181)

Co-authored-by: GitHub Action <action@github.com>

* ui: fix table column export

* Increment frontend chart version

* fix(auth): remove unnecessary captcha token validation (#3188)

The token validation checks were redundant as the validation is already
handled by the captcha wrapper component. This change simplifies the
password reset flow while maintaining security.

* Increment frontend chart version (#3189)

Co-authored-by: GitHub Action <action@github.com>

* ui: onboarding fixes

* ui: fixes for onboarding ui

* Increment frontend chart version

* feat(helm): add TOKEN_SECRET environment variable

Add TOKEN_SECRET environment variable to HTTP service deployment and
generate a random value for it in vars.yaml.

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* fix(GraphQL): remove unused useTranslation hook (#3200) (#3206)

Co-authored-by: PiRDub <pirddeveloppeur@gmail.com>

* Increment frontend chart version

* chore(http): remove default token_string

scripts/helmcharts/openreplay/charts/http/scripts/entrypoint.sh

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* fix(dashboard): update filter condition in MetricsList

Change the filter type comparison from checking against 'all' to
checking against an empty string. This ensures proper filtering
behavior when filtering metrics in the dashboard component.

* Increment frontend chart version

* ui: shrink icons when no space, adjust player area for events export … (#3217)

* ui: shrink icons when no space, adjust player area for events export panel, fix panel size

* ui: rm log

* Increment frontend chart version

* refactor(chalice): changed user-journey

* Increment chalice chart version

* refactor(auth): separate SSO support from enterprise edition

Add dedicated isSSOSupported property to correctly identify when SSO
authentication is available, properly handling the 'msaas' edition
case separately from enterprise edition checks. This fixes SSO
visibility in the login interface.

* Increment frontend chart version

* UI patches (28.03) (#3231)

* ui: force getting url for location in tabmanagers

* Assist add turn servers (#3229)

* fixed conflicts

* add offers

* add config to sicket query

* add config to sicket query

* add config init

* removed console logs

* removed wrong updates

* fixed conflicts

* add offers

* add config to sicket query

* add config to sicket query

* add config init

* removed console logs

* removed wrong updates

* ui: fix chat draggable, fix default params

---------

Co-authored-by: nick-delirium <nikita@openreplay.com>

* ui: fix spritemap generation for assist sessions

* ui: fix yarnlock

* fix errors

* updated widget link

* resolved conflicts

* updated widget url

---------

Co-authored-by: Andrey Babushkin <55714097+reyand43@users.noreply.github.com>
Co-authored-by: Андрей Бабушкин <andreybabushkin2000@gmail.com>

* fix(init): remove duplicate clone

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* Increment assist chart version

* Increment frontend chart version

* ui: add old devtool filters

* ui: filter keys

* Increment frontend chart version

* ui: fix modules mapper

* ui: fix modules label

* Increment frontend chart version

* ui: fix double fetches for sessions

* Increment frontend chart version

* pulled updates (#3254)

* Increment frontend chart version (#3255)

Co-authored-by: GitHub Action <action@github.com>

* Increment assist chart version (#3256)

Co-authored-by: GitHub Action <action@github.com>

* feat(chalice): added for_spot=True for authenticate_sso (#3259)

* Increment chalice chart version (#3260)

Co-authored-by: GitHub Action <action@github.com>

* Assist patch canvas (#3265)

* add agent info to assist and tracker

* removed AGENTS_CONNECTED event

* Increment frontend chart version (#3266)

Co-authored-by: GitHub Action <action@github.com>

* Increment assist chart version (#3267)

Co-authored-by: GitHub Action <action@github.com>

* resolved conflict

* removed comments

* add global method support

* fix errors

* remove wrong updates

* remove wrong updates

* add onDrag as option

---------

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>
Co-authored-by: Shekar Siri <sshekarsiri@gmail.com>
Co-authored-by: Mehdi Osman <estradino@users.noreply.github.com>
Co-authored-by: GitHub Action <action@github.com>
Co-authored-by: Taha Yassine Kraiem <tahayk2@gmail.com>
Co-authored-by: Alexander <zavorotynskiy@pm.me>
Co-authored-by: nick-delirium <nikita@openreplay.com>
Co-authored-by: rjshrjndrn <rjshrjndrn@gmail.com>
Co-authored-by: PiRDub <pirddeveloppeur@gmail.com>
2025-04-14 11:25:17 +02:00

1024 lines
31 KiB
TypeScript

/* eslint-disable @typescript-eslint/no-empty-function */
import type { Socket } from "socket.io-client";
import { connect } from "socket.io-client";
import type { Properties } from "csstype";
import { App } from "@openreplay/tracker";
import RequestLocalStream, { LocalStream } from "./LocalStream.js";
import { hasTag } from "./guards.js";
import RemoteControl, { RCStatus } from "./RemoteControl.js";
import CallWindow from "./CallWindow.js";
import AnnotationCanvas from "./AnnotationCanvas.js";
import ConfirmWindow from "./ConfirmWindow/ConfirmWindow.js";
import { callConfirmDefault } from "./ConfirmWindow/defaults.js";
import type { Options as ConfirmOptions } from "./ConfirmWindow/defaults.js";
import ScreenRecordingState from "./ScreenRecordingState.js";
import { pkgVersion } from "./version.js";
import Canvas from "./Canvas.js";
import { gzip } from "fflate";
type StartEndCallback = (agentInfo?: Record<string, any>) => (() => any) | void;
interface AgentInfo {
config: string;
email: string;
id: number;
name: string;
peerId: string;
query: string;
socketId?: string;
}
export interface Options {
onAgentConnect: StartEndCallback;
onCallStart: StartEndCallback;
onRemoteControlStart: StartEndCallback;
onRecordingRequest?: (agentInfo: Record<string, any>) => any;
onCallDeny?: () => any;
onRemoteControlDeny?: (agentInfo: Record<string, any>) => any;
onRecordingDeny?: (agentInfo: Record<string, any>) => any;
onDragCamera?: (dx: number, dy: number) => void;
session_calling_peer_key: string;
session_control_peer_key: string;
callConfirm: ConfirmOptions;
controlConfirm: ConfirmOptions;
recordingConfirm: ConfirmOptions;
socketHost?: string;
// @deprecated
confirmText?: string;
// @deprecated
confirmStyle?: Properties;
serverURL: string;
callUITemplate?: string;
compressionEnabled: boolean;
/**
* Minimum amount of messages in a batch to trigger compression run
* @default 5000
*/
compressionMinBatchSize: number;
}
enum CallingState {
Requesting,
True,
False,
}
type OptionalCallback = (() => Record<string, unknown>) | void;
type Agent = {
onDisconnect?: OptionalCallback;
onControlReleased?: OptionalCallback;
agentInfo: AgentInfo | undefined;
//
};
export default class Assist {
readonly version = pkgVersion;
private socket: Socket | null = null;
private calls: Map<string, RTCPeerConnection> = new Map();
private canvasPeers: { [id: number]: RTCPeerConnection | null } = {};
private canvasNodeCheckers: Map<number, any> = new Map();
private assistDemandedRestart = false;
private callingState: CallingState = CallingState.False;
private remoteControl: RemoteControl | null = null;
private peerReconnectTimeout: ReturnType<typeof setTimeout> | null = null;
private agents: Record<string, Agent> = {};
private config: RTCIceServer[] | undefined;
private readonly options: Options;
private readonly canvasMap: Map<number, Canvas> = new Map();
private iceCandidatesBuffer: Map<string, RTCIceCandidateInit[]> = new Map();
constructor(
private readonly app: App,
options?: Partial<Options>,
private readonly noSecureMode: boolean = false
) {
// @ts-ignore
window.__OR_ASSIST_VERSION = this.version;
this.options = Object.assign(
{
session_calling_peer_key: "__openreplay_calling_peer",
session_control_peer_key: "__openreplay_control_peer",
config: null,
serverURL: null,
onCallStart: () => {},
onAgentConnect: () => {},
onRemoteControlStart: () => {},
onDragCamera: () => {},
callConfirm: {},
controlConfirm: {}, // TODO: clear options passing/merging/overwriting
recordingConfirm: {},
socketHost: "",
compressionEnabled: false,
compressionMinBatchSize: 5000,
},
options
);
if (this.app.options.assistSocketHost) {
this.options.socketHost = this.app.options.assistSocketHost;
}
if (document.hidden !== undefined) {
const sendActivityState = (): void =>
this.emit("UPDATE_SESSION", { active: !document.hidden });
app.attachEventListener(
document,
"visibilitychange",
sendActivityState,
false,
false
);
}
const titleNode = document.querySelector("title");
const observer =
titleNode &&
new MutationObserver(() => {
this.emit("UPDATE_SESSION", { pageTitle: document.title });
});
app.addOnUxtCb((uxtId: number) => {
this.emit("UPDATE_SESSION", { uxtId });
});
app.attachStartCallback(() => {
if (this.assistDemandedRestart) {
return;
}
this.onStart();
observer &&
observer.observe(titleNode, {
subtree: true,
characterData: true,
childList: true,
});
});
app.attachStopCallback(() => {
if (this.assistDemandedRestart) {
return;
}
this.clean();
observer && observer.disconnect();
});
app.attachCommitCallback((messages) => {
if (this.agentsConnected) {
const batchSize = messages.length;
// @ts-ignore No need in statistics messages. TODO proper filter
if (
batchSize === 2 &&
// @ts-ignore No need in statistics messages. TODO proper filter
messages[0]._id === 0 &&
// @ts-ignore No need in statistics messages. TODO proper filter
messages[1]._id === 49
) {
return;
}
if (
batchSize > this.options.compressionMinBatchSize &&
this.options.compressionEnabled
) {
const toSend: any[] = [];
if (batchSize > 10000) {
const middle = Math.floor(batchSize / 2);
const firstHalf = messages.slice(0, middle);
const secondHalf = messages.slice(middle);
toSend.push(firstHalf);
toSend.push(secondHalf);
} else {
toSend.push(messages);
}
toSend.forEach((batch) => {
const str = JSON.stringify(batch);
const byteArr = new TextEncoder().encode(str);
gzip(byteArr, { mtime: 0 }, (err, result) => {
if (err) {
this.emit("messages", batch);
} else {
this.emit("messages_gz", result);
}
});
});
} else {
this.emit("messages", messages);
}
}
});
app.session.attachUpdateCallback((sessInfo) =>
this.emit("UPDATE_SESSION", sessInfo)
);
}
private emit(ev: string, args?: any): void {
this.socket &&
this.socket.emit(ev, {
meta: { tabId: this.app.getTabId() },
data: args,
});
}
private get agentsConnected(): boolean {
return Object.keys(this.agents).length > 0;
}
private readonly setCallingState = (newState: CallingState): void => {
this.callingState = newState;
};
private getHost(): string {
if (this.options.socketHost) {
return this.options.socketHost;
}
if (this.options.serverURL) {
return new URL(this.options.serverURL).host;
}
return this.app.getHost();
}
private getBasePrefixUrl(): string {
if (this.options.serverURL) {
return new URL(this.options.serverURL).pathname;
}
return "";
}
private onStart() {
const app = this.app;
const sessionId = app.getSessionID();
// Common for all incoming call requests
let callUI: CallWindow | null = null;
let annot: AnnotationCanvas | null = null;
// TODO: encapsulate
let callConfirmWindow: ConfirmWindow | null = null;
let callConfirmAnswer: Promise<boolean> | null = null;
let callEndCallback: ReturnType<StartEndCallback> | null = null;
if (!sessionId) {
return app.debug.error("No session ID");
}
const peerID = `${app.getProjectKey()}-${sessionId}-${this.app.getTabId()}`;
// SocketIO
const socket = (this.socket = connect(this.getHost(), {
path: this.getBasePrefixUrl() + "/ws-assist/socket",
query: {
peerId: peerID,
identity: "session",
tabId: this.app.getTabId(),
sessionInfo: JSON.stringify({
uxtId: this.app.getUxtId() ?? undefined,
pageTitle: document.title,
active: true,
assistOnly: this.app.socketMode,
...this.app.getSessionInfo(),
}),
},
extraHeaders: {
sessionId,
},
transports: ["websocket"],
withCredentials: true,
reconnection: true,
reconnectionAttempts: 30,
reconnectionDelay: 1000,
reconnectionDelayMax: 25000,
randomizationFactor: 0.5,
}));
socket.onAny((...args) => {
if (args[0] === "messages" || args[0] === "UPDATE_SESSION") {
return;
}
if (args[0] !== "webrtc_call_ice_candidate") {
app.debug.log("Socket:", ...args);
}
socket.on("close", (e) => {
app.debug.warn("Socket closed:", e);
});
});
const onGrand = (id: string) => {
if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate);
}
if (this.remoteControl) {
callUI?.showRemoteControl(this.remoteControl.releaseControl);
}
this.agents[id] = {
...this.agents[id],
onControlReleased: this.options.onRemoteControlStart(
this.agents[id]?.agentInfo
),
};
this.emit("control_granted", id);
annot = new AnnotationCanvas();
annot.mount();
return callingAgents.get(id);
};
const onRelease = (id?: string | null, isDenied?: boolean) => {
if (id) {
const cb = this.agents[id].onControlReleased;
delete this.agents[id].onControlReleased;
typeof cb === "function" && cb();
this.emit("control_rejected", id);
}
if (annot != null) {
annot.remove();
annot = null;
}
callUI?.hideRemoteControl();
if (this.callingState !== CallingState.True) {
callUI?.remove();
callUI = null;
}
if (isDenied) {
const info = id ? this.agents[id]?.agentInfo : {};
this.options.onRemoteControlDeny?.(info || {});
}
};
this.remoteControl = new RemoteControl(
this.options,
onGrand,
(id, isDenied) => onRelease(id, isDenied),
(id) => this.emit("control_busy", id)
);
const onAcceptRecording = () => {
socket.emit("recording_accepted");
};
const onRejectRecording = (agentData: AgentInfo) => {
socket.emit("recording_rejected");
this.options.onRecordingDeny?.(agentData || {});
};
const recordingState = new ScreenRecordingState(
this.options.recordingConfirm
);
function processEvent(
agentId: string,
event: { meta: { tabId: string }; data?: any },
callback?: (id: string, data: any) => void
) {
if (app.getTabId() === event.meta.tabId) {
return callback?.(agentId, event.data);
}
}
if (this.remoteControl !== null) {
socket.on("request_control", (agentId, dataObj) => {
processEvent(agentId, dataObj, this.remoteControl?.requestControl);
});
socket.on("release_control", (agentId, dataObj) => {
processEvent(agentId, dataObj, (_, data) =>
this.remoteControl?.releaseControl(data)
);
});
socket.on("scroll", (id, event) =>
processEvent(id, event, this.remoteControl?.scroll)
);
socket.on("click", (id, event) =>
processEvent(id, event, this.remoteControl?.click)
);
socket.on("move", (id, event) =>
processEvent(id, event, this.remoteControl?.move)
);
socket.on("startDrag", (id, event) =>
processEvent(id, event, this.remoteControl?.startDrag)
);
socket.on("drag", (id, event) =>
processEvent(id, event, this.remoteControl?.drag)
);
socket.on("stopDrag", (id, event) =>
processEvent(id, event, this.remoteControl?.stopDrag)
);
socket.on("focus", (id, event) =>
processEvent(id, event, (clientID, nodeID) => {
const el = app.nodes.getNode(nodeID);
if (el instanceof HTMLElement && this.remoteControl) {
this.remoteControl.focus(clientID, el);
}
})
);
socket.on("input", (id, event) =>
processEvent(id, event, this.remoteControl?.input)
);
}
// TODO: restrict by id
socket.on("moveAnnotation", (id, event) =>
processEvent(id, event, (_, d) => annot && annot.move(d))
);
socket.on("startAnnotation", (id, event) =>
processEvent(id, event, (_, d) => annot?.start(d))
);
socket.on("stopAnnotation", (id, event) =>
processEvent(id, event, annot?.stop)
);
socket.on(
"WEBRTC_CONFIG",
(config: string) => {
if (config) {
this.config = JSON.parse(config)
}
}
);
socket.on("NEW_AGENT", (id: string, info: AgentInfo) => {
this.cleanCanvasConnections();
this.agents[id] = {
onDisconnect: this.options.onAgentConnect?.(info),
agentInfo: info, // TODO ?
};
if (this.app.active()) {
this.assistDemandedRestart = true;
this.app.stop();
this.app.clearBuffers();
this.app.waitStatus(0).then(() => {
this.app.allowAppStart();
setTimeout(() => {
this.app
.start()
.then(() => {
this.assistDemandedRestart = false;
})
.then(() => {
this.remoteControl?.reconnect([id]);
})
.catch((e) => app.debug.error(e));
// TODO: check if it's needed; basically allowing some time for the app to finish everything before starting again
}, 100);
});
}
});
socket.on("AGENTS_INFO_CONNECTED", (agentsInfo: AgentInfo[]) => {
this.cleanCanvasConnections();
agentsInfo.forEach((agentInfo) => {
if (!agentInfo.socketId) return;
this.agents[agentInfo.socketId] = {
agentInfo,
onDisconnect: this.options.onAgentConnect?.(agentInfo),
};
});
if (this.app.active()) {
this.assistDemandedRestart = true;
this.app.stop();
this.app.clearBuffers();
this.app.waitStatus(0).then(() => {
this.app.allowAppStart();
setTimeout(() => {
this.app
.start()
.then(() => {
this.assistDemandedRestart = false;
})
.then(() => {
this.remoteControl?.reconnect(Object.keys(this.agents));
})
.catch((e) => app.debug.error(e));
}, 100);
});
}
});
socket.on("AGENT_DISCONNECTED", (id) => {
this.remoteControl?.releaseControl();
this.agents[id]?.onDisconnect?.();
delete this.agents[id];
Object.values(this.calls).forEach((pc) => pc.close());
this.calls.clear();
recordingState.stopAgentRecording(id);
endAgentCall({ socketId: id });
});
socket.on("NO_AGENT", () => {
Object.values(this.agents).forEach((a) => a.onDisconnect?.());
this.cleanCanvasConnections();
this.agents = {};
if (recordingState.isActive) recordingState.stopRecording();
});
socket.on("call_end", (socketId, msg) => {
if (!callingAgents.has(socketId) || !msg) {
app.debug.warn("Received call_end from unknown agent", socketId);
return;
}
const { data: callId } = msg;
endAgentCall({ socketId, callId });
});
socket.on("_agent_name", (id, info) => {
if (app.getTabId() !== info.meta.tabId) return;
const name = info.data;
callingAgents.set(id, name);
updateCallerNames();
});
socket.on("webrtc_canvas_answer", async (_, data: { answer; id }) => {
const pc = this.canvasPeers[data.id];
if (pc) {
try {
await pc.setRemoteDescription(new RTCSessionDescription(data.answer));
} catch (e) {
app.debug.error("Error adding ICE candidate", e);
}
}
});
socket.on(
"webrtc_canvas_ice_candidate",
async (_, data: { candidate; id }) => {
const pc = this.canvasPeers[data.id];
if (pc) {
try {
await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
} catch (e) {
app.debug.error("Error adding ICE candidate", e);
}
} else {
this.iceCandidatesBuffer.set(
data.id,
this.iceCandidatesBuffer
.get(data.id)
?.concat([data.candidate]) || [data.candidate]
);
}
}
);
// If a videofeed arrives, then we show the video in the ui
socket.on("videofeed", (_, info) => {
if (app.getTabId() !== info.meta.tabId) return;
const feedState = info.data;
callUI?.toggleVideoStream(feedState);
});
socket.on("request_recording", (id, info) => {
if (app.getTabId() !== info.meta.tabId) return;
const agentData = info.data;
if (!recordingState.isActive) {
this.options.onRecordingRequest?.(JSON.parse(agentData));
recordingState.requestRecording(id, onAcceptRecording, () =>
onRejectRecording(agentData)
);
} else {
this.emit("recording_busy");
}
});
socket.on("stop_recording", (id, info) => {
if (app.getTabId() !== info.meta.tabId) return;
if (recordingState.isActive) {
recordingState.stopAgentRecording(id);
}
});
socket.on(
"webrtc_call_offer",
async (_, data: { from: string; offer: RTCSessionDescriptionInit }) => {
if (!this.calls.has(data.from)) {
await handleIncomingCallOffer(data.from, data.offer);
}
}
);
socket.on(
"webrtc_call_ice_candidate",
async (_, data: { from: string; candidate: RTCIceCandidateInit }) => {
const pc = this.calls[data.from];
if (pc) {
try {
await pc.addIceCandidate(new RTCIceCandidate(data.candidate));
} catch (e) {
app.debug.error("Error adding ICE candidate", e);
}
} else {
this.iceCandidatesBuffer.set(
data.from,
this.iceCandidatesBuffer
.get(data.from)
?.concat([data.candidate]) || [data.candidate]
);
}
}
);
const callingAgents: Map<string, string> = new Map(); // !! uses socket.io ID
// TODO: merge peerId & socket.io id (simplest way - send peerId with the name)
const lStreams: Record<string, LocalStream> = {};
function updateCallerNames() {
callUI?.setAssistentName(callingAgents);
}
function endAgentCall({
socketId,
callId,
}: {
socketId: string;
callId?: string;
}) {
callingAgents.delete(socketId);
if (callingAgents.size === 0) {
handleCallEnd();
} else {
updateCallerNames();
if (callId) {
handleCallEndWithAgent(callId);
}
}
}
const handleCallEndWithAgent = (id: string) => {
this.calls.get(id)?.close();
this.calls.delete(id);
};
// call end handling
const handleCallEnd = () => {
Object.values(this.calls).forEach((pc) => pc.close());
this.calls.clear();
Object.values(lStreams).forEach((stream) => {
stream.stop();
});
Object.keys(lStreams).forEach((peerId: string) => {
delete lStreams[peerId];
});
// UI
closeCallConfirmWindow();
if (this.remoteControl?.status === RCStatus.Disabled) {
callUI?.remove();
annot?.remove();
callUI = null;
annot = null;
} else {
callUI?.hideControls();
}
this.emit("UPDATE_SESSION", { agentIds: [], isCallActive: false });
this.setCallingState(CallingState.False);
sessionStorage.removeItem(this.options.session_calling_peer_key);
callEndCallback?.();
};
const closeCallConfirmWindow = () => {
if (callConfirmWindow) {
callConfirmWindow.remove();
callConfirmWindow = null;
callConfirmAnswer = null;
}
};
const renegotiateConnection = async ({
pc,
from,
}: {
pc: RTCPeerConnection;
from: string;
}) => {
try {
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
this.emit("webrtc_call_offer", { from, offer });
} catch (error) {
app.debug.error("Error with renegotiation:", error);
}
};
const handleIncomingCallOffer = async (
from: string,
offer: RTCSessionDescriptionInit
) => {
app.debug.log("handleIncomingCallOffer", from);
let confirmAnswer: Promise<boolean>;
const callingPeerIds = JSON.parse(
sessionStorage.getItem(this.options.session_calling_peer_key) || "[]"
);
// if the caller is already in the list, then we immediately accept the call without ui
if (
callingPeerIds.includes(from) ||
this.callingState === CallingState.True
) {
confirmAnswer = Promise.resolve(true);
} else {
// set the state to wait for confirmation
this.setCallingState(CallingState.Requesting);
// call the call confirmation window
confirmAnswer = requestCallConfirm();
// sound notification of a call
this.playNotificationSound();
// after 30 seconds we drop the call
setTimeout(() => {
if (this.callingState !== CallingState.Requesting) {
return;
}
initiateCallEnd();
}, 30000);
}
try {
// waiting for a decision on accepting the challenge
const agreed = await confirmAnswer;
// if rejected, then terminate the call
if (!agreed) {
initiateCallEnd();
this.options.onCallDeny?.();
return;
}
// create a new RTCPeerConnection with ice server config
const pc = new RTCPeerConnection({
iceServers: this.config,
});
this.calls.set(from, pc);
if (!callUI) {
callUI = new CallWindow(app.debug.error, this.options.callUITemplate);
callUI.setVideoToggleCallback((args: { enabled: boolean }) => {
this.emit("videofeed", { streamId: from, enabled: args.enabled });
});
}
// show buttons in the call window
callUI.showControls(initiateCallEnd);
if (!annot) {
annot = new AnnotationCanvas();
annot.mount();
}
// callUI.setLocalStreams(Object.values(lStreams))
try {
// if there are no local streams in lStrems then we set
if (!lStreams[from]) {
app.debug.log("starting new stream for", from);
// request a local stream, and set it to lStreams
lStreams[from] = await RequestLocalStream(
pc,
renegotiateConnection.bind(null, { pc, from })
);
}
// we pass the received tracks to Call ui
callUI.setLocalStreams(Object.values(lStreams));
} catch (e) {
app.debug.error("Error requesting local stream", e);
// if something didn't work out, we terminate the call
initiateCallEnd();
this.options.onCallDeny?.();
return;
}
// get all local tracks and add them to RTCPeerConnection
// When we receive local ice candidates, we emit them via socket
pc.onicecandidate = (event) => {
if (event.candidate) {
socket.emit("webrtc_call_ice_candidate", {
from,
candidate: event.candidate,
});
}
};
// when we get a remote stream, add it to call ui
pc.ontrack = (event) => {
const rStream = event.streams[0];
if (rStream && callUI) {
callUI.addRemoteStream(rStream, from);
const onInteraction = () => {
callUI?.playRemote();
document.removeEventListener("click", onInteraction);
};
document.addEventListener("click", onInteraction);
}
};
// set remote description on incoming request
await pc.setRemoteDescription(new RTCSessionDescription(offer));
// create a response to the incoming request
const answer = await pc.createAnswer();
// set answer as local description
await pc.setLocalDescription(answer);
// set the response as local
socket.emit("webrtc_call_answer", { from, answer });
this.applyBufferedIceCandidates(from);
// If the state changes to an error, we terminate the call
// pc.onconnectionstatechange = () => {
// if (pc.connectionState === 'disconnected' || pc.connectionState === 'failed') {
// initiateCallEnd();
// }
// };
// Update track when local video changes
lStreams[from].onVideoTrack((vTrack) => {
const sender = pc.getSenders().find((s) => s.track?.kind === "video");
if (!sender) {
app.debug.warn("No video sender found");
return;
}
sender.replaceTrack(vTrack);
});
// if the user closed the tab or switched, then we end the call
document.addEventListener("visibilitychange", () => {
initiateCallEnd();
});
// when everything is set, we change the state to true
this.setCallingState(CallingState.True);
if (!callEndCallback) {
callEndCallback = this.options.onCallStart?.();
}
const callingPeerIdsNow = Array.from(this.calls.keys());
// in session storage we write down everyone with whom the call is established
sessionStorage.setItem(
this.options.session_calling_peer_key,
JSON.stringify(callingPeerIdsNow)
);
this.emit("UPDATE_SESSION", {
agentIds: callingPeerIdsNow,
isCallActive: true,
});
} catch (reason) {
app.debug.log(reason);
}
};
// Functions for requesting confirmation, ending a call, notifying, etc.
const requestCallConfirm = () => {
if (callConfirmAnswer) {
// If confirmation has already been requested
return callConfirmAnswer;
}
callConfirmWindow = new ConfirmWindow(
callConfirmDefault(
this.options.callConfirm || {
text: this.options.confirmText,
style: this.options.confirmStyle,
}
)
);
return (callConfirmAnswer = callConfirmWindow.mount().then((answer) => {
closeCallConfirmWindow();
return answer;
}));
};
const initiateCallEnd = () => {
this.emit("call_end");
handleCallEnd();
};
const startCanvasStream = async (stream: MediaStream, id: number) => {
for (const agent of Object.values(this.agents)) {
if (!agent.agentInfo) return;
const uniqueId = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas-${id}`;
if (!this.canvasPeers[uniqueId]) {
this.canvasPeers[uniqueId] = new RTCPeerConnection({
iceServers: this.config,
});
this.setupPeerListeners(uniqueId);
this.applyBufferedIceCandidates(uniqueId);
stream.getTracks().forEach((track) => {
this.canvasPeers[uniqueId]?.addTrack(track, stream);
});
// Create SDP offer
const offer = await this.canvasPeers[uniqueId].createOffer();
await this.canvasPeers[uniqueId].setLocalDescription(offer);
// Send offer via signaling server
socket.emit("webrtc_canvas_offer", { offer, id: uniqueId });
}
}
};
app.nodes.attachNodeCallback((node) => {
const id = app.nodes.getID(node);
if (id && hasTag(node, "canvas") && !app.sanitizer.isHidden(id)) {
app.debug.log(`Creating stream for canvas ${id}`);
const canvasHandler = new Canvas(
node as unknown as HTMLCanvasElement,
id,
30,
(stream: MediaStream) => {
startCanvasStream(stream, id);
},
app.debug.error
);
this.canvasMap.set(id, canvasHandler);
if (this.canvasNodeCheckers.has(id)) {
clearInterval(this.canvasNodeCheckers.get(id));
}
const int = setInterval(() => {
const isPresent = node.ownerDocument.defaultView && node.isConnected;
if (!isPresent) {
this.stopCanvasStream(id);
clearInterval(int);
}
}, 5000);
this.canvasNodeCheckers.set(id, int);
}
});
}
private setupPeerListeners(id: string) {
const peer = this.canvasPeers[id];
if (!peer) return;
// ICE candidates
peer.onicecandidate = (event) => {
if (event.candidate && this.socket) {
this.socket.emit("webrtc_canvas_ice_candidate", {
candidate: event.candidate,
id,
});
}
};
}
private playNotificationSound() {
if ("Audio" in window) {
new Audio("https://static.openreplay.com/tracker-assist/notification.mp3")
.play()
.catch((e) => {
this.app.debug.warn(e);
});
}
}
// clear all data
private clean() {
// sometimes means new agent connected, so we keep id for control
this.remoteControl?.releaseControl(false, true);
if (this.peerReconnectTimeout) {
clearTimeout(this.peerReconnectTimeout);
this.peerReconnectTimeout = null;
}
this.cleanCanvasConnections();
Object.values(this.calls).forEach((pc) => pc.close());
this.calls.clear();
if (this.socket) {
this.socket.disconnect();
this.app.debug.log("Socket disconnected");
}
this.canvasMap.clear();
this.canvasPeers = {};
this.canvasNodeCheckers.forEach((int) => clearInterval(int));
this.canvasNodeCheckers.clear();
this.iceCandidatesBuffer.clear();
}
private cleanCanvasConnections() {
Object.values(this.canvasPeers).forEach((pc) => pc?.close());
this.canvasPeers = {};
this.socket?.emit("webrtc_canvas_restart");
}
private stopCanvasStream(id: number) {
for (const agent of Object.values(this.agents)) {
if (!agent.agentInfo) return;
const uniqueId = `${agent.agentInfo.peerId}-${agent.agentInfo.id}-canvas-${id}`;
this.socket?.emit("webrtc_canvas_stop", { id: uniqueId });
if (this.canvasPeers[uniqueId]) {
this.canvasPeers[uniqueId]?.close();
delete this.canvasPeers[uniqueId];
this.canvasMap.get(id)?.stop();
this.canvasMap.delete(id);
this.canvasNodeCheckers.get(id) && clearInterval(this.canvasNodeCheckers.get(id));
this.canvasNodeCheckers.delete(id);
}
}
}
private applyBufferedIceCandidates(from) {
const buffer = this.iceCandidatesBuffer.get(from);
if (buffer) {
buffer.forEach((candidate) => {
this.calls.get(from)?.addIceCandidate(new RTCIceCandidate(candidate));
});
this.iceCandidatesBuffer.delete(from);
}
}
}
/** simple peers impl
* const slPeer = new SLPeer({ initiator: true, stream: stream, })
* // slPeer.on('signal', (data: any) => {
* // this.emit('c_signal', { data, id, })
* // })
* // this.socket?.on('c_signal', (tab: string, data: any) => {
* // console.log(data)
* // slPeer.signal(data)
* // })
* // slPeer.on('error', console.error)
* // this.emit('canvas_stream', { canvasId, })
* */