* fix(ui): merge audio tracks on node chain for recording * fix(ui): fix Co-authored-by: Delirium <nikita@openreplay.com>
This commit is contained in:
Kraiem Taha Yassine
GitHub
parent
f6123c1c08
commit
4bb08d2a5e
5 changed files with 421 additions and 349 deletions
|
|
@ -3,12 +3,7 @@ import { Button, Tooltip } from 'UI';
|
|||
import { connect } from 'react-redux';
|
||||
import cn from 'classnames';
|
||||
import ChatWindow from '../../ChatWindow';
|
||||
import {
|
||||
CallingState,
|
||||
ConnectionStatus,
|
||||
RemoteControlStatus,
|
||||
RequestLocalStream,
|
||||
} from 'Player';
|
||||
import { CallingState, ConnectionStatus, RemoteControlStatus, RequestLocalStream } from 'Player';
|
||||
import type { LocalStream } from 'Player';
|
||||
import { PlayerContext, ILivePlayerContext } from 'App/components/Session/playerContext';
|
||||
import { observer } from 'mobx-react-lite';
|
||||
|
|
@ -16,12 +11,14 @@ import { toast } from 'react-toastify';
|
|||
import { confirm } from 'UI';
|
||||
import stl from './AassistActions.module.css';
|
||||
import ScreenRecorder from 'App/components/Session_/ScreenRecorder/ScreenRecorder';
|
||||
import { audioContextManager } from 'App/utils/screenRecorder';
|
||||
|
||||
function onReject() {
|
||||
toast.info(`Call was rejected.`);
|
||||
}
|
||||
|
||||
function onControlReject() {
|
||||
toast.info('Remote control request was rejected by user')
|
||||
toast.info('Remote control request was rejected by user');
|
||||
}
|
||||
|
||||
function onError(e: any) {
|
||||
|
|
@ -47,7 +44,7 @@ function AssistActions({
|
|||
userDisplayName,
|
||||
}: Props) {
|
||||
// @ts-ignore ???
|
||||
const { player, store } = React.useContext<ILivePlayerContext>(PlayerContext)
|
||||
const { player, store } = React.useContext<ILivePlayerContext>(PlayerContext);
|
||||
|
||||
const {
|
||||
assistManager: {
|
||||
|
|
@ -55,17 +52,17 @@ function AssistActions({
|
|||
setCallArgs,
|
||||
requestReleaseRemoteControl,
|
||||
toggleAnnotation,
|
||||
setRemoteControlCallbacks
|
||||
setRemoteControlCallbacks,
|
||||
},
|
||||
toggleUserName,
|
||||
} = player
|
||||
toggleUserName,
|
||||
} = player;
|
||||
const {
|
||||
calling,
|
||||
annotating,
|
||||
peerConnectionStatus,
|
||||
remoteControl: remoteControlStatus,
|
||||
livePlay,
|
||||
} = store.get()
|
||||
} = store.get();
|
||||
|
||||
const [isPrestart, setPrestart] = useState(false);
|
||||
const [incomeStream, setIncomeStream] = useState<MediaStream[] | null>([]);
|
||||
|
|
@ -121,8 +118,9 @@ function AssistActions({
|
|||
|
||||
const addIncomeStream = (stream: MediaStream) => {
|
||||
setIncomeStream((oldState) => {
|
||||
if (oldState === null) return [stream]
|
||||
if (oldState === null) return [stream];
|
||||
if (!oldState.find((existingStream) => existingStream.id === stream.id)) {
|
||||
audioContextManager.mergeAudioStreams(stream);
|
||||
return [...oldState, stream];
|
||||
}
|
||||
return oldState;
|
||||
|
|
@ -133,7 +131,16 @@ function AssistActions({
|
|||
RequestLocalStream()
|
||||
.then((lStream) => {
|
||||
setLocalStream(lStream);
|
||||
setCallArgs(lStream, addIncomeStream, lStream.stop.bind(lStream), onReject, onError);
|
||||
audioContextManager.mergeAudioStreams(lStream.stream);
|
||||
setCallArgs(
|
||||
lStream,
|
||||
addIncomeStream,
|
||||
() => {
|
||||
lStream.stop.bind(lStream);
|
||||
},
|
||||
onReject,
|
||||
onError
|
||||
);
|
||||
setCallObject(callPeer());
|
||||
if (additionalAgentIds) {
|
||||
callPeer(additionalAgentIds);
|
||||
|
|
@ -157,7 +164,7 @@ function AssistActions({
|
|||
};
|
||||
|
||||
const requestControl = () => {
|
||||
setRemoteControlCallbacks({ onReject: onControlReject })
|
||||
setRemoteControlCallbacks({ onReject: onControlReject });
|
||||
if (callRequesting || remoteRequesting) return;
|
||||
requestReleaseRemoteControl();
|
||||
};
|
||||
|
|
@ -249,17 +256,13 @@ function AssistActions({
|
|||
);
|
||||
}
|
||||
|
||||
const con = connect(
|
||||
(state: any) => {
|
||||
const permissions = state.getIn(['user', 'account', 'permissions']) || [];
|
||||
return {
|
||||
hasPermission: permissions.includes('ASSIST_CALL'),
|
||||
isEnterprise: state.getIn(['user', 'account', 'edition']) === 'ee',
|
||||
userDisplayName: state.getIn(['sessions', 'current']).userDisplayName,
|
||||
};
|
||||
}
|
||||
);
|
||||
const con = connect((state: any) => {
|
||||
const permissions = state.getIn(['user', 'account', 'permissions']) || [];
|
||||
return {
|
||||
hasPermission: permissions.includes('ASSIST_CALL'),
|
||||
isEnterprise: state.getIn(['user', 'account', 'edition']) === 'ee',
|
||||
userDisplayName: state.getIn(['sessions', 'current']).userDisplayName,
|
||||
};
|
||||
});
|
||||
|
||||
export default con(
|
||||
observer(AssistActions)
|
||||
);
|
||||
export default con(observer(AssistActions));
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import {audioContextManager} from "App/utils/screenRecorder";
|
||||
import React from 'react';
|
||||
import { useEffect, useState } from 'react';
|
||||
import { connect } from 'react-redux';
|
||||
|
|
@ -82,6 +83,7 @@ function LivePlayer({
|
|||
return () => {
|
||||
if (!location.pathname.includes('multiview') || !location.pathname.includes(usedSession.sessionId)) {
|
||||
console.debug('cleaning live player for', usedSession.sessionId)
|
||||
audioContextManager.clear();
|
||||
playerInst?.clean?.();
|
||||
// @ts-ignore default empty
|
||||
setContextValue(defaultContextValue)
|
||||
|
|
|
|||
|
|
@ -2,293 +2,306 @@ import type Peer from 'peerjs';
|
|||
import type { MediaConnection } from 'peerjs';
|
||||
|
||||
import type { LocalStream } from './LocalStream';
|
||||
import type { Socket } from './types'
|
||||
import type { Store } from '../../common/types'
|
||||
import type { Socket } from './types';
|
||||
import type { Store } from '../../common/types';
|
||||
|
||||
import appStore from 'App/store';
|
||||
|
||||
|
||||
export enum CallingState {
|
||||
NoCall,
|
||||
Connecting,
|
||||
Requesting,
|
||||
Reconnecting,
|
||||
OnCall,
|
||||
NoCall,
|
||||
Connecting,
|
||||
Requesting,
|
||||
Reconnecting,
|
||||
OnCall,
|
||||
}
|
||||
|
||||
export interface State {
|
||||
calling: CallingState;
|
||||
currentTab?: string;
|
||||
calling: CallingState;
|
||||
currentTab?: string;
|
||||
}
|
||||
|
||||
export default class Call {
|
||||
private assistVersion = 1
|
||||
static readonly INITIAL_STATE: Readonly<State> = {
|
||||
calling: CallingState.NoCall
|
||||
}
|
||||
private assistVersion = 1;
|
||||
static readonly INITIAL_STATE: Readonly<State> = {
|
||||
calling: CallingState.NoCall,
|
||||
};
|
||||
|
||||
private _peer: Peer | null = null
|
||||
private connectionAttempts: number = 0
|
||||
private callConnection: MediaConnection[] = []
|
||||
private videoStreams: Record<string, MediaStreamTrack> = {}
|
||||
private _peer: Peer | null = null;
|
||||
private connectionAttempts: number = 0;
|
||||
private callConnection: MediaConnection[] = [];
|
||||
private videoStreams: Record<string, MediaStreamTrack> = {};
|
||||
|
||||
constructor(
|
||||
private store: Store<State>,
|
||||
private socket: Socket,
|
||||
private config: RTCIceServer[] | null,
|
||||
private peerID: string,
|
||||
private getAssistVersion: () => number
|
||||
) {
|
||||
socket.on('call_end', this.onRemoteCallEnd)
|
||||
socket.on('videofeed', ({ streamId, enabled }) => {
|
||||
console.log(streamId, enabled)
|
||||
console.log(this.videoStreams)
|
||||
constructor(
|
||||
private store: Store<State>,
|
||||
private socket: Socket,
|
||||
private config: RTCIceServer[] | null,
|
||||
private peerID: string,
|
||||
private getAssistVersion: () => number
|
||||
) {
|
||||
socket.on('call_end', this.onRemoteCallEnd);
|
||||
socket.on('videofeed', ({ streamId, enabled }) => {
|
||||
console.log(streamId, enabled);
|
||||
console.log(this.videoStreams);
|
||||
if (this.videoStreams[streamId]) {
|
||||
this.videoStreams[streamId].enabled = enabled
|
||||
this.videoStreams[streamId].enabled = enabled;
|
||||
}
|
||||
console.log(this.videoStreams)
|
||||
})
|
||||
let reconnecting = false
|
||||
console.log(this.videoStreams);
|
||||
});
|
||||
let reconnecting = false;
|
||||
socket.on('SESSION_DISCONNECTED', () => {
|
||||
if (this.store.get().calling === CallingState.OnCall) {
|
||||
this.store.update({ calling: CallingState.Reconnecting })
|
||||
reconnecting = true
|
||||
} else if (this.store.get().calling === CallingState.Requesting){
|
||||
this.store.update({ calling: CallingState.NoCall })
|
||||
if (this.store.get().calling === CallingState.OnCall) {
|
||||
this.store.update({ calling: CallingState.Reconnecting });
|
||||
reconnecting = true;
|
||||
} else if (this.store.get().calling === CallingState.Requesting) {
|
||||
this.store.update({ calling: CallingState.NoCall });
|
||||
}
|
||||
})
|
||||
});
|
||||
socket.on('messages', () => {
|
||||
if (reconnecting) { // 'messages' come frequently, so it is better to have Reconnecting
|
||||
this._callSessionPeer()
|
||||
reconnecting = false
|
||||
}
|
||||
})
|
||||
socket.on("disconnect", () => {
|
||||
this.store.update({ calling: CallingState.NoCall })
|
||||
})
|
||||
this.assistVersion = this.getAssistVersion()
|
||||
}
|
||||
|
||||
private getPeer(): Promise<Peer> {
|
||||
if (this._peer && !this._peer.disconnected) { return Promise.resolve(this._peer) }
|
||||
|
||||
// @ts-ignore
|
||||
const urlObject = new URL(window.env.API_EDP || window.location.origin)
|
||||
|
||||
// @ts-ignore TODO: set module in ts settings
|
||||
return import('peerjs').then(({ default: Peer }) => {
|
||||
if (this.cleaned) {return Promise.reject("Already cleaned")}
|
||||
const peerOpts: Peer.PeerJSOption = {
|
||||
host: urlObject.hostname,
|
||||
path: '/assist',
|
||||
port: urlObject.port === "" ? (location.protocol === 'https:' ? 443 : 80 ): parseInt(urlObject.port),
|
||||
}
|
||||
if (this.config) {
|
||||
peerOpts['config'] = {
|
||||
iceServers: this.config,
|
||||
//@ts-ignore
|
||||
sdpSemantics: 'unified-plan',
|
||||
iceTransportPolicy: 'all',
|
||||
};
|
||||
}
|
||||
const peer = this._peer = new Peer(peerOpts)
|
||||
peer.on('call', call => {
|
||||
console.log('getting call from', call.peer)
|
||||
call.answer(this.callArgs.localStream.stream)
|
||||
this.callConnection.push(call)
|
||||
|
||||
this.callArgs.localStream.onVideoTrack(vTrack => {
|
||||
const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video")
|
||||
if (!sender) {
|
||||
console.warn("No video sender found")
|
||||
return
|
||||
}
|
||||
sender.replaceTrack(vTrack)
|
||||
})
|
||||
|
||||
call.on('stream', stream => {
|
||||
this.videoStreams[call.peer] = stream.getVideoTracks()[0]
|
||||
this.callArgs && this.callArgs.onStream(stream)
|
||||
});
|
||||
// call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
|
||||
|
||||
call.on("close", this.onRemoteCallEnd)
|
||||
call.on("error", (e) => {
|
||||
console.error("PeerJS error (on call):", e)
|
||||
this.initiateCallEnd();
|
||||
this.callArgs && this.callArgs.onError && this.callArgs.onError();
|
||||
});
|
||||
})
|
||||
peer.on('error', e => {
|
||||
if (e.type === 'disconnected') {
|
||||
return peer.reconnect()
|
||||
} else if (e.type !== 'peer-unavailable') {
|
||||
console.error(`PeerJS error (on peer). Type ${e.type}`, e);
|
||||
}
|
||||
|
||||
// call-reconnection connected
|
||||
// if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) {
|
||||
// this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT
|
||||
// ? ConnectionStatus.Connecting
|
||||
// : ConnectionStatus.Disconnected);
|
||||
// Reconnect...
|
||||
})
|
||||
|
||||
return new Promise(resolve => {
|
||||
peer.on("open", () => resolve(peer))
|
||||
})
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
||||
private handleCallEnd() {
|
||||
this.callArgs && this.callArgs.onCallEnd()
|
||||
this.callConnection[0] && this.callConnection[0].close()
|
||||
this.store.update({ calling: CallingState.NoCall })
|
||||
this.callArgs = null
|
||||
// TODO: We have it separated, right? (check)
|
||||
//this.toggleAnnotation(false)
|
||||
}
|
||||
private onRemoteCallEnd = () => {
|
||||
if ([CallingState.Requesting, CallingState.Connecting].includes(this.store.get().calling)) {
|
||||
this.callArgs && this.callArgs.onReject()
|
||||
this.callConnection[0] && this.callConnection[0].close()
|
||||
this.store.update({ calling: CallingState.NoCall })
|
||||
this.callArgs = null
|
||||
} else {
|
||||
this.handleCallEnd()
|
||||
}
|
||||
}
|
||||
|
||||
initiateCallEnd = async () => {
|
||||
this.emitData("call_end", appStore.getState().getIn([ 'user', 'account', 'name']))
|
||||
this.handleCallEnd()
|
||||
// TODO: We have it separated, right? (check)
|
||||
// const remoteControl = this.store.get().remoteControl
|
||||
// if (remoteControl === RemoteControlStatus.Enabled) {
|
||||
// this.socket.emit("release_control")
|
||||
// this.toggleRemoteControl(false)
|
||||
// }
|
||||
}
|
||||
|
||||
private emitData = (event: string, data?: any) => {
|
||||
if (this.getAssistVersion() === 1) {
|
||||
this.socket?.emit(event, data)
|
||||
} else {
|
||||
this.socket?.emit(event, { meta: { tabId: this.store.get().currentTab }, data })
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private callArgs: {
|
||||
localStream: LocalStream,
|
||||
onStream: (s: MediaStream)=>void,
|
||||
onCallEnd: () => void,
|
||||
onReject: () => void,
|
||||
onError?: ()=> void,
|
||||
} | null = null
|
||||
|
||||
setCallArgs(
|
||||
localStream: LocalStream,
|
||||
onStream: (s: MediaStream)=>void,
|
||||
onCallEnd: () => void,
|
||||
onReject: () => void,
|
||||
onError?: (e?: any)=> void,
|
||||
) {
|
||||
this.callArgs = {
|
||||
localStream,
|
||||
onStream,
|
||||
onCallEnd,
|
||||
onReject,
|
||||
onError,
|
||||
}
|
||||
}
|
||||
|
||||
call(thirdPartyPeers?: string[]): { end: () => void } {
|
||||
if (thirdPartyPeers && thirdPartyPeers.length > 0) {
|
||||
this.addPeerCall(thirdPartyPeers)
|
||||
} else {
|
||||
this._callSessionPeer()
|
||||
}
|
||||
return {
|
||||
end: this.initiateCallEnd,
|
||||
}
|
||||
}
|
||||
|
||||
toggleVideoLocalStream(enabled: boolean) {
|
||||
this.getPeer().then((peer) => {
|
||||
this.emitData('videofeed', { streamId: peer.id, enabled })
|
||||
})
|
||||
if (reconnecting) {
|
||||
// 'messages' come frequently, so it is better to have Reconnecting
|
||||
this._callSessionPeer();
|
||||
reconnecting = false;
|
||||
}
|
||||
});
|
||||
socket.on('disconnect', () => {
|
||||
this.store.update({ calling: CallingState.NoCall });
|
||||
});
|
||||
this.assistVersion = this.getAssistVersion();
|
||||
}
|
||||
|
||||
private getPeer(): Promise<Peer> {
|
||||
if (this._peer && !this._peer.disconnected) {
|
||||
return Promise.resolve(this._peer);
|
||||
}
|
||||
|
||||
/** Connecting to the other agents that are already
|
||||
* in the call with the user
|
||||
*/
|
||||
addPeerCall(thirdPartyPeers: string[]) {
|
||||
thirdPartyPeers.forEach(peer => this._peerConnection(peer))
|
||||
}
|
||||
// @ts-ignore
|
||||
const urlObject = new URL(window.env.API_EDP || window.location.origin);
|
||||
|
||||
/** Connecting to the app user */
|
||||
private _callSessionPeer() {
|
||||
if (![CallingState.NoCall, CallingState.Reconnecting].includes(this.store.get().calling)) { return }
|
||||
this.store.update({ calling: CallingState.Connecting })
|
||||
const tab = this.store.get().currentTab
|
||||
if (!this.store.get().currentTab) {
|
||||
console.warn('No tab data to connect to peer')
|
||||
}
|
||||
const peerId = this.getAssistVersion() === 1 ? this.peerID : `${this.peerID}-${tab || Object.keys(this.store.get().tabs)[0]}`
|
||||
console.log(peerId, this.getAssistVersion())
|
||||
void this._peerConnection(peerId);
|
||||
this.emitData("_agent_name", appStore.getState().getIn([ 'user', 'account', 'name']))
|
||||
}
|
||||
// @ts-ignore TODO: set module in ts settings
|
||||
return import('peerjs').then(({ default: Peer }) => {
|
||||
if (this.cleaned) {
|
||||
return Promise.reject('Already cleaned');
|
||||
}
|
||||
const peerOpts: Peer.PeerJSOption = {
|
||||
host: urlObject.hostname,
|
||||
path: '/assist',
|
||||
port:
|
||||
urlObject.port === ''
|
||||
? location.protocol === 'https:'
|
||||
? 443
|
||||
: 80
|
||||
: parseInt(urlObject.port),
|
||||
};
|
||||
if (this.config) {
|
||||
peerOpts['config'] = {
|
||||
iceServers: this.config,
|
||||
//@ts-ignore
|
||||
sdpSemantics: 'unified-plan',
|
||||
iceTransportPolicy: 'all',
|
||||
};
|
||||
}
|
||||
const peer = (this._peer = new Peer(peerOpts));
|
||||
peer.on('call', (call) => {
|
||||
console.log('getting call from', call.peer);
|
||||
call.answer(this.callArgs.localStream.stream);
|
||||
this.callConnection.push(call);
|
||||
|
||||
private async _peerConnection(remotePeerId: string) {
|
||||
try {
|
||||
const peer = await this.getPeer();
|
||||
const call = peer.call(remotePeerId, this.callArgs.localStream.stream)
|
||||
this.callConnection.push(call)
|
||||
this.callArgs.localStream.onVideoTrack((vTrack) => {
|
||||
const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video');
|
||||
if (!sender) {
|
||||
console.warn('No video sender found');
|
||||
return;
|
||||
}
|
||||
sender.replaceTrack(vTrack);
|
||||
});
|
||||
|
||||
this.callArgs.localStream.onVideoTrack(vTrack => {
|
||||
const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video")
|
||||
if (!sender) {
|
||||
console.warn("No video sender found")
|
||||
return
|
||||
}
|
||||
sender.replaceTrack(vTrack)
|
||||
})
|
||||
call.on('stream', (stream) => {
|
||||
this.videoStreams[call.peer] = stream.getVideoTracks()[0];
|
||||
this.callArgs && this.callArgs.onStream(stream);
|
||||
});
|
||||
// call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
|
||||
|
||||
call.on('stream', stream => {
|
||||
this.store.get().calling !== CallingState.OnCall && this.store.update({ calling: CallingState.OnCall })
|
||||
call.on('close', this.onRemoteCallEnd);
|
||||
call.on('error', (e) => {
|
||||
console.error('PeerJS error (on call):', e);
|
||||
this.initiateCallEnd();
|
||||
this.callArgs && this.callArgs.onError && this.callArgs.onError();
|
||||
});
|
||||
});
|
||||
peer.on('error', (e) => {
|
||||
if (e.type === 'disconnected') {
|
||||
return peer.reconnect();
|
||||
} else if (e.type !== 'peer-unavailable') {
|
||||
console.error(`PeerJS error (on peer). Type ${e.type}`, e);
|
||||
}
|
||||
|
||||
this.videoStreams[call.peer] = stream.getVideoTracks()[0]
|
||||
// call-reconnection connected
|
||||
// if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) {
|
||||
// this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT
|
||||
// ? ConnectionStatus.Connecting
|
||||
// : ConnectionStatus.Disconnected);
|
||||
// Reconnect...
|
||||
});
|
||||
|
||||
this.callArgs && this.callArgs.onStream(stream)
|
||||
});
|
||||
// call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
|
||||
return new Promise((resolve) => {
|
||||
peer.on('open', () => resolve(peer));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
call.on("close", this.onRemoteCallEnd)
|
||||
call.on("error", (e) => {
|
||||
console.error("PeerJS error (on call):", e)
|
||||
this.initiateCallEnd();
|
||||
this.callArgs && this.callArgs.onError && this.callArgs.onError();
|
||||
});
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
}
|
||||
private handleCallEnd() {
|
||||
this.callArgs && this.callArgs.onCallEnd();
|
||||
this.callConnection[0] && this.callConnection[0].close();
|
||||
this.store.update({ calling: CallingState.NoCall });
|
||||
this.callArgs = null;
|
||||
// TODO: We have it separated, right? (check)
|
||||
//this.toggleAnnotation(false)
|
||||
}
|
||||
|
||||
private cleaned: boolean = false
|
||||
clean() {
|
||||
this.cleaned = true // sometimes cleaned before modules loaded
|
||||
this.initiateCallEnd()
|
||||
if (this._peer) {
|
||||
console.log("destroying peer...")
|
||||
const peer = this._peer; // otherwise it calls reconnection on data chan close
|
||||
this._peer = null;
|
||||
peer.disconnect();
|
||||
peer.destroy();
|
||||
}
|
||||
}
|
||||
}
|
||||
private onRemoteCallEnd = () => {
|
||||
if ([CallingState.Requesting, CallingState.Connecting].includes(this.store.get().calling)) {
|
||||
this.callArgs && this.callArgs.onReject();
|
||||
this.callConnection[0] && this.callConnection[0].close();
|
||||
this.store.update({ calling: CallingState.NoCall });
|
||||
this.callArgs = null;
|
||||
} else {
|
||||
this.handleCallEnd();
|
||||
}
|
||||
};
|
||||
|
||||
initiateCallEnd = async () => {
|
||||
this.emitData('call_end', appStore.getState().getIn(['user', 'account', 'name']));
|
||||
this.handleCallEnd();
|
||||
// TODO: We have it separated, right? (check)
|
||||
// const remoteControl = this.store.get().remoteControl
|
||||
// if (remoteControl === RemoteControlStatus.Enabled) {
|
||||
// this.socket.emit("release_control")
|
||||
// this.toggleRemoteControl(false)
|
||||
// }
|
||||
};
|
||||
|
||||
private emitData = (event: string, data?: any) => {
|
||||
if (this.getAssistVersion() === 1) {
|
||||
this.socket?.emit(event, data);
|
||||
} else {
|
||||
this.socket?.emit(event, { meta: { tabId: this.store.get().currentTab }, data });
|
||||
}
|
||||
};
|
||||
|
||||
private callArgs: {
|
||||
localStream: LocalStream;
|
||||
onStream: (s: MediaStream) => void;
|
||||
onCallEnd: () => void;
|
||||
onReject: () => void;
|
||||
onError?: () => void;
|
||||
} | null = null;
|
||||
|
||||
setCallArgs(
|
||||
localStream: LocalStream,
|
||||
onStream: (s: MediaStream) => void,
|
||||
onCallEnd: () => void,
|
||||
onReject: () => void,
|
||||
onError?: (e?: any) => void
|
||||
) {
|
||||
this.callArgs = {
|
||||
localStream,
|
||||
onStream,
|
||||
onCallEnd,
|
||||
onReject,
|
||||
onError,
|
||||
};
|
||||
}
|
||||
|
||||
call(thirdPartyPeers?: string[]): { end: () => void } {
|
||||
if (thirdPartyPeers && thirdPartyPeers.length > 0) {
|
||||
this.addPeerCall(thirdPartyPeers);
|
||||
} else {
|
||||
this._callSessionPeer();
|
||||
}
|
||||
return {
|
||||
end: this.initiateCallEnd,
|
||||
};
|
||||
}
|
||||
|
||||
toggleVideoLocalStream(enabled: boolean) {
|
||||
this.getPeer().then((peer) => {
|
||||
this.emitData('videofeed', { streamId: peer.id, enabled });
|
||||
});
|
||||
}
|
||||
|
||||
/** Connecting to the other agents that are already
|
||||
* in the call with the user
|
||||
*/
|
||||
addPeerCall(thirdPartyPeers: string[]) {
|
||||
thirdPartyPeers.forEach((peer) => this._peerConnection(peer));
|
||||
}
|
||||
|
||||
/** Connecting to the app user */
|
||||
private _callSessionPeer() {
|
||||
if (![CallingState.NoCall, CallingState.Reconnecting].includes(this.store.get().calling)) {
|
||||
return;
|
||||
}
|
||||
this.store.update({ calling: CallingState.Connecting });
|
||||
const tab = this.store.get().currentTab;
|
||||
if (!this.store.get().currentTab) {
|
||||
console.warn('No tab data to connect to peer');
|
||||
}
|
||||
const peerId =
|
||||
this.getAssistVersion() === 1
|
||||
? this.peerID
|
||||
: `${this.peerID}-${tab || Object.keys(this.store.get().tabs)[0]}`;
|
||||
console.log(peerId, this.getAssistVersion());
|
||||
void this._peerConnection(peerId);
|
||||
this.emitData('_agent_name', appStore.getState().getIn(['user', 'account', 'name']));
|
||||
}
|
||||
|
||||
private async _peerConnection(remotePeerId: string) {
|
||||
try {
|
||||
const peer = await this.getPeer();
|
||||
const call = peer.call(remotePeerId, this.callArgs.localStream.stream);
|
||||
this.callConnection.push(call);
|
||||
|
||||
this.callArgs.localStream.onVideoTrack((vTrack) => {
|
||||
const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video');
|
||||
if (!sender) {
|
||||
console.warn('No video sender found');
|
||||
return;
|
||||
}
|
||||
sender.replaceTrack(vTrack);
|
||||
});
|
||||
|
||||
call.on('stream', (stream) => {
|
||||
this.store.get().calling !== CallingState.OnCall &&
|
||||
this.store.update({ calling: CallingState.OnCall });
|
||||
|
||||
this.videoStreams[call.peer] = stream.getVideoTracks()[0];
|
||||
|
||||
this.callArgs && this.callArgs.onStream(stream);
|
||||
});
|
||||
// call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
|
||||
|
||||
call.on('close', this.onRemoteCallEnd);
|
||||
call.on('error', (e) => {
|
||||
console.error('PeerJS error (on call):', e);
|
||||
this.initiateCallEnd();
|
||||
this.callArgs && this.callArgs.onError && this.callArgs.onError();
|
||||
});
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
private cleaned: boolean = false;
|
||||
|
||||
clean() {
|
||||
this.cleaned = true; // sometimes cleaned before modules loaded
|
||||
this.initiateCallEnd();
|
||||
if (this._peer) {
|
||||
console.log('destroying peer...');
|
||||
const peer = this._peer; // otherwise it calls reconnection on data chan close
|
||||
this._peer = null;
|
||||
peer.disconnect();
|
||||
peer.destroy();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +1,18 @@
|
|||
import { audioContextManager } from 'App/utils/screenRecorder';
|
||||
|
||||
declare global {
|
||||
interface HTMLCanvasElement {
|
||||
captureStream(frameRate?: number): MediaStream;
|
||||
interface HTMLCanvasElement {
|
||||
captureStream(frameRate?: number): MediaStream;
|
||||
}
|
||||
}
|
||||
|
||||
function dummyTrack(): MediaStreamTrack {
|
||||
const canvas = document.createElement("canvas")//, { width: 0, height: 0})
|
||||
canvas.width=canvas.height=2 // Doesn't work when 1 (?!)
|
||||
function dummyTrack(): MediaStreamTrack {
|
||||
const canvas = document.createElement('canvas'); //, { width: 0, height: 0})
|
||||
canvas.width = canvas.height = 2; // Doesn't work when 1 (?!)
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx?.fillRect(0, 0, canvas.width, canvas.height);
|
||||
requestAnimationFrame(function draw(){
|
||||
ctx?.fillRect(0,0, canvas.width, canvas.height)
|
||||
requestAnimationFrame(function draw() {
|
||||
ctx?.fillRect(0, 0, canvas.width, canvas.height);
|
||||
requestAnimationFrame(draw);
|
||||
});
|
||||
// Also works. Probably it should be done once connected.
|
||||
|
|
@ -19,68 +21,72 @@ function dummyTrack(): MediaStreamTrack {
|
|||
}
|
||||
|
||||
export function RequestLocalStream(): Promise<LocalStream> {
|
||||
return navigator.mediaDevices.getUserMedia({ audio:true })
|
||||
.then(aStream => {
|
||||
const aTrack = aStream.getAudioTracks()[0]
|
||||
if (!aTrack) { throw new Error("No audio tracks provided") }
|
||||
return new _LocalStream(aTrack)
|
||||
})
|
||||
return navigator.mediaDevices.getUserMedia({ audio: true }).then((aStream) => {
|
||||
const aTrack = aStream.getAudioTracks()[0];
|
||||
if (!aTrack) {
|
||||
throw new Error('No audio tracks provided');
|
||||
}
|
||||
return new _LocalStream(aTrack);
|
||||
});
|
||||
}
|
||||
|
||||
class _LocalStream {
|
||||
private mediaRequested: boolean = false
|
||||
readonly stream: MediaStream
|
||||
private readonly vdTrack: MediaStreamTrack
|
||||
private mediaRequested: boolean = false;
|
||||
readonly stream: MediaStream;
|
||||
private readonly vdTrack: MediaStreamTrack;
|
||||
|
||||
constructor(aTrack: MediaStreamTrack) {
|
||||
this.vdTrack = dummyTrack()
|
||||
this.stream = new MediaStream([ aTrack, this.vdTrack ])
|
||||
this.vdTrack = dummyTrack();
|
||||
this.stream = new MediaStream([aTrack, this.vdTrack]);
|
||||
}
|
||||
|
||||
toggleVideo(): Promise<boolean> {
|
||||
if (!this.mediaRequested) {
|
||||
return navigator.mediaDevices.getUserMedia({video:true})
|
||||
.then(vStream => {
|
||||
const vTrack = vStream.getVideoTracks()[0]
|
||||
if (!vTrack) {
|
||||
throw new Error("No video track provided")
|
||||
}
|
||||
this.stream.addTrack(vTrack)
|
||||
this.stream.removeTrack(this.vdTrack)
|
||||
this.mediaRequested = true
|
||||
if (this.onVideoTrackCb) {
|
||||
this.onVideoTrackCb(vTrack)
|
||||
}
|
||||
return true
|
||||
})
|
||||
.catch(e => {
|
||||
// TODO: log
|
||||
console.error(e)
|
||||
return false
|
||||
})
|
||||
return navigator.mediaDevices
|
||||
.getUserMedia({ video: true })
|
||||
.then((vStream) => {
|
||||
const vTrack = vStream.getVideoTracks()[0];
|
||||
if (!vTrack) {
|
||||
throw new Error('No video track provided');
|
||||
}
|
||||
this.stream.addTrack(vTrack);
|
||||
this.stream.removeTrack(this.vdTrack);
|
||||
this.mediaRequested = true;
|
||||
if (this.onVideoTrackCb) {
|
||||
this.onVideoTrackCb(vTrack);
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.catch((e) => {
|
||||
// TODO: log
|
||||
console.error(e);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
let enabled = true
|
||||
this.stream.getVideoTracks().forEach(track => {
|
||||
track.enabled = enabled = enabled && !track.enabled
|
||||
})
|
||||
return Promise.resolve(enabled)
|
||||
let enabled = true;
|
||||
this.stream.getVideoTracks().forEach((track) => {
|
||||
track.enabled = enabled = enabled && !track.enabled;
|
||||
});
|
||||
return Promise.resolve(enabled);
|
||||
}
|
||||
|
||||
toggleAudio(): boolean {
|
||||
let enabled = true
|
||||
this.stream.getAudioTracks().forEach(track => {
|
||||
track.enabled = enabled = enabled && !track.enabled
|
||||
})
|
||||
return enabled
|
||||
let enabled = true;
|
||||
this.stream.getAudioTracks().forEach((track) => {
|
||||
track.enabled = enabled = enabled && !track.enabled;
|
||||
});
|
||||
return enabled;
|
||||
}
|
||||
|
||||
private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null
|
||||
private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null;
|
||||
|
||||
onVideoTrack(cb: (t: MediaStreamTrack) => void) {
|
||||
this.onVideoTrackCb = cb
|
||||
this.onVideoTrackCb = cb;
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.stream.getTracks().forEach(t => t.stop())
|
||||
this.stream.getTracks().forEach((t) => t.stop());
|
||||
}
|
||||
}
|
||||
|
||||
export type LocalStream = InstanceType<typeof _LocalStream>
|
||||
export type LocalStream = InstanceType<typeof _LocalStream>;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,29 @@
|
|||
import { toast } from 'react-toastify';
|
||||
|
||||
class AudioContextManager {
|
||||
context = new AudioContext();
|
||||
destination = this.context.createMediaStreamDestination();
|
||||
|
||||
getAllTracks() {
|
||||
return this.destination.stream.getAudioTracks() || [];
|
||||
}
|
||||
|
||||
mergeAudioStreams(stream: MediaStream) {
|
||||
const source = this.context.createMediaStreamSource(stream);
|
||||
const gain = this.context.createGain();
|
||||
gain.gain.value = 0.7;
|
||||
return source.connect(gain).connect(this.destination);
|
||||
}
|
||||
|
||||
clear() {
|
||||
// when everything is removed, tracks will be stopped automatically (hopefully)
|
||||
this.context = new AudioContext();
|
||||
this.destination = this.context.createMediaStreamDestination();
|
||||
}
|
||||
}
|
||||
|
||||
export const audioContextManager = new AudioContextManager();
|
||||
|
||||
const FILE_TYPE = 'video/webm';
|
||||
const FRAME_RATE = 30;
|
||||
|
||||
|
|
@ -16,7 +40,7 @@ function createFileRecorder(
|
|||
|
||||
let recordedChunks: BlobPart[] = [];
|
||||
const SAVE_INTERVAL_MS = 200;
|
||||
const mediaRecorder = new MediaRecorder(stream);
|
||||
const mediaRecorder = new MediaRecorder(stream, { mimeType: 'video/webm; codecs=vp8,opus' });
|
||||
|
||||
mediaRecorder.ondataavailable = function (e) {
|
||||
if (e.data.size > 0) {
|
||||
|
|
@ -29,7 +53,7 @@ function createFileRecorder(
|
|||
|
||||
ended = true;
|
||||
saveFile(recordedChunks, mimeType, start, recName, sessionId, saveCb);
|
||||
onStop()
|
||||
onStop();
|
||||
recordedChunks = [];
|
||||
}
|
||||
|
||||
|
|
@ -74,13 +98,24 @@ function saveFile(
|
|||
}
|
||||
|
||||
async function recordScreen() {
|
||||
return await navigator.mediaDevices.getDisplayMedia({
|
||||
audio: true,
|
||||
const desktopStreams = await navigator.mediaDevices.getDisplayMedia({
|
||||
audio: {
|
||||
// @ts-ignore
|
||||
restrictOwnAudio: false,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: false,
|
||||
sampleRate: 44100,
|
||||
},
|
||||
video: { frameRate: FRAME_RATE },
|
||||
// potential chrome hack
|
||||
// @ts-ignore
|
||||
preferCurrentTab: true,
|
||||
});
|
||||
audioContextManager.mergeAudioStreams(desktopStreams);
|
||||
return new MediaStream([
|
||||
...desktopStreams.getVideoTracks(),
|
||||
...audioContextManager.getAllTracks(),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -94,7 +129,18 @@ async function recordScreen() {
|
|||
*
|
||||
* @returns a promise that resolves to a function that stops the recording
|
||||
*/
|
||||
export async function screenRecorder(recName: string, sessionId: string, saveCb: (saveObj: { name: string; duration: number }, blob: Blob) => void, onStop: () => void) {
|
||||
export async function screenRecorder(
|
||||
recName: string,
|
||||
sessionId: string,
|
||||
saveCb: (
|
||||
saveObj: {
|
||||
name: string;
|
||||
duration: number;
|
||||
},
|
||||
blob: Blob
|
||||
) => void,
|
||||
onStop: () => void
|
||||
) {
|
||||
try {
|
||||
const stream = await recordScreen();
|
||||
const mediaRecorder = createFileRecorder(stream, FILE_TYPE, recName, sessionId, saveCb, onStop);
|
||||
|
|
@ -102,11 +148,13 @@ export async function screenRecorder(recName: string, sessionId: string, saveCb:
|
|||
return () => {
|
||||
if (mediaRecorder.state !== 'inactive') {
|
||||
mediaRecorder.stop();
|
||||
onStop()
|
||||
onStop();
|
||||
}
|
||||
}
|
||||
};
|
||||
} catch (e) {
|
||||
toast.error('Screen recording is not permitted by your system and/or browser. Make sure to enable it in your browser as well as in your system settings.');
|
||||
toast.error(
|
||||
'Screen recording is not permitted by your system and/or browser. Make sure to enable it in your browser as well as in your system settings.'
|
||||
);
|
||||
throw new Error('OpenReplay recording: ' + e);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue