From 4bb08d2a5e81f6db82eafceb1ac5fe6c93b087fc Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Thu, 3 Aug 2023 15:54:49 +0200 Subject: [PATCH] fix(ui): merge audio tracks on node chain for recording (#1433) (#1437) * fix(ui): merge audio tracks on node chain for recording * fix(ui): fix Co-authored-by: Delirium --- .../AssistActions/AssistActions.tsx | 59 +- .../app/components/Session/LivePlayer.tsx | 2 + frontend/app/player/web/assist/Call.ts | 537 +++++++++--------- frontend/app/player/web/assist/LocalStream.ts | 108 ++-- frontend/app/utils/screenRecorder.ts | 64 ++- 5 files changed, 421 insertions(+), 349 deletions(-) diff --git a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx index 71a117868..29d2c42a4 100644 --- a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx +++ b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx @@ -3,12 +3,7 @@ import { Button, Tooltip } from 'UI'; import { connect } from 'react-redux'; import cn from 'classnames'; import ChatWindow from '../../ChatWindow'; -import { - CallingState, - ConnectionStatus, - RemoteControlStatus, - RequestLocalStream, -} from 'Player'; +import { CallingState, ConnectionStatus, RemoteControlStatus, RequestLocalStream } from 'Player'; import type { LocalStream } from 'Player'; import { PlayerContext, ILivePlayerContext } from 'App/components/Session/playerContext'; import { observer } from 'mobx-react-lite'; @@ -16,12 +11,14 @@ import { toast } from 'react-toastify'; import { confirm } from 'UI'; import stl from './AassistActions.module.css'; import ScreenRecorder from 'App/components/Session_/ScreenRecorder/ScreenRecorder'; +import { audioContextManager } from 'App/utils/screenRecorder'; function onReject() { toast.info(`Call was rejected.`); } + function onControlReject() { - toast.info('Remote control request was rejected by user') + toast.info('Remote control request was rejected by user'); } function onError(e: any) { @@ -47,7 +44,7 @@ function AssistActions({ userDisplayName, }: Props) { // @ts-ignore ??? - const { player, store } = React.useContext(PlayerContext) + const { player, store } = React.useContext(PlayerContext); const { assistManager: { @@ -55,17 +52,17 @@ function AssistActions({ setCallArgs, requestReleaseRemoteControl, toggleAnnotation, - setRemoteControlCallbacks + setRemoteControlCallbacks, }, - toggleUserName, - } = player + toggleUserName, + } = player; const { calling, annotating, peerConnectionStatus, remoteControl: remoteControlStatus, livePlay, - } = store.get() + } = store.get(); const [isPrestart, setPrestart] = useState(false); const [incomeStream, setIncomeStream] = useState([]); @@ -121,8 +118,9 @@ function AssistActions({ const addIncomeStream = (stream: MediaStream) => { setIncomeStream((oldState) => { - if (oldState === null) return [stream] + if (oldState === null) return [stream]; if (!oldState.find((existingStream) => existingStream.id === stream.id)) { + audioContextManager.mergeAudioStreams(stream); return [...oldState, stream]; } return oldState; @@ -133,7 +131,16 @@ function AssistActions({ RequestLocalStream() .then((lStream) => { setLocalStream(lStream); - setCallArgs(lStream, addIncomeStream, lStream.stop.bind(lStream), onReject, onError); + audioContextManager.mergeAudioStreams(lStream.stream); + setCallArgs( + lStream, + addIncomeStream, + () => { + lStream.stop.bind(lStream); + }, + onReject, + onError + ); setCallObject(callPeer()); if (additionalAgentIds) { callPeer(additionalAgentIds); @@ -157,7 +164,7 @@ function AssistActions({ }; const requestControl = () => { - setRemoteControlCallbacks({ onReject: onControlReject }) + setRemoteControlCallbacks({ onReject: onControlReject }); if (callRequesting || remoteRequesting) return; requestReleaseRemoteControl(); }; @@ -249,17 +256,13 @@ function AssistActions({ ); } -const con = connect( - (state: any) => { - const permissions = state.getIn(['user', 'account', 'permissions']) || []; - return { - hasPermission: permissions.includes('ASSIST_CALL'), - isEnterprise: state.getIn(['user', 'account', 'edition']) === 'ee', - userDisplayName: state.getIn(['sessions', 'current']).userDisplayName, - }; - } -); +const con = connect((state: any) => { + const permissions = state.getIn(['user', 'account', 'permissions']) || []; + return { + hasPermission: permissions.includes('ASSIST_CALL'), + isEnterprise: state.getIn(['user', 'account', 'edition']) === 'ee', + userDisplayName: state.getIn(['sessions', 'current']).userDisplayName, + }; +}); -export default con( - observer(AssistActions) -); +export default con(observer(AssistActions)); diff --git a/frontend/app/components/Session/LivePlayer.tsx b/frontend/app/components/Session/LivePlayer.tsx index 03c8860d5..9cce9e151 100644 --- a/frontend/app/components/Session/LivePlayer.tsx +++ b/frontend/app/components/Session/LivePlayer.tsx @@ -1,3 +1,4 @@ +import {audioContextManager} from "App/utils/screenRecorder"; import React from 'react'; import { useEffect, useState } from 'react'; import { connect } from 'react-redux'; @@ -82,6 +83,7 @@ function LivePlayer({ return () => { if (!location.pathname.includes('multiview') || !location.pathname.includes(usedSession.sessionId)) { console.debug('cleaning live player for', usedSession.sessionId) + audioContextManager.clear(); playerInst?.clean?.(); // @ts-ignore default empty setContextValue(defaultContextValue) diff --git a/frontend/app/player/web/assist/Call.ts b/frontend/app/player/web/assist/Call.ts index ebd1ba846..6710a9cdd 100644 --- a/frontend/app/player/web/assist/Call.ts +++ b/frontend/app/player/web/assist/Call.ts @@ -2,293 +2,306 @@ import type Peer from 'peerjs'; import type { MediaConnection } from 'peerjs'; import type { LocalStream } from './LocalStream'; -import type { Socket } from './types' -import type { Store } from '../../common/types' +import type { Socket } from './types'; +import type { Store } from '../../common/types'; import appStore from 'App/store'; - export enum CallingState { - NoCall, - Connecting, - Requesting, - Reconnecting, - OnCall, + NoCall, + Connecting, + Requesting, + Reconnecting, + OnCall, } export interface State { - calling: CallingState; - currentTab?: string; + calling: CallingState; + currentTab?: string; } export default class Call { - private assistVersion = 1 - static readonly INITIAL_STATE: Readonly = { - calling: CallingState.NoCall - } + private assistVersion = 1; + static readonly INITIAL_STATE: Readonly = { + calling: CallingState.NoCall, + }; - private _peer: Peer | null = null - private connectionAttempts: number = 0 - private callConnection: MediaConnection[] = [] - private videoStreams: Record = {} + private _peer: Peer | null = null; + private connectionAttempts: number = 0; + private callConnection: MediaConnection[] = []; + private videoStreams: Record = {}; - constructor( - private store: Store, - private socket: Socket, - private config: RTCIceServer[] | null, - private peerID: string, - private getAssistVersion: () => number - ) { - socket.on('call_end', this.onRemoteCallEnd) - socket.on('videofeed', ({ streamId, enabled }) => { - console.log(streamId, enabled) - console.log(this.videoStreams) + constructor( + private store: Store, + private socket: Socket, + private config: RTCIceServer[] | null, + private peerID: string, + private getAssistVersion: () => number + ) { + socket.on('call_end', this.onRemoteCallEnd); + socket.on('videofeed', ({ streamId, enabled }) => { + console.log(streamId, enabled); + console.log(this.videoStreams); if (this.videoStreams[streamId]) { - this.videoStreams[streamId].enabled = enabled + this.videoStreams[streamId].enabled = enabled; } - console.log(this.videoStreams) - }) - let reconnecting = false + console.log(this.videoStreams); + }); + let reconnecting = false; socket.on('SESSION_DISCONNECTED', () => { - if (this.store.get().calling === CallingState.OnCall) { - this.store.update({ calling: CallingState.Reconnecting }) - reconnecting = true - } else if (this.store.get().calling === CallingState.Requesting){ - this.store.update({ calling: CallingState.NoCall }) + if (this.store.get().calling === CallingState.OnCall) { + this.store.update({ calling: CallingState.Reconnecting }); + reconnecting = true; + } else if (this.store.get().calling === CallingState.Requesting) { + this.store.update({ calling: CallingState.NoCall }); } - }) + }); socket.on('messages', () => { - if (reconnecting) { // 'messages' come frequently, so it is better to have Reconnecting - this._callSessionPeer() - reconnecting = false - } - }) - socket.on("disconnect", () => { - this.store.update({ calling: CallingState.NoCall }) - }) - this.assistVersion = this.getAssistVersion() - } - - private getPeer(): Promise { - if (this._peer && !this._peer.disconnected) { return Promise.resolve(this._peer) } - - // @ts-ignore - const urlObject = new URL(window.env.API_EDP || window.location.origin) - - // @ts-ignore TODO: set module in ts settings - return import('peerjs').then(({ default: Peer }) => { - if (this.cleaned) {return Promise.reject("Already cleaned")} - const peerOpts: Peer.PeerJSOption = { - host: urlObject.hostname, - path: '/assist', - port: urlObject.port === "" ? (location.protocol === 'https:' ? 443 : 80 ): parseInt(urlObject.port), - } - if (this.config) { - peerOpts['config'] = { - iceServers: this.config, - //@ts-ignore - sdpSemantics: 'unified-plan', - iceTransportPolicy: 'all', - }; - } - const peer = this._peer = new Peer(peerOpts) - peer.on('call', call => { - console.log('getting call from', call.peer) - call.answer(this.callArgs.localStream.stream) - this.callConnection.push(call) - - this.callArgs.localStream.onVideoTrack(vTrack => { - const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video") - if (!sender) { - console.warn("No video sender found") - return - } - sender.replaceTrack(vTrack) - }) - - call.on('stream', stream => { - this.videoStreams[call.peer] = stream.getVideoTracks()[0] - this.callArgs && this.callArgs.onStream(stream) - }); - // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) - - call.on("close", this.onRemoteCallEnd) - call.on("error", (e) => { - console.error("PeerJS error (on call):", e) - this.initiateCallEnd(); - this.callArgs && this.callArgs.onError && this.callArgs.onError(); - }); - }) - peer.on('error', e => { - if (e.type === 'disconnected') { - return peer.reconnect() - } else if (e.type !== 'peer-unavailable') { - console.error(`PeerJS error (on peer). Type ${e.type}`, e); - } - - // call-reconnection connected - // if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) { - // this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT - // ? ConnectionStatus.Connecting - // : ConnectionStatus.Disconnected); - // Reconnect... - }) - - return new Promise(resolve => { - peer.on("open", () => resolve(peer)) - }) - }); - - } - - - private handleCallEnd() { - this.callArgs && this.callArgs.onCallEnd() - this.callConnection[0] && this.callConnection[0].close() - this.store.update({ calling: CallingState.NoCall }) - this.callArgs = null - // TODO: We have it separated, right? (check) - //this.toggleAnnotation(false) - } - private onRemoteCallEnd = () => { - if ([CallingState.Requesting, CallingState.Connecting].includes(this.store.get().calling)) { - this.callArgs && this.callArgs.onReject() - this.callConnection[0] && this.callConnection[0].close() - this.store.update({ calling: CallingState.NoCall }) - this.callArgs = null - } else { - this.handleCallEnd() - } - } - - initiateCallEnd = async () => { - this.emitData("call_end", appStore.getState().getIn([ 'user', 'account', 'name'])) - this.handleCallEnd() - // TODO: We have it separated, right? (check) - // const remoteControl = this.store.get().remoteControl - // if (remoteControl === RemoteControlStatus.Enabled) { - // this.socket.emit("release_control") - // this.toggleRemoteControl(false) - // } - } - - private emitData = (event: string, data?: any) => { - if (this.getAssistVersion() === 1) { - this.socket?.emit(event, data) - } else { - this.socket?.emit(event, { meta: { tabId: this.store.get().currentTab }, data }) - } - } - - - private callArgs: { - localStream: LocalStream, - onStream: (s: MediaStream)=>void, - onCallEnd: () => void, - onReject: () => void, - onError?: ()=> void, - } | null = null - - setCallArgs( - localStream: LocalStream, - onStream: (s: MediaStream)=>void, - onCallEnd: () => void, - onReject: () => void, - onError?: (e?: any)=> void, - ) { - this.callArgs = { - localStream, - onStream, - onCallEnd, - onReject, - onError, - } - } - - call(thirdPartyPeers?: string[]): { end: () => void } { - if (thirdPartyPeers && thirdPartyPeers.length > 0) { - this.addPeerCall(thirdPartyPeers) - } else { - this._callSessionPeer() - } - return { - end: this.initiateCallEnd, - } - } - - toggleVideoLocalStream(enabled: boolean) { - this.getPeer().then((peer) => { - this.emitData('videofeed', { streamId: peer.id, enabled }) - }) + if (reconnecting) { + // 'messages' come frequently, so it is better to have Reconnecting + this._callSessionPeer(); + reconnecting = false; + } + }); + socket.on('disconnect', () => { + this.store.update({ calling: CallingState.NoCall }); + }); + this.assistVersion = this.getAssistVersion(); } + private getPeer(): Promise { + if (this._peer && !this._peer.disconnected) { + return Promise.resolve(this._peer); + } - /** Connecting to the other agents that are already - * in the call with the user - */ - addPeerCall(thirdPartyPeers: string[]) { - thirdPartyPeers.forEach(peer => this._peerConnection(peer)) - } + // @ts-ignore + const urlObject = new URL(window.env.API_EDP || window.location.origin); - /** Connecting to the app user */ - private _callSessionPeer() { - if (![CallingState.NoCall, CallingState.Reconnecting].includes(this.store.get().calling)) { return } - this.store.update({ calling: CallingState.Connecting }) - const tab = this.store.get().currentTab - if (!this.store.get().currentTab) { - console.warn('No tab data to connect to peer') - } - const peerId = this.getAssistVersion() === 1 ? this.peerID : `${this.peerID}-${tab || Object.keys(this.store.get().tabs)[0]}` - console.log(peerId, this.getAssistVersion()) - void this._peerConnection(peerId); - this.emitData("_agent_name", appStore.getState().getIn([ 'user', 'account', 'name'])) - } + // @ts-ignore TODO: set module in ts settings + return import('peerjs').then(({ default: Peer }) => { + if (this.cleaned) { + return Promise.reject('Already cleaned'); + } + const peerOpts: Peer.PeerJSOption = { + host: urlObject.hostname, + path: '/assist', + port: + urlObject.port === '' + ? location.protocol === 'https:' + ? 443 + : 80 + : parseInt(urlObject.port), + }; + if (this.config) { + peerOpts['config'] = { + iceServers: this.config, + //@ts-ignore + sdpSemantics: 'unified-plan', + iceTransportPolicy: 'all', + }; + } + const peer = (this._peer = new Peer(peerOpts)); + peer.on('call', (call) => { + console.log('getting call from', call.peer); + call.answer(this.callArgs.localStream.stream); + this.callConnection.push(call); - private async _peerConnection(remotePeerId: string) { - try { - const peer = await this.getPeer(); - const call = peer.call(remotePeerId, this.callArgs.localStream.stream) - this.callConnection.push(call) + this.callArgs.localStream.onVideoTrack((vTrack) => { + const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video'); + if (!sender) { + console.warn('No video sender found'); + return; + } + sender.replaceTrack(vTrack); + }); - this.callArgs.localStream.onVideoTrack(vTrack => { - const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video") - if (!sender) { - console.warn("No video sender found") - return - } - sender.replaceTrack(vTrack) - }) + call.on('stream', (stream) => { + this.videoStreams[call.peer] = stream.getVideoTracks()[0]; + this.callArgs && this.callArgs.onStream(stream); + }); + // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) - call.on('stream', stream => { - this.store.get().calling !== CallingState.OnCall && this.store.update({ calling: CallingState.OnCall }) + call.on('close', this.onRemoteCallEnd); + call.on('error', (e) => { + console.error('PeerJS error (on call):', e); + this.initiateCallEnd(); + this.callArgs && this.callArgs.onError && this.callArgs.onError(); + }); + }); + peer.on('error', (e) => { + if (e.type === 'disconnected') { + return peer.reconnect(); + } else if (e.type !== 'peer-unavailable') { + console.error(`PeerJS error (on peer). Type ${e.type}`, e); + } - this.videoStreams[call.peer] = stream.getVideoTracks()[0] + // call-reconnection connected + // if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) { + // this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT + // ? ConnectionStatus.Connecting + // : ConnectionStatus.Disconnected); + // Reconnect... + }); - this.callArgs && this.callArgs.onStream(stream) - }); - // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) + return new Promise((resolve) => { + peer.on('open', () => resolve(peer)); + }); + }); + } - call.on("close", this.onRemoteCallEnd) - call.on("error", (e) => { - console.error("PeerJS error (on call):", e) - this.initiateCallEnd(); - this.callArgs && this.callArgs.onError && this.callArgs.onError(); - }); - } catch (e) { - console.error(e) - } - } + private handleCallEnd() { + this.callArgs && this.callArgs.onCallEnd(); + this.callConnection[0] && this.callConnection[0].close(); + this.store.update({ calling: CallingState.NoCall }); + this.callArgs = null; + // TODO: We have it separated, right? (check) + //this.toggleAnnotation(false) + } - private cleaned: boolean = false - clean() { - this.cleaned = true // sometimes cleaned before modules loaded - this.initiateCallEnd() - if (this._peer) { - console.log("destroying peer...") - const peer = this._peer; // otherwise it calls reconnection on data chan close - this._peer = null; - peer.disconnect(); - peer.destroy(); - } - } -} \ No newline at end of file + private onRemoteCallEnd = () => { + if ([CallingState.Requesting, CallingState.Connecting].includes(this.store.get().calling)) { + this.callArgs && this.callArgs.onReject(); + this.callConnection[0] && this.callConnection[0].close(); + this.store.update({ calling: CallingState.NoCall }); + this.callArgs = null; + } else { + this.handleCallEnd(); + } + }; + + initiateCallEnd = async () => { + this.emitData('call_end', appStore.getState().getIn(['user', 'account', 'name'])); + this.handleCallEnd(); + // TODO: We have it separated, right? (check) + // const remoteControl = this.store.get().remoteControl + // if (remoteControl === RemoteControlStatus.Enabled) { + // this.socket.emit("release_control") + // this.toggleRemoteControl(false) + // } + }; + + private emitData = (event: string, data?: any) => { + if (this.getAssistVersion() === 1) { + this.socket?.emit(event, data); + } else { + this.socket?.emit(event, { meta: { tabId: this.store.get().currentTab }, data }); + } + }; + + private callArgs: { + localStream: LocalStream; + onStream: (s: MediaStream) => void; + onCallEnd: () => void; + onReject: () => void; + onError?: () => void; + } | null = null; + + setCallArgs( + localStream: LocalStream, + onStream: (s: MediaStream) => void, + onCallEnd: () => void, + onReject: () => void, + onError?: (e?: any) => void + ) { + this.callArgs = { + localStream, + onStream, + onCallEnd, + onReject, + onError, + }; + } + + call(thirdPartyPeers?: string[]): { end: () => void } { + if (thirdPartyPeers && thirdPartyPeers.length > 0) { + this.addPeerCall(thirdPartyPeers); + } else { + this._callSessionPeer(); + } + return { + end: this.initiateCallEnd, + }; + } + + toggleVideoLocalStream(enabled: boolean) { + this.getPeer().then((peer) => { + this.emitData('videofeed', { streamId: peer.id, enabled }); + }); + } + + /** Connecting to the other agents that are already + * in the call with the user + */ + addPeerCall(thirdPartyPeers: string[]) { + thirdPartyPeers.forEach((peer) => this._peerConnection(peer)); + } + + /** Connecting to the app user */ + private _callSessionPeer() { + if (![CallingState.NoCall, CallingState.Reconnecting].includes(this.store.get().calling)) { + return; + } + this.store.update({ calling: CallingState.Connecting }); + const tab = this.store.get().currentTab; + if (!this.store.get().currentTab) { + console.warn('No tab data to connect to peer'); + } + const peerId = + this.getAssistVersion() === 1 + ? this.peerID + : `${this.peerID}-${tab || Object.keys(this.store.get().tabs)[0]}`; + console.log(peerId, this.getAssistVersion()); + void this._peerConnection(peerId); + this.emitData('_agent_name', appStore.getState().getIn(['user', 'account', 'name'])); + } + + private async _peerConnection(remotePeerId: string) { + try { + const peer = await this.getPeer(); + const call = peer.call(remotePeerId, this.callArgs.localStream.stream); + this.callConnection.push(call); + + this.callArgs.localStream.onVideoTrack((vTrack) => { + const sender = call.peerConnection.getSenders().find((s) => s.track?.kind === 'video'); + if (!sender) { + console.warn('No video sender found'); + return; + } + sender.replaceTrack(vTrack); + }); + + call.on('stream', (stream) => { + this.store.get().calling !== CallingState.OnCall && + this.store.update({ calling: CallingState.OnCall }); + + this.videoStreams[call.peer] = stream.getVideoTracks()[0]; + + this.callArgs && this.callArgs.onStream(stream); + }); + // call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) + + call.on('close', this.onRemoteCallEnd); + call.on('error', (e) => { + console.error('PeerJS error (on call):', e); + this.initiateCallEnd(); + this.callArgs && this.callArgs.onError && this.callArgs.onError(); + }); + } catch (e) { + console.error(e); + } + } + + private cleaned: boolean = false; + + clean() { + this.cleaned = true; // sometimes cleaned before modules loaded + this.initiateCallEnd(); + if (this._peer) { + console.log('destroying peer...'); + const peer = this._peer; // otherwise it calls reconnection on data chan close + this._peer = null; + peer.disconnect(); + peer.destroy(); + } + } +} diff --git a/frontend/app/player/web/assist/LocalStream.ts b/frontend/app/player/web/assist/LocalStream.ts index e7d3bae15..42021c1b7 100644 --- a/frontend/app/player/web/assist/LocalStream.ts +++ b/frontend/app/player/web/assist/LocalStream.ts @@ -1,16 +1,18 @@ +import { audioContextManager } from 'App/utils/screenRecorder'; + declare global { - interface HTMLCanvasElement { - captureStream(frameRate?: number): MediaStream; + interface HTMLCanvasElement { + captureStream(frameRate?: number): MediaStream; } } -function dummyTrack(): MediaStreamTrack { - const canvas = document.createElement("canvas")//, { width: 0, height: 0}) - canvas.width=canvas.height=2 // Doesn't work when 1 (?!) +function dummyTrack(): MediaStreamTrack { + const canvas = document.createElement('canvas'); //, { width: 0, height: 0}) + canvas.width = canvas.height = 2; // Doesn't work when 1 (?!) const ctx = canvas.getContext('2d'); ctx?.fillRect(0, 0, canvas.width, canvas.height); - requestAnimationFrame(function draw(){ - ctx?.fillRect(0,0, canvas.width, canvas.height) + requestAnimationFrame(function draw() { + ctx?.fillRect(0, 0, canvas.width, canvas.height); requestAnimationFrame(draw); }); // Also works. Probably it should be done once connected. @@ -19,68 +21,72 @@ function dummyTrack(): MediaStreamTrack { } export function RequestLocalStream(): Promise { - return navigator.mediaDevices.getUserMedia({ audio:true }) - .then(aStream => { - const aTrack = aStream.getAudioTracks()[0] - if (!aTrack) { throw new Error("No audio tracks provided") } - return new _LocalStream(aTrack) - }) + return navigator.mediaDevices.getUserMedia({ audio: true }).then((aStream) => { + const aTrack = aStream.getAudioTracks()[0]; + if (!aTrack) { + throw new Error('No audio tracks provided'); + } + return new _LocalStream(aTrack); + }); } class _LocalStream { - private mediaRequested: boolean = false - readonly stream: MediaStream - private readonly vdTrack: MediaStreamTrack + private mediaRequested: boolean = false; + readonly stream: MediaStream; + private readonly vdTrack: MediaStreamTrack; + constructor(aTrack: MediaStreamTrack) { - this.vdTrack = dummyTrack() - this.stream = new MediaStream([ aTrack, this.vdTrack ]) + this.vdTrack = dummyTrack(); + this.stream = new MediaStream([aTrack, this.vdTrack]); } toggleVideo(): Promise { if (!this.mediaRequested) { - return navigator.mediaDevices.getUserMedia({video:true}) - .then(vStream => { - const vTrack = vStream.getVideoTracks()[0] - if (!vTrack) { - throw new Error("No video track provided") - } - this.stream.addTrack(vTrack) - this.stream.removeTrack(this.vdTrack) - this.mediaRequested = true - if (this.onVideoTrackCb) { - this.onVideoTrackCb(vTrack) - } - return true - }) - .catch(e => { - // TODO: log - console.error(e) - return false - }) + return navigator.mediaDevices + .getUserMedia({ video: true }) + .then((vStream) => { + const vTrack = vStream.getVideoTracks()[0]; + if (!vTrack) { + throw new Error('No video track provided'); + } + this.stream.addTrack(vTrack); + this.stream.removeTrack(this.vdTrack); + this.mediaRequested = true; + if (this.onVideoTrackCb) { + this.onVideoTrackCb(vTrack); + } + return true; + }) + .catch((e) => { + // TODO: log + console.error(e); + return false; + }); } - let enabled = true - this.stream.getVideoTracks().forEach(track => { - track.enabled = enabled = enabled && !track.enabled - }) - return Promise.resolve(enabled) + let enabled = true; + this.stream.getVideoTracks().forEach((track) => { + track.enabled = enabled = enabled && !track.enabled; + }); + return Promise.resolve(enabled); } toggleAudio(): boolean { - let enabled = true - this.stream.getAudioTracks().forEach(track => { - track.enabled = enabled = enabled && !track.enabled - }) - return enabled + let enabled = true; + this.stream.getAudioTracks().forEach((track) => { + track.enabled = enabled = enabled && !track.enabled; + }); + return enabled; } - private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null + private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null; + onVideoTrack(cb: (t: MediaStreamTrack) => void) { - this.onVideoTrackCb = cb + this.onVideoTrackCb = cb; } stop() { - this.stream.getTracks().forEach(t => t.stop()) + this.stream.getTracks().forEach((t) => t.stop()); } } -export type LocalStream = InstanceType +export type LocalStream = InstanceType; diff --git a/frontend/app/utils/screenRecorder.ts b/frontend/app/utils/screenRecorder.ts index 3905d7b05..4ac6aab62 100644 --- a/frontend/app/utils/screenRecorder.ts +++ b/frontend/app/utils/screenRecorder.ts @@ -1,5 +1,29 @@ import { toast } from 'react-toastify'; +class AudioContextManager { + context = new AudioContext(); + destination = this.context.createMediaStreamDestination(); + + getAllTracks() { + return this.destination.stream.getAudioTracks() || []; + } + + mergeAudioStreams(stream: MediaStream) { + const source = this.context.createMediaStreamSource(stream); + const gain = this.context.createGain(); + gain.gain.value = 0.7; + return source.connect(gain).connect(this.destination); + } + + clear() { + // when everything is removed, tracks will be stopped automatically (hopefully) + this.context = new AudioContext(); + this.destination = this.context.createMediaStreamDestination(); + } +} + +export const audioContextManager = new AudioContextManager(); + const FILE_TYPE = 'video/webm'; const FRAME_RATE = 30; @@ -16,7 +40,7 @@ function createFileRecorder( let recordedChunks: BlobPart[] = []; const SAVE_INTERVAL_MS = 200; - const mediaRecorder = new MediaRecorder(stream); + const mediaRecorder = new MediaRecorder(stream, { mimeType: 'video/webm; codecs=vp8,opus' }); mediaRecorder.ondataavailable = function (e) { if (e.data.size > 0) { @@ -29,7 +53,7 @@ function createFileRecorder( ended = true; saveFile(recordedChunks, mimeType, start, recName, sessionId, saveCb); - onStop() + onStop(); recordedChunks = []; } @@ -74,13 +98,24 @@ function saveFile( } async function recordScreen() { - return await navigator.mediaDevices.getDisplayMedia({ - audio: true, + const desktopStreams = await navigator.mediaDevices.getDisplayMedia({ + audio: { + // @ts-ignore + restrictOwnAudio: false, + echoCancellation: true, + noiseSuppression: false, + sampleRate: 44100, + }, video: { frameRate: FRAME_RATE }, // potential chrome hack // @ts-ignore preferCurrentTab: true, }); + audioContextManager.mergeAudioStreams(desktopStreams); + return new MediaStream([ + ...desktopStreams.getVideoTracks(), + ...audioContextManager.getAllTracks(), + ]); } /** @@ -94,7 +129,18 @@ async function recordScreen() { * * @returns a promise that resolves to a function that stops the recording */ -export async function screenRecorder(recName: string, sessionId: string, saveCb: (saveObj: { name: string; duration: number }, blob: Blob) => void, onStop: () => void) { +export async function screenRecorder( + recName: string, + sessionId: string, + saveCb: ( + saveObj: { + name: string; + duration: number; + }, + blob: Blob + ) => void, + onStop: () => void +) { try { const stream = await recordScreen(); const mediaRecorder = createFileRecorder(stream, FILE_TYPE, recName, sessionId, saveCb, onStop); @@ -102,11 +148,13 @@ export async function screenRecorder(recName: string, sessionId: string, saveCb: return () => { if (mediaRecorder.state !== 'inactive') { mediaRecorder.stop(); - onStop() + onStop(); } - } + }; } catch (e) { - toast.error('Screen recording is not permitted by your system and/or browser. Make sure to enable it in your browser as well as in your system settings.'); + toast.error( + 'Screen recording is not permitted by your system and/or browser. Make sure to enable it in your browser as well as in your system settings.' + ); throw new Error('OpenReplay recording: ' + e); } }