jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix (assist): connection state sharing fix && agent name

Browse source
This commit is contained in:
ShiKhu 2021-07-14 16:01:25 +03:00
parent d148a9f55e
commit 4fab5d42e0
8 changed files with 192 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
frontend/app/components/Assist/components/AssistActions/AssistActions.tsx
View file

@ -21,17 +21,14 @@ function AssistActions({ toggleChatWindow, userId, calling, peerConnectionStatus
const [ incomeStream, setIncomeStream ] = useState<MediaStream | null>(null);
const [ localStream, setLocalStream ] = useState<MediaStream | null>(null);
const [ endCall, setEndCall ] = useState<()=>void>(()=>{});
const [ disconnected, setDisconnected ] = useState(false);
useEffect(() => {
return endCall
}, [])
useEffect(() => {
console.log('peerConnectionStatus', peerConnectionStatus)
if (peerConnectionStatus == 4) {
toast.info(`Live session is closed.`);
setDisconnected(true)
if (peerConnectionStatus == ConnectionStatus.Disconnected) {
toast.info(`Live session was closed.`);
}
}, [peerConnectionStatus])
@ -61,7 +58,7 @@ function AssistActions({ toggleChatWindow, userId, calling, peerConnectionStatus
}).catch(onError);
}
const inCall = calling == 0 || calling == 1
const inCall = calling !== CallingState.False;
return (
<div className="flex items-center">
@ -72,10 +69,10 @@ function AssistActions({ toggleChatWindow, userId, calling, peerConnectionStatus
cn(
'cursor-pointer p-2 mr-2 flex items-center',
{[stl.inCall] : inCall },
{[stl.disabled]: disconnected}
{[stl.disabled]: peerConnectionStatus !== ConnectionStatus.Connected}
)
}
onClick={inCall ? endCall : call}
onClick={ inCall ? endCall : call}
role="button"
>
<Icon

12
frontend/app/components/Session_/Player/Player.js
View file

@ -10,6 +10,7 @@ import Controls from './Controls';
import stl from './player.css';
import AutoplayTimer from '../AutoplayTimer';
import EventsToggleButton from '../../Session/EventsToggleButton';
import { getStatusText } from 'Player/MessageDistributor/managers/AssistManager';
const ScreenWrapper = withOverlay()(React.memo(() => <div className={ stl.screenWrapper } />));
@ -19,10 +20,11 @@ const ScreenWrapper = withOverlay()(React.memo(() => <div className={ stl.screen
loading: state.messagesLoading,
disconnected: state.disconnected,
disabled: state.cssLoading || state.messagesLoading || state.inspectorMode,
removeOverlay: !state.messagesLoading && state.inspectorMode || state.live,
removeOverlay: !state.messagesLoading && state.inspectorMode,
completed: state.completed,
autoplay: state.autoplay,
live: state.live
live: state.live,
liveStatusText: getStatusText(state.peerConnectionStatus),
}))
@connect(state => ({
//session: state.getIn([ 'sessions', 'current' ]),
@ -108,6 +110,7 @@ export default class Player extends React.PureComponent {
autoplay,
nextId,
live,
liveStatusText,
} = this.props;
return (
@ -134,7 +137,10 @@ export default class Player extends React.PureComponent {
className={ stl.overlay }
onClick={ disabled ? null : this.togglePlay }
>
<Loader loading={ loading } />
{ live && liveStatusText
? <span className={stl.liveStatusText}>{liveStatusText}</span>
: <Loader loading={ loading } />
}
<div
className={ cn(stl.iconWrapper, {
[ stl.zoomIcon ]: showPlayOverlayIcon

5
frontend/app/components/Session_/Player/player.css
View file

@ -77,4 +77,9 @@
.inspectorMode {
z-index: 99991 !important;
}
.liveStatusText {
color: $gray-light;
font-size: 40px;
}

132
frontend/app/player/MessageDistributor/managers/AssistManager.ts
View file

@ -23,6 +23,22 @@ export enum ConnectionStatus {
Error,
};
export function getStatusText(status: ConnectionStatus): string {
switch(status) {
case ConnectionStatus.Connecting:
return "Connecting...";
case ConnectionStatus.Connected:
return "";
case ConnectionStatus.Inactive:
return "Client tab is inactive";
case ConnectionStatus.Disconnected:
return "Disconnected";
case ConnectionStatus.Error:
return "Something went wrong. Try to reload the page.";
}
}
export interface State {
calling: CallingState,
peerConnectionStatus: ConnectionStatus,
@ -119,14 +135,14 @@ export default class AssistManager {
port: location.protocol === 'https:' ? 443 : 80,
});
this.peer = peer;
this.peer.on('error', e => {
peer.on('error', e => {
if (e.type === 'peer-unavailable') {
if (this.peer && this.connectionAttempts++ < MAX_RECONNECTION_COUNT) {
update({ peerConnectionStatus: ConnectionStatus.Connecting })
this.connectToPeer();
} else {
update({ peerConnectionStatus: ConnectionStatus.Disconnected })
update({ peerConnectionStatus: ConnectionStatus.Disconnected });
this.dataCheckIntervalID && clearInterval(this.dataCheckIntervalID);
}
} else {
console.error(`PeerJS error (on peer). Type ${e.type}`, e);
@ -134,27 +150,26 @@ export default class AssistManager {
}
})
peer.on("open", me => {
console.log("peer opened", me);
this.connectToPeer();
});
});
}
private dataCheckIntervalID: ReturnType<typeof setInterval> | undefined;
private connectToPeer() {
if (!this.peer) { return; }
update({ peerConnectionStatus: ConnectionStatus.Connecting })
const id = this.peerID;
console.log("trying to connect to", id)
const conn = this.peer.connect(id, { serialization: 'json', reliable: true});
conn.on('open', () => {
update({ peerConnectionStatus: ConnectionStatus.Inactive });
console.log("peer connected")
let i = 0;
let firstMessage = true;
conn.on('data', (data) => {
if (typeof data === 'string') { return this.handleCommand(data); }
if (!Array.isArray(data)) { return; }
if (!Array.isArray(data)) { return this.handleCommand(data); }
if (firstMessage) {
firstMessage = false;
this.md.setMessagesLoading(false);
@ -204,25 +219,21 @@ export default class AssistManager {
});
});
const intervalID = setInterval(() => {
if (!conn.open) {
this.md.setMessagesLoading(true);
this.assistentCallEnd();
update({ peerConnectionStatus: ConnectionStatus.Disconnected })
clearInterval(intervalID);
const onDataClose = () => {
this.initiateCallEnd();
this.md.setMessagesLoading(true);
update({ peerConnectionStatus: ConnectionStatus.Connecting });
console.log('closed peer conn. Reconnecting...')
this.connectToPeer();
}
this.dataCheckIntervalID = setInterval(() => {
if (!this.dataConnection && getState().peerConnectionStatus === ConnectionStatus.Connected) {
onDataClose();
}
}, 5000);
conn.on('close', () => this.onDataClose());// Doesn't work ?
}
private onDataClose() {
this.md.setMessagesLoading(true);
this.assistentCallEnd();
console.log('closed peer conn. Reconnecting...')
update({ peerConnectionStatus: ConnectionStatus.Connecting })
setTimeout(() => this.connectToPeer(), 300); // reconnect
}, 3000);
conn.on('close', onDataClose);// Does it work ?
}
@ -234,23 +245,34 @@ export default class AssistManager {
return this.peer?.connections[this.peerID]?.find(c => c.type === 'media' && c.open);
}
private send(data: any) {
this.dataConnection?.send(data);
}
private onCallEnd: null | (()=>void) = null;
private assistentCallEnd = () => {
console.log('assistentCallEnd')
const conn = this.callConnection?.close();
private onReject: null | (()=>void) = null;
private forceCallEnd() {
this.callConnection?.close();
}
private notifyCallEnd() {
const dataConn = this.dataConnection;
if (dataConn) {
console.log("call_end send")
console.log("notifyCallEnd send")
dataConn.send("call_end");
}
}
private initiateCallEnd = () => {
console.log('initiateCallEnd')
this.forceCallEnd();
this.notifyCallEnd();
this.onCallEnd?.();
}
private onTrackerCallEnd = () => {
const conn = this.callConnection;
if (conn && conn.open) {
conn.close();
this.forceCallEnd();
if (getState().calling === CallingState.Requesting) {
this.onReject?.();
}
this.onCallEnd?.();
}
@ -258,6 +280,10 @@ export default class AssistManager {
private handleCommand(command: string) {
switch (command) {
case "unload":
this.onTrackerCallEnd();
this.dataConnection?.close();
return;
case "call_end":
this.onTrackerCallEnd();
return;
@ -268,28 +294,25 @@ export default class AssistManager {
}
}
//private blocked: boolean = false;
private onMouseMove = (e: MouseEvent ): void => {
//if (this.blocked) { return; }
//this.blocked = true;
//setTimeout(() => this.blocked = false, 200);
const conn = this.dataConnection;
if (!conn || !conn.open) { return; }
if (!conn) { return; }
// @ts-ignore ???
const data = this.md.getInternalCoordinates(e);
conn.send({ x: Math.round(data.x), y: Math.round(data.y) });
}
call(localStream: MediaStream, onStream: (s: MediaStream)=>void, onClose: () => void, onReject: () => void, onError?: ()=> void): null | Function {
call(localStream: MediaStream, onStream: (s: MediaStream)=>void, onCallEnd: () => void, onReject: () => void, onError?: ()=> void): null | Function {
if (!this.peer || getState().calling !== CallingState.False) { return null; }
update({ calling: CallingState.Requesting });
console.log('calling...')
console.log('calling...')
const call = this.peer.call(this.peerID, localStream);
call.on('stream', stream => {
update({ calling: CallingState.True });
onStream(stream);
this.dataConnection?.send({
this.send({
name: store.getState().getIn([ 'user', 'account', 'name']),
});
@ -298,38 +321,37 @@ export default class AssistManager {
});
this.onCallEnd = () => {
if (getState().calling === CallingState.Requesting) {
onReject();
}
onClose();
onCallEnd();
// @ts-ignore ??
this.md.overlay.removeEventListener("mousemove", this.onMouseMove);
update({ calling: CallingState.False });
this.onCallEnd = null;
}
//call.on("close", this.onCallEnd);
call.on("close", this.onCallEnd);
call.on("error", (e) => {
console.error("PeerJS error (on call):", e)
this.onCallEnd?.();
onError?.();
});
const intervalID = setInterval(() => {
if (!call.open && getState().calling !== CallingState.Requesting) {
this.onCallEnd?.();
clearInterval(intervalID);
}
}, 5000);
// const intervalID = setInterval(() => {
// if (!call.open && getState().calling === CallingState.True) {
// this.onCallEnd?.();
// clearInterval(intervalID);
// }
// }, 5000);
window.addEventListener("beforeunload", this.assistentCallEnd)
window.addEventListener("beforeunload", this.initiateCallEnd)
return this.assistentCallEnd;
return this.initiateCallEnd;
}
clear() {
console.log('clearing', this.peerID)
this.assistentCallEnd();
this.initiateCallEnd();
this.dataCheckIntervalID && clearInterval(this.dataCheckIntervalID);
this.dataConnection?.close();
console.log("destroying peer...")
this.peer?.destroy();
this.peer = null;

19
tracker/tracker-assist/src/CallWindow.ts
View file

@ -6,6 +6,7 @@ export default class CallWindow {
private vLocal: HTMLVideoElement | null = null;
private audioBtn: HTMLAnchorElement | null = null;
private videoBtn: HTMLAnchorElement | null = null;
private userNameSpan: HTMLSpanElement | null = null;
private tsInterval: ReturnType<typeof setInterval>;
constructor(endCall: () => void) {
@ -18,6 +19,7 @@ export default class CallWindow {
border: "none",
bottom: "10px",
right: "10px",
display: "none",
});
//iframe.src = "//static.openreplay.com/tracker-assist/index.html";
iframe.onload = () => {
@ -31,6 +33,7 @@ export default class CallWindow {
.then(r => r.text())
.then((text) => {
iframe.onload = () => {
iframe.style.display = "block";
iframe.style.height = doc.body.scrollHeight + 'px';
iframe.style.width = doc.body.scrollWidth + 'px';
}
@ -51,6 +54,9 @@ export default class CallWindow {
this.videoBtn = doc.getElementById("video-btn") as HTMLAnchorElement;
this.videoBtn.onclick = () => this.toggleVideo();
this.userNameSpan = doc.getElementById("username") as HTMLSpanElement;
this._trySetAssistentName();
const endCallBtn = doc.getElementById("end-call-btn") as HTMLAnchorElement;
endCallBtn.onclick = endCall;
@ -109,6 +115,19 @@ export default class CallWindow {
this._trySetStreams();
}
// TODO: determined workflow
_trySetAssistentName() {
if (this.userNameSpan && this.assistentName) {
this.userNameSpan.innerText = this.assistentName;
}
}
private assistentName: string = "";
setAssistentName(name: string) {
this.assistentName = name;
this._trySetAssistentName();
}
toggleAudio() {
let enabled = true;
this.localStream?.getAudioTracks().forEach(track => {

7
tracker/tracker-assist/src/Confirm.ts
View file

@ -1,5 +1,5 @@
const declineIcon = `<svg xmlns="http://www.w3.org/2000/svg" height="20" width="20" viewBox="0 0 128 128" ><g id="Circle_Grid" data-name="Circle Grid"><circle cx="64" cy="64" fill="#ef5261" r="64"/></g><g id="icon"><path d="m57.831 70.1c8.79 8.79 17.405 12.356 20.508 9.253l4.261-4.26a7.516 7.516 0 0 1 10.629 0l9.566 9.566a7.516 7.516 0 0 1 0 10.629l-7.453 7.453c-7.042 7.042-27.87-2.358-47.832-22.319-9.976-9.981-16.519-19.382-20.748-28.222s-5.086-16.091-1.567-19.61l7.453-7.453a7.516 7.516 0 0 1 10.629 0l9.566 9.563a7.516 7.516 0 0 1 0 10.629l-4.264 4.271c-3.103 3.1.462 11.714 9.252 20.5z" fill="#eeefee"/></g></svg>`;
const declineIcon = `<svg xmlns="http://www.w3.org/2000/svg" height="22" width="22" viewBox="0 0 128 128" ><g id="Circle_Grid" data-name="Circle Grid"><circle cx="64" cy="64" fill="#ef5261" r="64"/></g><g id="icon"><path d="m57.831 70.1c8.79 8.79 17.405 12.356 20.508 9.253l4.261-4.26a7.516 7.516 0 0 1 10.629 0l9.566 9.566a7.516 7.516 0 0 1 0 10.629l-7.453 7.453c-7.042 7.042-27.87-2.358-47.832-22.319-9.976-9.981-16.519-19.382-20.748-28.222s-5.086-16.091-1.567-19.61l7.453-7.453a7.516 7.516 0 0 1 10.629 0l9.566 9.563a7.516 7.516 0 0 1 0 10.629l-4.264 4.271c-3.103 3.1.462 11.714 9.252 20.5z" fill="#eeefee"/></g></svg>`;
export default class Confirm {
private wrapper: HTMLDivElement;
@ -21,8 +21,8 @@ export default class Confirm {
const btnStyles = {
borderRadius: "50%",
width: "20px",
height: "20px",
width: "22px",
height: "22px",
background: "transparent",
padding: 0,
margin: 0,
@ -32,6 +32,7 @@ export default class Confirm {
Object.assign(answerBtn.style, btnStyles);
Object.assign(declineBtn.style, btnStyles);
Object.assign(buttons.style, {
marginTop: "10px",
display: "flex",
alignItems: "center",
justifyContent: "space-evenly",

7
tracker/tracker-assist/src/confirm.ts
View file

@ -1,5 +1,5 @@
const declineIcon = `<svg xmlns="http://www.w3.org/2000/svg" height="20" width="20" viewBox="0 0 128 128" ><g id="Circle_Grid" data-name="Circle Grid"><circle cx="64" cy="64" fill="#ef5261" r="64"/></g><g id="icon"><path d="m57.831 70.1c8.79 8.79 17.405 12.356 20.508 9.253l4.261-4.26a7.516 7.516 0 0 1 10.629 0l9.566 9.566a7.516 7.516 0 0 1 0 10.629l-7.453 7.453c-7.042 7.042-27.87-2.358-47.832-22.319-9.976-9.981-16.519-19.382-20.748-28.222s-5.086-16.091-1.567-19.61l7.453-7.453a7.516 7.516 0 0 1 10.629 0l9.566 9.563a7.516 7.516 0 0 1 0 10.629l-4.264 4.271c-3.103 3.1.462 11.714 9.252 20.5z" fill="#eeefee"/></g></svg>`;
const declineIcon = `<svg xmlns="http://www.w3.org/2000/svg" height="22" width="22" viewBox="0 0 128 128" ><g id="Circle_Grid" data-name="Circle Grid"><circle cx="64" cy="64" fill="#ef5261" r="64"/></g><g id="icon"><path d="m57.831 70.1c8.79 8.79 17.405 12.356 20.508 9.253l4.261-4.26a7.516 7.516 0 0 1 10.629 0l9.566 9.566a7.516 7.516 0 0 1 0 10.629l-7.453 7.453c-7.042 7.042-27.87-2.358-47.832-22.319-9.976-9.981-16.519-19.382-20.748-28.222s-5.086-16.091-1.567-19.61l7.453-7.453a7.516 7.516 0 0 1 10.629 0l9.566 9.563a7.516 7.516 0 0 1 0 10.629l-4.264 4.271c-3.103 3.1.462 11.714 9.252 20.5z" fill="#eeefee"/></g></svg>`;
export default class Confirm {
private wrapper: HTMLDivElement;
@ -21,8 +21,8 @@ export default class Confirm {
const btnStyles = {
borderRadius: "50%",
width: "20px",
height: "20px",
width: "22px",
height: "22px",
background: "transparent",
padding: 0,
margin: 0,
@ -32,6 +32,7 @@ export default class Confirm {
Object.assign(answerBtn.style, btnStyles);
Object.assign(declineBtn.style, btnStyles);
Object.assign(buttons.style, {
marginTop: "10px",
display: "flex",
alignItems: "center",
justifyContent: "space-evenly",

118
tracker/tracker-assist/src/index.ts
View file

@ -14,6 +14,12 @@ export interface Options {
}
enum CallingState {
Requesting,
True,
False,
};
export default function(opts: Partial<Options> = {}) {
const options: Options = Object.assign(
{
@ -28,11 +34,7 @@ export default function(opts: Partial<Options> = {}) {
return;
}
let callingPeerDataConn
app.attachStartCallback(function() {
// new CallWindow(()=>{console.log('endcall')});
// @ts-ignore
const peerID = `${app.projectKey}-${app.getSessionID()}`
const peer = new Peer(peerID, {
@ -43,6 +45,8 @@ export default function(opts: Partial<Options> = {}) {
});
console.log(peerID)
peer.on('connection', function(conn) {
window.addEventListener("beforeunload", () => conn.open && conn.send("unload"));
console.log('connection')
conn.on('open', function() {
@ -52,14 +56,14 @@ export default function(opts: Partial<Options> = {}) {
const buffer: Message[][] = [];
let buffering = false;
function sendNext() {
setTimeout(() => {
if (buffer.length) {
conn.send(buffer.shift());
sendNext();
} else {
buffering = false;
}
}, 50);
if (buffer.length) {
setTimeout(() => {
conn.send(buffer.shift());
sendNext();
}, 50);
} else {
buffering = false;
}
}
app.stop();
//@ts-ignore (should update tracker dependency)
@ -76,32 +80,36 @@ export default function(opts: Partial<Options> = {}) {
app.start();
});
});
let calling = false;
let calling: CallingState = CallingState.False;
peer.on('call', function(call) {
const dataConn: DataConnection | undefined = peer
.connections[call.peer].find(c => c.type === 'data');
if (calling || !dataConn) {
if (calling !== CallingState.False || !dataConn) {
call.close();
dataConn?.send("call_error");
return;
}
calling = true;
window.addEventListener("beforeunload", () => {
calling = CallingState.Requesting;
const notifyCallEnd = () => {
dataConn.open && dataConn.send("call_end");
});
dataConn.on('data', (data) => { // if call closed be a caller before confirm
}
const confirm = new Confirm(options.confirmText, options.confirmStyle);
dataConn.on('data', (data) => { // if call closed by a caller before confirm
if (data === "call_end") {
calling = false;
confirm.remove();
console.log('receiving callend onconfirm')
calling = CallingState.False;
confirm.remove();
}
});
const confirm = new Confirm(options.confirmText, options.confirmStyle);
confirm.mount();
confirm.onAnswer(conf => {
if (!conf || !dataConn.open) {
confirm.onAnswer(agreed => {
if (!agreed || !dataConn.open) {
call.close();
dataConn.open && dataConn.send("call_end");
calling = false;
notifyCallEnd();
calling = CallingState.False;
return;
}
@ -109,41 +117,53 @@ export default function(opts: Partial<Options> = {}) {
let callUI;
navigator.mediaDevices.getUserMedia({video:true, audio:true})
.then(oStream => {
const onClose = () => {
console.log("close call...")
if (call.open) { call.close(); }
.then(lStream => {
const onCallEnd = () => {
console.log("on callend", call.open)
mouse.remove();
callUI?.remove();
oStream.getTracks().forEach(t => t.stop());
calling = false;
if (dataConn.open) {
dataConn.send("call_end");
}
lStream.getTracks().forEach(t => t.stop());
calling = CallingState.False;
}
const initiateCallEnd = () => {
console.log("callend initiated")
call.close()
notifyCallEnd();
onCallEnd();
}
dataConn.on("close", onClose);
call.answer(oStream);
call.on('close', onClose); // Works from time to time (peerjs bug)
call.answer(lStream);
dataConn.on("close", onCallEnd);
//call.on('close', onClose); // Works from time to time (peerjs bug)
const intervalID = setInterval(() => {
if (!call.open) {
onClose();
if (!dataConn.open) {
initiateCallEnd();
clearInterval(intervalID);
}
}, 5000);
call.on('error', onClose); // notify about error?
if (!call.open) {
onCallEnd();
clearInterval(intervalID);
}
}, 3000);
call.on('error', initiateCallEnd);
callUI = new CallWindow(onClose);
callUI.setLocalStream(oStream);
call.on('stream', function(iStream) {
callUI.setRemoteStream(iStream);
callUI = new CallWindow(initiateCallEnd);
callUI.setLocalStream(lStream);
call.on('stream', function(rStream) {
callUI.setRemoteStream(rStream);
dataConn.on('data', (data: any) => {
if (data === "call_end") {
onClose();
console.log('receiving callend on call')
onCallEnd();
return;
}
if (call.open && data && typeof data.x === 'number' && typeof data.y === 'number') {
if (data && typeof data.name === 'string') {
console.log("name",data)
callUI.setAssistentName(data.name);
}
if (data && typeof data.x === 'number' && typeof data.y === 'number') {
mouse.move(data);
}
});