diff --git a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx index de9f037d7..8b2cf5245 100644 --- a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx +++ b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx @@ -5,8 +5,8 @@ import cn from 'classnames' import { toggleChatWindow } from 'Duck/sessions'; import { connectPlayer } from 'Player/store'; import ChatWindow from '../../ChatWindow'; -import { callPeer } from 'Player' -import { CallingState, ConnectionStatus } from 'Player/MessageDistributor/managers/AssistManager'; +import { callPeer, requestReleaseRemoteControl } from 'Player' +import { CallingState, ConnectionStatus, RemoteControlStatus } from 'Player/MessageDistributor/managers/AssistManager'; import RequestLocalStream from 'Player/MessageDistributor/managers/LocalStream'; import type { LocalStream } from 'Player/MessageDistributor/managers/LocalStream'; @@ -32,15 +32,15 @@ interface Props { toggleChatWindow: (state) => void, calling: CallingState, peerConnectionStatus: ConnectionStatus, - remoteControlEnabled: boolean, + remoteControlStatus: RemoteControlStatus, hasPermission: boolean, isEnterprise: boolean, } -function AssistActions({ toggleChatWindow, userId, calling, peerConnectionStatus, remoteControlEnabled, hasPermission, isEnterprise }: Props) { +function AssistActions({ toggleChatWindow, userId, calling, peerConnectionStatus, remoteControlStatus, hasPermission, isEnterprise }: Props) { const [ incomeStream, setIncomeStream ] = useState(null); const [ localStream, setLocalStream ] = useState(null); - const [ callObject, setCallObject ] = useState<{ end: ()=>void, toggleRemoteControl: ()=>void } | null >(null); + const [ callObject, setCallObject ] = useState<{ end: ()=>void } | null >(null); useEffect(() => { return callObject?.end() @@ -75,7 +75,7 @@ function AssistActions({ toggleChatWindow, userId, calling, peerConnectionStatus } } - const inCall = calling !== CallingState.False; + const onCall = calling === CallingState.OnCall || calling === CallingState.Reconnecting const cannotCall = (peerConnectionStatus !== ConnectionStatus.Connected) || (isEnterprise && !hasPermission) return ( @@ -86,19 +86,18 @@ function AssistActions({ toggleChatWindow, userId, calling, peerConnectionStatus className={ cn( 'cursor-pointer p-2 mr-2 flex items-center', - // {[stl.inCall] : inCall }, {[stl.disabled]: cannotCall} ) } - onClick={ inCall ? callObject?.end : confirmCall} + onClick={ onCall ? callObject?.end : confirmCall} role="button" > - { inCall ? 'End Call' : 'Call' } + { onCall ? 'End Call' : 'Call' } } content={ cannotCall ? "You don’t have the permissions to perform this action." : `Call ${userId ? userId : 'User'}` } @@ -106,26 +105,24 @@ function AssistActions({ toggleChatWindow, userId, calling, peerConnectionStatus inverted position="top right" /> - { calling === CallingState.True && -
- - { 'Remote Control' } -
- } +
+ + { 'Remote Control' } +
- { inCall && callObject && } + { onCall && callObject && }
) @@ -141,6 +138,6 @@ const con = connect(state => { export default con(connectPlayer(state => ({ calling: state.calling, - remoteControlEnabled: state.remoteControl, + remoteControlStatus: state.remoteControl, peerConnectionStatus: state.peerConnectionStatus, }))(AssistActions)) diff --git a/frontend/app/player/MessageDistributor/MessageDistributor.ts b/frontend/app/player/MessageDistributor/MessageDistributor.ts index 69e4b4836..ed3c49cce 100644 --- a/frontend/app/player/MessageDistributor/MessageDistributor.ts +++ b/frontend/app/player/MessageDistributor/MessageDistributor.ts @@ -181,7 +181,6 @@ export default class MessageDistributor extends StatedScreen { while (r.hasNext()) { const next = r.next(); if (next != null) { - this.lastMessageTime = next[0].time; this.distributeMessage(next[0], next[1]); msgs.push(next[0]); } @@ -326,6 +325,8 @@ export default class MessageDistributor extends StatedScreen { /* Binded */ distributeMessage = (msg: Message, index: number): void => { + this.lastMessageTime = msg.time; + if ([ "mouse_move", "mouse_click", diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager.ts b/frontend/app/player/MessageDistributor/managers/AssistManager.ts index 2b0ac4e63..3edf85b4c 100644 --- a/frontend/app/player/MessageDistributor/managers/AssistManager.ts +++ b/frontend/app/player/MessageDistributor/managers/AssistManager.ts @@ -1,5 +1,6 @@ +import type { Socket } from 'socket.io-client'; import type Peer from 'peerjs'; -import type { DataConnection, MediaConnection } from 'peerjs'; +import type { MediaConnection } from 'peerjs'; import type MessageDistributor from '../MessageDistributor'; import type { Message } from '../messages' import store from 'App/store'; @@ -11,11 +12,12 @@ import MStreamReader from '../messages/MStreamReader';; import JSONRawMessageReader from '../messages/JSONRawMessageReader' export enum CallingState { - Reconnecting, + NoCall, + Connecting, Requesting, - True, - False, -}; + Reconnecting, + OnCall, +} export enum ConnectionStatus { Connecting, @@ -24,7 +26,13 @@ export enum ConnectionStatus { Inactive, Disconnected, Error, -}; +} + +export enum RemoteControlStatus { + Disabled = 0, + Requesting, + Enabled, +} export function getStatusText(status: ConnectionStatus): string { @@ -47,13 +55,13 @@ export function getStatusText(status: ConnectionStatus): string { export interface State { calling: CallingState, peerConnectionStatus: ConnectionStatus, - remoteControl: boolean, + remoteControl: RemoteControlStatus, } export const INITIAL_STATE: State = { - calling: CallingState.False, + calling: CallingState.NoCall, peerConnectionStatus: ConnectionStatus.Connecting, - remoteControl: false, + remoteControl: RemoteControlStatus.Disabled, } const MAX_RECONNECTION_COUNT = 4; @@ -85,200 +93,112 @@ export default class AssistManager { return `${this.session.projectKey}-${this.session.sessionId}` } - private peer: Peer | null = null; - connectionAttempts: number = 0; - private peeropened: boolean = false; - connect() { - if (this.peer != null) { - console.error("AssistManager: trying to connect more than once"); - return; + private onVisChange = () => { + let inactiveTimeout: ReturnType | undefined + if (document.hidden) { + inactiveTimeout = setTimeout(() => { + if (document.hidden && getState().calling === CallingState.NoCall) { + this.socket?.close() + } + }, 2000) // TODO: test on 2000 + } else { + inactiveTimeout && clearTimeout(inactiveTimeout) + this.socket?.open() } - this.setStatus(ConnectionStatus.Connecting) - // @ts-ignore - const urlObject = new URL(window.ENV.API_EDP) - import('peerjs').then(({ default: Peer }) => { - if (this.closed) {return} - const _config = { - host: urlObject.hostname, - path: '/assist', - port: urlObject.port === "" ? (location.protocol === 'https:' ? 443 : 80 ): parseInt(urlObject.port), - } - - if (this.config) { - _config['config'] = { - iceServers: this.config, - sdpSemantics: 'unified-plan', - iceTransportPolicy: 'relay', - }; - } - - const peer = new Peer(_config); - this.peer = peer; - peer.on('error', e => { - if (e.type !== 'peer-unavailable') { - console.warn("AssistManager PeerJS peer error: ", e.type, e) - } - if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) { - if (this.peer) { - this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT - ? ConnectionStatus.Connecting - : ConnectionStatus.Disconnected); - this.connectToPeer(); - } - } else { - console.error(`PeerJS error (on peer). Type ${e.type}`, e); - this.setStatus(ConnectionStatus.Error) - } - }) - peer.on("open", () => { - if (this.peeropened) { return; } - this.peeropened = true; - this.connectToPeer(); - }); - }); } - private connectToPeer() { - if (!this.peer) { return; } - this.setStatus(ConnectionStatus.Connecting); - const id = this.peerID; - const conn = this.peer.connect(id, { serialization: "json", reliable: true}); - conn.on('open', () => { - window.addEventListener("beforeunload", ()=>conn.open &&conn.send("unload")); + private socket: Socket | null = null + connect() { + const jmr = new JSONRawMessageReader() + const reader = new MStreamReader(jmr) + let waitingForMessages = true + let showDisconnectTimeout: ReturnType | undefined + import('socket.io-client').then(({ default: io }) => { + if (this.cleaned) { return } + if (this.socket) { this.socket.close() } // TODO: single socket connection + // @ts-ignore + const urlObject = new URL(window.ENV.API_EDP) // does it handle ssl automatically? - //console.log("peer connected") + // @ts-ignore WTF, socket.io ??? + const socket: Socket = this.socket = io(urlObject.origin, { + path: '/ws-assist/socket', + query: { + peerId: this.peerID, + identity: "agent", + //agentInfo: JSON.stringify({}) + } + }) + //socket.onAny((...args) => console.log(...args)) + socket.on("connect", () => { + waitingForMessages = true + this.setStatus(ConnectionStatus.WaitingMessages) + }) + socket.on('messages', messages => { + showDisconnectTimeout && clearTimeout(showDisconnectTimeout); + jmr.append(messages) // as RawMessage[] - - if (getState().calling === CallingState.Reconnecting) { - this._call() - } - - let firstMessage = true; - - this.setStatus(ConnectionStatus.WaitingMessages) - - const jmr = new JSONRawMessageReader() - const reader = new MStreamReader(jmr) - - conn.on('data', (data) => { - this.disconnectTimeout && clearTimeout(this.disconnectTimeout); - - - if (Array.isArray(data)) { - jmr.append(data) // as RawMessage[] - } else if (data instanceof ArrayBuffer) { - //rawMessageReader.append(new Uint8Array(data)) - } else { return this.handleCommand(data); } - - if (firstMessage) { - firstMessage = false; + if (waitingForMessages) { + waitingForMessages = false // TODO: more explicit this.setStatus(ConnectionStatus.Connected) + + // Call State + if (getState().calling === CallingState.Reconnecting) { + this._call() // reconnecting call (todo improve code separation) + } } for (let msg = reader.readNext();msg !== null;msg = reader.readNext()) { //@ts-ignore - this.md.distributeMessage(msg, msg._index); + this.md.distributeMessage(msg, msg._index) } - }); - }); + }) + socket.on("control_granted", id => { + this.toggleRemoteControl(id === socket.id) + }) + socket.on("control_rejected", id => { + id === socket.id && this.toggleRemoteControl(false) + }) + socket.on('SESSION_DISCONNECTED', e => { + waitingForMessages = true + showDisconnectTimeout = setTimeout(() => { + if (this.cleaned) { return } + this.setStatus(ConnectionStatus.Disconnected) + }, 12000) + // Call State + if (getState().calling === CallingState.OnCall) { + update({ calling: CallingState.Reconnecting }) + } + }) + socket.on('error', e => { + console.warn("Socket error: ", e ) + this.setStatus(ConnectionStatus.Error); + }) + socket.on('call_end', this.onRemoteCallEnd) - const onDataClose = () => { - this.onCallDisconnect() - this.connectToPeer(); - } + document.addEventListener('visibilitychange', this.onVisChange) - conn.on('close', onDataClose);// What case does it work ? - conn.on("error", (e) => { - this.setStatus(ConnectionStatus.Error); }) } - - private get dataConnection(): DataConnection | undefined { - return this.peer?.connections[this.peerID]?.find(c => c.type === 'data' && c.open); - } - private get callConnection(): MediaConnection | undefined { - return this.peer?.connections[this.peerID]?.find(c => c.type === 'media' && c.open); - } - private send(data: any) { - this.dataConnection?.send(data); - } - - - private forceCallEnd() { - this.callConnection?.close(); - } - private notifyCallEnd() { - const dataConn = this.dataConnection; - if (dataConn) { - dataConn.send("call_end"); - } - } - private initiateCallEnd = () => { - this.forceCallEnd(); - this.notifyCallEnd(); - this.localCallData && this.localCallData.onCallEnd(); - } - - private onTrackerCallEnd = () => { - console.log('onTrackerCallEnd') - this.forceCallEnd(); - if (getState().calling === CallingState.Requesting) { - this.localCallData && this.localCallData.onReject(); - } - this.localCallData && this.localCallData.onCallEnd(); - } - - private onCallDisconnect = () => { - if (getState().calling === CallingState.True) { - update({ calling: CallingState.Reconnecting }); - } - } - - - private disconnectTimeout: ReturnType | undefined; - private closeDataConnectionTimeout: ReturnType | undefined; - private handleCommand(command: string) { - console.log("Data command", command) - switch (command) { - case "unload": - //this.onTrackerCallEnd(); - this.closeDataConnectionTimeout = setTimeout(() => { - this.onCallDisconnect() - this.dataConnection?.close(); - }, 1500); - this.disconnectTimeout = setTimeout(() => { - this.onTrackerCallEnd(); - this.setStatus(ConnectionStatus.Disconnected); - }, 15000); // TODO: more convenient way - return; - case "call_end": - this.onTrackerCallEnd(); - return; - case "call_error": - this.onTrackerCallEnd(); - this.setStatus(ConnectionStatus.Error); - return; - } - } + /* ==== Remote Control ==== */ private onMouseMove = (e: MouseEvent): void => { - const data = this.md.getInternalCoordinates(e); - this.send({ x: Math.round(data.x), y: Math.round(data.y) }); + if (!this.socket) { return } + const data = this.md.getInternalCoordinates(e) + this.socket.emit("move", [ Math.round(data.x), Math.round(data.y) ]) } - private onWheel = (e: WheelEvent): void => { e.preventDefault() + if (!this.socket) { return } //throttling makes movements less smooth, so it is omitted //this.onMouseMove(e) - this.send({ type: "scroll", delta: [ e.deltaX, e.deltaY ]}) + this.socket.emit("scroll", [ e.deltaX, e.deltaY ]) } private onMouseClick = (e: MouseEvent): void => { - const conn = this.dataConnection; - if (!conn) { return; } + if (!this.socket) { return; } const data = this.md.getInternalCoordinates(e); // const el = this.md.getElementFromPoint(e); // requires requestiong node_id from domManager const el = this.md.getElementFromInternalPoint(data) @@ -287,25 +207,114 @@ export default class AssistManager { el.oninput = e => e.preventDefault(); el.onkeydown = e => e.preventDefault(); } - conn.send({ type: "click", x: Math.round(data.x), y: Math.round(data.y) }); + this.socket.emit("click", [ Math.round(data.x), Math.round(data.y) ]); } - private toggleRemoteControl = (flag?: boolean) => { - const state = getState().remoteControl; - const newState = typeof flag === 'boolean' ? flag : !state; - if (state === newState) { return } + private toggleRemoteControl(newState: boolean){ if (newState) { - this.md.overlay.addEventListener("click", this.onMouseClick); + this.md.overlay.addEventListener("mousemove", this.onMouseMove) + this.md.overlay.addEventListener("click", this.onMouseClick) this.md.overlay.addEventListener("wheel", this.onWheel) - update({ remoteControl: true }) + update({ remoteControl: RemoteControlStatus.Enabled }) } else { - this.md.overlay.removeEventListener("click", this.onMouseClick); - this.md.overlay.removeEventListener("wheel", this.onWheel); - update({ remoteControl: false }) + this.md.overlay.removeEventListener("mousemove", this.onMouseMove) + this.md.overlay.removeEventListener("click", this.onMouseClick) + this.md.overlay.removeEventListener("wheel", this.onWheel) + update({ remoteControl: RemoteControlStatus.Disabled }) } } - private localCallData: { + requestReleaseRemoteControl = () => { + if (!this.socket) { return } + const remoteControl = getState().remoteControl + if (remoteControl === RemoteControlStatus.Requesting) { return } + if (remoteControl === RemoteControlStatus.Disabled) { + update({ remoteControl: RemoteControlStatus.Requesting }) + this.socket.emit("request_control") + // setTimeout(() => { + // if (getState().remoteControl !== RemoteControlStatus.Requesting) { return } + // this.socket?.emit("release_control") + // update({ remoteControl: RemoteControlStatus.Disabled }) + // }, 8000) + } else { + this.socket.emit("release_control") + this.toggleRemoteControl(false) + } + } + + + /* ==== PeerJS Call ==== */ + + private _peer: Peer | null = null + private connectionAttempts: number = 0 + private callConnection: MediaConnection | null = null + private getPeer(): Promise { + if (this._peer && !this._peer.disconnected) { return Promise.resolve(this._peer) } + + // @ts-ignore + const urlObject = new URL(window.ENV.API_EDP) + return import('peerjs').then(({ default: Peer }) => { + if (this.cleaned) {return Promise.reject("Already cleaned")} + const peerOpts = { + host: urlObject.hostname, + path: '/assist', + port: urlObject.port === "" ? (location.protocol === 'https:' ? 443 : 80 ): parseInt(urlObject.port), + } + if (this.config) { + peerOpts['config'] = { + iceServers: this.config, + sdpSemantics: 'unified-plan', + iceTransportPolicy: 'relay', + }; + } + const peer = this._peer = new Peer(peerOpts) + peer.on('error', e => { + if (e.type === 'disconnected') { + return peer.reconnect() + } else if (e.type !== 'peer-unavailable') { + console.error(`PeerJS error (on peer). Type ${e.type}`, e); + } + + //call-reconnection connected + // if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) { + // this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT + // ? ConnectionStatus.Connecting + // : ConnectionStatus.Disconnected); + // Reconnect... + }) + + return new Promise(resolve => { + peer.on("open", () => resolve(peer)) + }) + }); + + } + + + private handleCallEnd() { + this.callArgs && this.callArgs.onCallEnd() + this.callConnection && this.callConnection.close() + update({ calling: CallingState.NoCall }) + this.callArgs = null + } + + private initiateCallEnd = () => { + this.socket?.emit("call_end") + this.handleCallEnd() + } + + private onRemoteCallEnd = () => { + if (getState().calling === CallingState.Requesting) { + this.callArgs && this.callArgs.onReject() + this.callConnection && this.callConnection.close() + update({ calling: CallingState.NoCall }) + this.callArgs = null + } else { + this.handleCallEnd() + } + } + + private callArgs: { localStream: LocalStream, onStream: (s: MediaStream)=>void, onCallEnd: () => void, @@ -313,79 +322,79 @@ export default class AssistManager { onError?: ()=> void } | null = null - call(localStream: LocalStream, onStream: (s: MediaStream)=>void, onCallEnd: () => void, onReject: () => void, onError?: ()=> void): { end: Function, toggleRemoteControl: Function } { - this.localCallData = { + call( + localStream: LocalStream, + onStream: (s: MediaStream)=>void, + onCallEnd: () => void, + onReject: () => void, + onError?: ()=> void): { end: Function } { + this.callArgs = { localStream, onStream, - onCallEnd: () => { - onCallEnd(); - this.toggleRemoteControl(false); - this.md.overlay.removeEventListener("mousemove", this.onMouseMove); - this.md.overlay.removeEventListener("click", this.onMouseClick); - update({ calling: CallingState.False }); - this.localCallData = null; - }, + onCallEnd, onReject, onError, } this._call() return { end: this.initiateCallEnd, - toggleRemoteControl: this.toggleRemoteControl, } } private _call() { - if (!this.peer || !this.localCallData || ![CallingState.False, CallingState.Reconnecting].includes(getState().calling)) { return null; } - - update({ calling: CallingState.Requesting }); + if (![CallingState.NoCall, CallingState.Reconnecting].includes(getState().calling)) { return } + update({ calling: CallingState.Connecting }) + this.getPeer().then(peer => { + if (!this.callArgs) { return console.log("No call Args. Must not happen.") } + update({ calling: CallingState.Requesting }) - //console.log('calling...', this.localCallData.localStream) - - const call = this.peer.call(this.peerID, this.localCallData.localStream.stream); - this.localCallData.localStream.onVideoTrack(vTrack => { - const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video") - if (!sender) { - //logger.warn("No video sender found") - return - } - //logger.log("sender found:", sender) - sender.replaceTrack(vTrack) - }) + // TODO: in a proper way + this.socket && this.socket.emit("_agent_name", store.getState().getIn([ 'user', 'account', 'name'])) + + const call = this.callConnection = peer.call(this.peerID, this.callArgs.localStream.stream) + this.callArgs.localStream.onVideoTrack(vTrack => { + const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video") + if (!sender) { + console.warn("No video sender found") + return + } + //logger.log("sender found:", sender) + sender.replaceTrack(vTrack) + }) - call.on('stream', stream => { - update({ calling: CallingState.True }); - this.localCallData && this.localCallData.onStream(stream); - this.send({ - name: store.getState().getIn([ 'user', 'account', 'name']), + call.on('stream', stream => { + update({ calling: CallingState.OnCall }) + this.callArgs && this.callArgs.onStream(stream) + }); + //call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) + + call.on("close", this.onRemoteCallEnd) + call.on("error", (e) => { + console.error("PeerJS error (on call):", e) + this.initiateCallEnd(); + this.callArgs && this.callArgs.onError && this.callArgs.onError(); }); - this.md.overlay.addEventListener("mousemove", this.onMouseMove) - this.md.overlay.addEventListener("click", this.onMouseClick) - }); - //call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) - - call.on("close", this.localCallData.onCallEnd); - call.on("error", (e) => { - console.error("PeerJS error (on call):", e) - this.initiateCallEnd(); - this.localCallData && this.localCallData.onError && this.localCallData.onError(); - }); - - window.addEventListener("beforeunload", this.initiateCallEnd) + }) } - closed = false + + /* ==== Cleaning ==== */ + private cleaned: boolean = false clear() { - this.closed =true + this.cleaned = true // sometimes cleaned before modules loaded this.initiateCallEnd(); - if (this.peer) { + if (this._peer) { console.log("destroying peer...") - const peer = this.peer; // otherwise it calls reconnection on data chan close - this.peer = null; + const peer = this._peer; // otherwise it calls reconnection on data chan close + this._peer = null; peer.disconnect(); peer.destroy(); } + if (this.socket) { + this.socket.close() + document.removeEventListener('visibilitychange', this.onVisChange) + } } } diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager_old.ts b/frontend/app/player/MessageDistributor/managers/AssistManager_old.ts deleted file mode 100644 index b901dc076..000000000 --- a/frontend/app/player/MessageDistributor/managers/AssistManager_old.ts +++ /dev/null @@ -1,486 +0,0 @@ -// import type Peer from 'peerjs'; -// import type { DataConnection, MediaConnection } from 'peerjs'; -// import type MessageDistributor from '../MessageDistributor'; -// import type { Message } from '../messages' -// import store from 'App/store'; -// import type { LocalStream } from './LocalStream'; -// import { update, getState } from '../../store'; -// import { iceServerConfigFromString } from 'App/utils' - - -// export enum CallingState { -// Reconnecting, -// Requesting, -// True, -// False, -// }; - -// export enum ConnectionStatus { -// Connecting, -// WaitingMessages, -// Connected, -// Inactive, -// Disconnected, -// Error, -// }; - - -// export function getStatusText(status: ConnectionStatus): string { -// switch(status) { -// case ConnectionStatus.Connecting: -// return "Connecting..."; -// case ConnectionStatus.Connected: -// return ""; -// case ConnectionStatus.Inactive: -// return "Client tab is inactive"; -// case ConnectionStatus.Disconnected: -// return "Disconnected"; -// case ConnectionStatus.Error: -// return "Something went wrong. Try to reload the page."; -// case ConnectionStatus.WaitingMessages: -// return "Connected. Waiting for the data... (The tab might be inactive)" -// } -// } - -// export interface State { -// calling: CallingState, -// peerConnectionStatus: ConnectionStatus, -// remoteControl: boolean, -// } - -// export const INITIAL_STATE: State = { -// calling: CallingState.False, -// peerConnectionStatus: ConnectionStatus.Connecting, -// remoteControl: false, -// } - -// const MAX_RECONNECTION_COUNT = 4; - - -// function resolveURL(baseURL: string, relURL: string): string { -// if (relURL.startsWith('#') || relURL === "") { -// return relURL; -// } -// return new URL(relURL, baseURL).toString(); -// } - - -// var match = /bar/.exec("foobar"); -// const re1 = /url\(("[^"]*"|'[^']*'|[^)]*)\)/g -// const re2 = /@import "(.*?)"/g -// function cssUrlsIndex(css: string): Array<[number, number]> { -// const idxs: Array<[number, number]> = []; -// const i1 = css.matchAll(re1); -// // @ts-ignore -// for (let m of i1) { -// // @ts-ignore -// const s: number = m.index + m[0].indexOf(m[1]); -// const e: number = s + m[1].length; -// idxs.push([s, e]); -// } -// const i2 = css.matchAll(re2); -// // @ts-ignore -// for (let m of i2) { -// // @ts-ignore -// const s = m.index + m[0].indexOf(m[1]); -// const e = s + m[1].length; -// idxs.push([s, e]) -// } -// return idxs; -// } -// function unquote(str: string): [string, string] { -// str = str.trim(); -// if (str.length <= 2) { -// return [str, ""] -// } -// if (str[0] == '"' && str[str.length-1] == '"') { -// return [ str.substring(1, str.length-1), "\""]; -// } -// if (str[0] == '\'' && str[str.length-1] == '\'') { -// return [ str.substring(1, str.length-1), "'" ]; -// } -// return [str, ""] -// } -// function rewriteCSSLinks(css: string, rewriter: (rawurl: string) => string): string { -// for (let idx of cssUrlsIndex(css)) { -// const f = idx[0] -// const t = idx[1] -// const [ rawurl, q ] = unquote(css.substring(f, t)); -// css = css.substring(0,f) + q + rewriter(rawurl) + q + css.substring(t); -// } -// return css -// } - -// function resolveCSS(baseURL: string, css: string): string { -// return rewriteCSSLinks(css, rawurl => resolveURL(baseURL, rawurl)); -// } - -// export default class AssistManager { -// constructor(private session, private md: MessageDistributor, private config) {} - -// private setStatus(status: ConnectionStatus) { -// if (status === ConnectionStatus.Connecting) { -// this.md.setMessagesLoading(true); -// } else { -// this.md.setMessagesLoading(false); -// } -// if (status === ConnectionStatus.Connected) { -// this.md.display(true); -// } else { -// this.md.display(false); -// } -// update({ peerConnectionStatus: status }); -// } - -// private get peerID(): string { -// return `${this.session.projectKey}-${this.session.sessionId}` -// } - -// private peer: Peer | null = null; -// connectionAttempts: number = 0; -// private peeropened: boolean = false; -// connect() { -// if (this.peer != null) { -// console.error("AssistManager: trying to connect more than once"); -// return; -// } -// this.setStatus(ConnectionStatus.Connecting) -// import('peerjs').then(({ default: Peer }) => { -// const _config = { -// // @ts-ignore -// host: new URL(window.ENV.API_EDP).host, -// path: '/assist', -// port: location.protocol === 'https:' ? 443 : 80, -// } - -// if (this.config) { -// _config['config'] = { -// iceServers: this.config, -// sdpSemantics: 'unified-plan', -// iceTransportPolicy: 'relay', -// }; -// } - -// const peer = new Peer(_config); -// this.peer = peer; -// peer.on('error', e => { -// if (e.type !== 'peer-unavailable') { -// console.warn("AssistManager PeerJS peer error: ", e.type, e) -// } -// if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) { -// if (this.peer && this.connectionAttempts++ < MAX_RECONNECTION_COUNT) { -// this.setStatus(ConnectionStatus.Connecting); -// this.connectToPeer(); -// } else { -// this.setStatus(ConnectionStatus.Disconnected); -// this.dataCheckIntervalID && clearInterval(this.dataCheckIntervalID); -// } -// } else { -// console.error(`PeerJS error (on peer). Type ${e.type}`, e); -// this.setStatus(ConnectionStatus.Error) -// } -// }) -// peer.on("open", () => { -// if (this.peeropened) { return; } -// this.peeropened = true; -// this.connectToPeer(); -// }); -// }); -// } - -// private dataCheckIntervalID: ReturnType | undefined; -// private connectToPeer() { -// if (!this.peer) { return; } -// this.setStatus(ConnectionStatus.Connecting); -// const id = this.peerID; -// const conn = this.peer.connect(id, { serialization: 'json', reliable: true}); -// conn.on('open', () => { -// window.addEventListener("beforeunload", ()=>conn.open &&conn.send("unload")); - -// //console.log("peer connected") - - -// if (getState().calling === CallingState.Reconnecting) { -// this._call() -// } - -// let i = 0; -// let firstMessage = true; - -// this.setStatus(ConnectionStatus.WaitingMessages) - -// conn.on('data', (data) => { -// if (!Array.isArray(data)) { return this.handleCommand(data); } -// this.disconnectTimeout && clearTimeout(this.disconnectTimeout); -// if (firstMessage) { -// firstMessage = false; -// this.setStatus(ConnectionStatus.Connected) -// } - -// let time = 0; -// let ts0 = 0; -// (data as Array).forEach(msg => { - -// // TODO: more appropriate way to do it. -// if (msg._id === 60) { -// // @ts-ignore -// if (msg.name === 'src' || msg.name === 'href') { -// // @ts-ignore -// msg.value = resolveURL(msg.baseURL, msg.value); -// // @ts-ignore -// } else if (msg.name === 'style') { -// // @ts-ignore -// msg.value = resolveCSS(msg.baseURL, msg.value); -// } -// msg._id = 12; -// } else if (msg._id === 61) { // "SetCSSDataURLBased" -// // @ts-ignore -// msg.data = resolveCSS(msg.baseURL, msg.data); -// msg._id = 15; -// } else if (msg._id === 67) { // "insert_rule" -// // @ts-ignore -// msg.rule = resolveCSS(msg.baseURL, msg.rule); -// msg._id = 37; -// } - - -// msg.tp = ID_TP_MAP[msg._id]; // _id goes from tracker - -// if (msg.tp === "timestamp") { -// ts0 = ts0 || msg.timestamp -// time = msg.timestamp - ts0; -// return; -// } -// const tMsg: TimedMessage = Object.assign(msg, { -// time, -// _index: i, -// }); -// this.md.distributeMessage(tMsg, i++); -// }); -// }); -// }); - - -// const onDataClose = () => { -// this.onCallDisconnect() -// //console.log('closed peer conn. Reconnecting...') -// this.connectToPeer(); -// } - -// // this.dataCheckIntervalID = setInterval(() => { -// // if (!this.dataConnection && getState().peerConnectionStatus === ConnectionStatus.Connected) { -// // onDataClose(); -// // } -// // }, 3000); -// conn.on('close', onDataClose);// Does it work ? -// conn.on("error", (e) => { -// this.setStatus(ConnectionStatus.Error); -// }) -// } - - -// private get dataConnection(): DataConnection | undefined { -// return this.peer?.connections[this.peerID]?.find(c => c.type === 'data' && c.open); -// } - -// private get callConnection(): MediaConnection | undefined { -// return this.peer?.connections[this.peerID]?.find(c => c.type === 'media' && c.open); -// } - -// private send(data: any) { -// this.dataConnection?.send(data); -// } - - -// private forceCallEnd() { -// this.callConnection?.close(); -// } -// private notifyCallEnd() { -// const dataConn = this.dataConnection; -// if (dataConn) { -// dataConn.send("call_end"); -// } -// } -// private initiateCallEnd = () => { -// this.forceCallEnd(); -// this.notifyCallEnd(); -// this.localCallData && this.localCallData.onCallEnd(); -// } - -// private onTrackerCallEnd = () => { -// console.log('onTrackerCallEnd') -// this.forceCallEnd(); -// if (getState().calling === CallingState.Requesting) { -// this.localCallData && this.localCallData.onReject(); -// } -// this.localCallData && this.localCallData.onCallEnd(); -// } - -// private onCallDisconnect = () => { -// if (getState().calling === CallingState.True) { -// update({ calling: CallingState.Reconnecting }); -// } -// } - - -// private disconnectTimeout: ReturnType | undefined; -// private handleCommand(command: string) { -// console.log("Data command", command) -// switch (command) { -// case "unload": -// //this.onTrackerCallEnd(); -// this.onCallDisconnect() -// this.dataConnection?.close(); -// this.disconnectTimeout = setTimeout(() => { -// this.onTrackerCallEnd(); -// this.setStatus(ConnectionStatus.Disconnected); -// }, 15000); // TODO: more convenient way -// //this.dataConnection?.close(); -// return; -// case "call_end": -// this.onTrackerCallEnd(); -// return; -// case "call_error": -// this.onTrackerCallEnd(); -// this.setStatus(ConnectionStatus.Error); -// return; -// } -// } - -// // private mmtid?:ReturnType -// private onMouseMove = (e: MouseEvent): void => { -// // this.mmtid && clearTimeout(this.mmtid) -// // this.mmtid = setTimeout(() => { -// const data = this.md.getInternalCoordinates(e); -// this.send({ x: Math.round(data.x), y: Math.round(data.y) }); -// // }, 5) -// } - - -// // private wtid?: ReturnType -// // private scrollDelta: [number, number] = [0,0] -// private onWheel = (e: WheelEvent): void => { -// e.preventDefault() -// //throttling makes movements less smooth -// // this.wtid && clearTimeout(this.wtid) -// // this.scrollDelta[0] += e.deltaX -// // this.scrollDelta[1] += e.deltaY -// // this.wtid = setTimeout(() => { -// this.send({ type: "scroll", delta: [ e.deltaX, e.deltaY ]})//this.scrollDelta }); -// this.onMouseMove(e) -// // this.scrollDelta = [0,0] -// // }, 20) -// } - -// private onMouseClick = (e: MouseEvent): void => { -// const conn = this.dataConnection; -// if (!conn) { return; } -// const data = this.md.getInternalCoordinates(e); -// // const el = this.md.getElementFromPoint(e); // requires requestiong node_id from domManager -// const el = this.md.getElementFromInternalPoint(data) -// if (el instanceof HTMLElement) { -// el.focus() -// el.oninput = e => e.preventDefault(); -// el.onkeydown = e => e.preventDefault(); -// } -// conn.send({ type: "click", x: Math.round(data.x), y: Math.round(data.y) }); -// } - -// private toggleRemoteControl = (flag?: boolean) => { -// const state = getState().remoteControl; -// const newState = typeof flag === 'boolean' ? flag : !state; -// if (state === newState) { return } -// if (newState) { -// this.md.overlay.addEventListener("click", this.onMouseClick); -// this.md.overlay.addEventListener("wheel", this.onWheel) -// update({ remoteControl: true }) -// } else { -// this.md.overlay.removeEventListener("click", this.onMouseClick); -// this.md.overlay.removeEventListener("wheel", this.onWheel); -// update({ remoteControl: false }) -// } -// } - -// private localCallData: { -// localStream: LocalStream, -// onStream: (s: MediaStream)=>void, -// onCallEnd: () => void, -// onReject: () => void, -// onError?: ()=> void -// } | null = null - -// call(localStream: LocalStream, onStream: (s: MediaStream)=>void, onCallEnd: () => void, onReject: () => void, onError?: ()=> void): { end: Function, toggleRemoteControl: Function } { -// this.localCallData = { -// localStream, -// onStream, -// onCallEnd: () => { -// onCallEnd(); -// this.toggleRemoteControl(false); -// this.md.overlay.removeEventListener("mousemove", this.onMouseMove); -// this.md.overlay.removeEventListener("click", this.onMouseClick); -// update({ calling: CallingState.False }); -// this.localCallData = null; -// }, -// onReject, -// onError, -// } -// this._call() -// return { -// end: this.initiateCallEnd, -// toggleRemoteControl: this.toggleRemoteControl, -// } -// } - -// private _call() { -// if (!this.peer || !this.localCallData || ![CallingState.False, CallingState.Reconnecting].includes(getState().calling)) { return null; } - -// update({ calling: CallingState.Requesting }); - -// //console.log('calling...', this.localCallData.localStream) - -// const call = this.peer.call(this.peerID, this.localCallData.localStream.stream); -// this.localCallData.localStream.onVideoTrack(vTrack => { -// const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video") -// if (!sender) { -// //logger.warn("No video sender found") -// return -// } -// //logger.log("sender found:", sender) -// sender.replaceTrack(vTrack) -// }) - -// call.on('stream', stream => { -// update({ calling: CallingState.True }); -// this.localCallData && this.localCallData.onStream(stream); -// this.send({ -// name: store.getState().getIn([ 'user', 'account', 'name']), -// }); - -// this.md.overlay.addEventListener("mousemove", this.onMouseMove) -// // this.md.overlay.addEventListener("click", this.onMouseClick) -// }); -// //call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) - -// call.on("close", this.localCallData.onCallEnd); -// call.on("error", (e) => { -// console.error("PeerJS error (on call):", e) -// this.initiateCallEnd(); -// this.localCallData && this.localCallData.onError && this.localCallData.onError(); -// }); - -// window.addEventListener("beforeunload", this.initiateCallEnd) -// } - -// clear() { -// this.initiateCallEnd(); -// this.dataCheckIntervalID && clearInterval(this.dataCheckIntervalID); -// if (this.peer) { -// //console.log("destroying peer...") -// const peer = this.peer; // otherwise it calls reconnection on data chan close -// this.peer = null; -// peer.destroy(); -// } -// } -// } - - diff --git a/frontend/app/player/Player.ts b/frontend/app/player/Player.ts index f99434be5..67875b530 100644 --- a/frontend/app/player/Player.ts +++ b/frontend/app/player/Player.ts @@ -87,7 +87,7 @@ export default class Player extends MessageDistributor { const diffTime = messagesLoading || cssLoading || disconnected ? 0 - : Math.max(animationCurrentTime - animationPrevTime, 0) * speed; + : Math.max(animationCurrentTime - animationPrevTime, 0) * (live ? 1 : speed); let time = prevTime + diffTime; diff --git a/frontend/app/player/singletone.js b/frontend/app/player/singletone.js index 9d811023e..177bb0388 100644 --- a/frontend/app/player/singletone.js +++ b/frontend/app/player/singletone.js @@ -69,6 +69,7 @@ export const markElement = initCheck((...args) => instance.marker && instance.ma export const scale = initCheck(() => instance.scale()); export const toggleInspectorMode = initCheck((...args) => instance.toggleInspectorMode(...args)); export const callPeer = initCheck((...args) => instance.assistManager.call(...args)) +export const requestReleaseRemoteControl = initCheck((...args) => instance.assistManager.requestReleaseRemoteControl(...args)) export const markTargets = initCheck((...args) => instance.markTargets(...args)) export const activeTarget = initCheck((...args) => instance.activeTarget(...args)) diff --git a/frontend/app/types/session/session.js b/frontend/app/types/session/session.js index 61ca9e489..9fea7db30 100644 --- a/frontend/app/types/session/session.js +++ b/frontend/app/types/session/session.js @@ -75,21 +75,23 @@ export default Record({ crashes: [], socket: null, isIOS: false, - revId: '' + revId: '', }, { fromJS:({ - startTs=0, + startTs=0, + timestamp = 0, backendErrors=0, consoleErrors=0, projectId, errors, stackEvents = [], issues = [], - ...session + sessionId, sessionID, + ...session }) => { const duration = Duration.fromMillis(session.duration < 1000 ? 1000 : session.duration); const durationSeconds = duration.valueOf(); - const startedAt = +startTs; + const startedAt = +startTs || +timestamp; const userDevice = session.userDevice || session.userDeviceType || 'Other'; const userDeviceType = session.userDeviceType || 'other'; @@ -139,6 +141,7 @@ export default Record({ userDisplayName: session.userId || session.userAnonymousId || 'Anonymous User', firstResourceTime, issues: issuesList, + sessionId: sessionId || sessionID, }; }, idKey: "sessionId", diff --git a/frontend/env.js b/frontend/env.js index 85984b376..cb38ae85d 100644 --- a/frontend/env.js +++ b/frontend/env.js @@ -1,6 +1,6 @@ require('dotenv').config() -// TODO: derive version from the tracker package on build +// TODO: (the problem is during the build time the frontend is isolated, as far as I remember) //const trackerInfo = require('../tracker/tracker/package.json'); const oss = { @@ -21,7 +21,7 @@ const oss = { MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, ICE_SERVERS: process.env.ICE_SERVERS, - TRACKER_VERSION: '3.4.17', // trackerInfo.version, + TRACKER_VERSION: '3.5.0' // trackerInfo.version, } module.exports = { diff --git a/frontend/package.json b/frontend/package.json index 5f6c53723..0d946f50c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -56,7 +56,7 @@ "redux-immutable": "^4.0.0", "redux-thunk": "^2.3.0", "semantic-ui-react": "^0.87.3", - "socket.io-client": "^3.0.3", + "socket.io-client": "^3.1.3", "source-map": "^0.7.3", "syncod": "^0.0.1", "tailwindcss": "^1.5.2"