diff --git a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx index 88713a6ae..2ea90b143 100644 --- a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx +++ b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx @@ -82,16 +82,16 @@ function AssistActions({ toggleChatWindow, userId, calling, annotating, peerConn return (
- {onCall && ( + {(onCall || remoteActive) && ( <>
toggleAnnotation(!annotating) } role="button" > diff --git a/frontend/app/player/MessageDistributor/MessageDistributor.ts b/frontend/app/player/MessageDistributor/MessageDistributor.ts index d60ae1790..cdcb1cd8b 100644 --- a/frontend/app/player/MessageDistributor/MessageDistributor.ts +++ b/frontend/app/player/MessageDistributor/MessageDistributor.ts @@ -436,7 +436,7 @@ export default class MessageDistributor extends StatedScreen { } getFirstMessageTime(): number { - return 0; + return this.pagesManager.minTime; } // TODO: clean managers? diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager.ts b/frontend/app/player/MessageDistributor/managers/AssistManager.ts index dc48b03f4..04fcc40d0 100644 --- a/frontend/app/player/MessageDistributor/managers/AssistManager.ts +++ b/frontend/app/player/MessageDistributor/managers/AssistManager.ts @@ -251,8 +251,8 @@ export default class AssistManager { this.socket.emit("click", [ data.x, data.y ]); } - private toggleRemoteControl(newState: boolean){ - if (newState) { + private toggleRemoteControl(enable: boolean){ + if (enable) { this.md.overlay.addEventListener("mousemove", this.onMouseMove) this.md.overlay.addEventListener("click", this.onMouseClick) this.md.overlay.addEventListener("wheel", this.onWheel) @@ -262,6 +262,7 @@ export default class AssistManager { this.md.overlay.removeEventListener("click", this.onMouseClick) this.md.overlay.removeEventListener("wheel", this.onWheel) update({ remoteControl: RemoteControlStatus.Disabled }) + this.toggleAnnotation(false) } } @@ -335,10 +336,9 @@ export default class AssistManager { private handleCallEnd() { this.callArgs && this.callArgs.onCallEnd() this.callConnection && this.callConnection.close() - update({ calling: CallingState.NoCall, annotating: false }) + update({ calling: CallingState.NoCall }) this.callArgs = null - this.annot?.remove() - this.annot = null + this.toggleAnnotation(false) } private initiateCallEnd = () => { @@ -352,6 +352,7 @@ export default class AssistManager { this.callConnection && this.callConnection.close() update({ calling: CallingState.NoCall }) this.callArgs = null + this.toggleAnnotation(false) } else { this.handleCallEnd() } @@ -385,11 +386,11 @@ export default class AssistManager { } toggleAnnotation(enable?: boolean) { - if (getState().calling !== CallingState.OnCall) { return } + // if (getState().calling !== CallingState.OnCall) { return } if (typeof enable !== "boolean") { enable = !!getState().annotating } - if (!enable && !this.annot) { + if (enable && !this.annot) { const annot = this.annot = new AnnotationCanvas() annot.mount(this.md.overlay) annot.canvas.addEventListener("mousedown", e => { @@ -416,7 +417,7 @@ export default class AssistManager { this.socket.emit("moveAnnotation", [ data.x, data.y ]) }) update({ annotating: true }) - } else if (enable && !!this.annot) { + } else if (!enable && !!this.annot) { this.annot.remove() this.annot = null update({ annotating: false }) diff --git a/frontend/app/player/MessageDistributor/managers/DOMManager.ts b/frontend/app/player/MessageDistributor/managers/DOMManager.ts index 187ebc8a0..f2c48ab5c 100644 --- a/frontend/app/player/MessageDistributor/managers/DOMManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOMManager.ts @@ -149,11 +149,11 @@ export default class DOMManager extends ListWalker { //this.screen.setDisconnected(false); this.stylesManager.reset(); - break; + return case "create_text_node": this.nl[ msg.id ] = document.createTextNode(''); this.insertNode(msg); - break; + return case "create_element_node": // console.log('elementnode', msg) if (msg.svg) { @@ -168,20 +168,20 @@ export default class DOMManager extends ListWalker { } this.removeBodyScroll(msg.id); this.removeAutocomplete(msg); - break; + return case "move_node": this.insertNode(msg); - break; + return case "remove_node": node = this.nl[ msg.id ] - if (!node) { logger.error("Node not found", msg); break; } - if (!node.parentElement) { logger.error("Parent node not found", msg); break; } + if (!node) { logger.error("Node not found", msg); return } + if (!node.parentElement) { logger.error("Parent node not found", msg); return } node.parentElement.removeChild(node); - break; + return case "set_node_attribute": let { id, name, value } = msg; node = this.nl[ id ]; - if (!node) { logger.error("Node not found", msg); break; } + if (!node) { logger.error("Node not found", msg); return } if (this.isLink[ id ] && name === "href") { // @ts-ignore TODO: global ENV type if (value.startsWith(window.ENV.ASSETS_HOST)) { // Hack for queries in rewrited urls @@ -198,43 +198,54 @@ export default class DOMManager extends ListWalker { logger.error(e, msg); } this.removeBodyScroll(msg.id); - break; + return case "remove_node_attribute": - if (!this.nl[ msg.id ]) { logger.error("Node not found", msg); break; } + if (!this.nl[ msg.id ]) { logger.error("Node not found", msg); return } try { (this.nl[ msg.id ] as HTMLElement).removeAttribute(msg.name); } catch(e) { logger.error(e, msg); } - break; + return case "set_input_value": - if (!this.nl[ msg.id ]) { logger.error("Node not found", msg); break; } - const val = msg.mask > 0 ? '*'.repeat(msg.mask) : msg.value; - (this.nl[ msg.id ] as HTMLInputElement).value = val; - break; + node = this.nl[ msg.id ] + if (!node) { logger.error("Node not found", msg); return } + if (!(node instanceof HTMLInputElement || node instanceof HTMLTextAreaElement)) { + logger.error("Trying to set value of non-Input element", msg) + return + } + const val = msg.mask > 0 ? '*'.repeat(msg.mask) : msg.value + doc = this.screen.document + if (doc && node === doc.activeElement) { + // For the case of Remote Control + node.onblur = () => { node.value = val } + return + } + node.value = val + return case "set_input_checked": node = this.nl[ msg.id ]; - if (!node) { logger.error("Node not found", msg); break; } + if (!node) { logger.error("Node not found", msg); return } (node as HTMLInputElement).checked = msg.checked; - break; + return case "set_node_data": case "set_css_data": node = this.nl[ msg.id ] - if (!node) { logger.error("Node not found", msg); break; } + if (!node) { logger.error("Node not found", msg); return } // @ts-ignore node.data = msg.data; if (node instanceof HTMLStyleElement) { doc = this.screen.document doc && rewriteNodeStyleSheet(doc, node) } - break; + return case "css_insert_rule": node = this.nl[ msg.id ]; - if (!node) { logger.error("Node not found", msg); break; } + if (!node) { logger.error("Node not found", msg); return } if (!(node instanceof HTMLStyleElement) // link or null || node.sheet == null) { logger.warn("Non-style node in CSS rules message (or sheet is null)", msg); - break; + return } try { node.sheet.insertRule(msg.rule, msg.index) @@ -246,21 +257,21 @@ export default class DOMManager extends ListWalker { logger.warn("Cannot insert rule.", e, msg) } } - break; + return case "css_delete_rule": node = this.nl[ msg.id ]; - if (!node) { logger.error("Node not found", msg); break; } + if (!node) { logger.error("Node not found", msg); return } if (!(node instanceof HTMLStyleElement) // link or null || node.sheet == null) { logger.warn("Non-style node in CSS rules message (or sheet is null)", msg); - break; + return } try { node.sheet.deleteRule(msg.index) } catch (e) { logger.warn(e, msg) } - break; + return case "create_i_frame_document": node = this.nl[ msg.frameID ]; // console.log('ifr', msg, node) @@ -282,17 +293,7 @@ export default class DOMManager extends ListWalker { } else { logger.warn("Context message host is not Element", msg) } - - break; - //not sure what to do with this one - //case "disconnected": - //setTimeout(() => { - // if last one - //if (this.msgs[ this.msgs.length - 1 ] === msg) { - // this.setDisconnected(true); - // } - //}, 10000); - //break; + return } } diff --git a/tracker/tracker-assist/README.md b/tracker/tracker-assist/README.md index 0c7bfe00f..4897477b3 100644 --- a/tracker/tracker-assist/README.md +++ b/tracker/tracker-assist/README.md @@ -1,86 +1,131 @@ -# OpenReplay Tracker Assist plugin +# OpenReplay Assist -Tracker plugin for WebRTC video support at your site. +OpenReplay Assist Plugin allows you to support your users by seeing their live screen and instantly hopping on call (WebRTC) with them without requiring any 3rd-party screen sharing software. ## Installation ```bash npm i @openreplay/tracker-assist ``` +OR +```bash +yarn add @openreplay/tracker-assist +``` ## Usage -Initialize the `@openreplay/tracker` package as usual and load the plugin into it. +### With NPM + +Initialize the tracker then load the `@openreplay/tracker-assist` plugin. + +#### If your website is a Single Page Application (SPA) ```js import Tracker from '@openreplay/tracker'; import trackerAssist from '@openreplay/tracker-assist'; const tracker = new Tracker({ - projectKey: YOUR_PROJECT_KEY, + projectKey: PROJECT_KEY, }); +tracker.use(trackerAssist(options)); // check the list of available options below + tracker.start(); -tracker.use(trackerAssist()); ``` -Options: +#### If your web app is Server-Side-Rendered (SSR) -```ts -{ - confirmText: string, - confirmStyle: Object, - config: RTCConfiguration, - onAgentConnect: () => (()=>void | void), - onCallStart: () => (()=>void | void), +Follow the below example if your app is SSR. Ensure `tracker.start()` is called once the app is started (in `useEffect` or `componentDidMount`). + +```js +import OpenReplay from '@openreplay/tracker/cjs'; +import trackerFetch from '@openreplay/tracker-assist/cjs'; + +const tracker = new OpenReplay({ + projectKey: PROJECT_KEY +}); +tracker.use(trackerAssist(options)); // check the list of available options below + +//... +function MyApp() { + useEffect(() => { // use componentDidMount in case of React Class Component + tracker.start(); + }, []) +//... } ``` -Use `confirmText` option to specify a text in the call confirmation popup. -You can specify its styles as well with `confirmStyle` style object. -```ts -{ - background: "#555" - color: "orange" -} +#### Options +```js +trackerAssist({ + callConfirm?: string|ConfirmOptions; + controlConfirm?: string|ConfirmOptions; + config?: object; + onAgentConnect?: () => (()=>void | void); + onCallStart?: () => (()=>void | void); + onRemoteControlStart?: () => (()=>void | void); +}) ``` -It is possible to pass `config` RTCConfiguration object in order to configure TURN server or other parameters. -```ts -config: { - iceServers: [{ - urls: "stun:stun.services.mozilla.com", - username: "louis@mozilla.com", - credential: "webrtcdemo" - }, { - urls: ["stun:stun.example.com", "stun:stun-1.example.com"] - }] +```js +type ConfirmOptions = { + text?:string, + style?: StyleObject, // style object (i.e {color: 'red', borderRadius: '10px'}) + confirmBtn?: ButtonOptions, + declineBtn?: ButtonOptions } +type ButtonOptions = HTMLButtonElement | string | { + innerHTML?: string, // to pass an svg string or text + style?: StyleObject, // style object (i.e {color: 'red', borderRadius: '10px'}) +} ``` -You can pass `onAgentConnect` callback. It will be called when someone from OpenReplay UI connects to the current live session. It can return another function. In this case, returned callback will be called when the same agent connection gets closed. -```ts -onAgentConnect: () => { - console.log("Hello!") - const onAgentDisconnect = () => console.log("Bye!") +- `callConfirm`: Customize the text and/or layout of the call request popup. +- `controlConfirm`: Customize the text and/or layout of the remote control request popup. +- `config`: Contains any custom ICE/TURN server configuration. Defaults to `{ 'iceServers': [{ 'urls': 'stun:stun.l.google.com:19302' }], 'sdpSemantics': 'unified-plan' }`. +- `onAgentConnect: () => (()=>void | void)`: This callback function is fired when someone from OpenReplay UI connects to the current live session. It can return another function. In this case, returned callback will be called when the same agent connection gets closed. + +```js +onAgentConnect = () => { + console.log("Live session started") + const onAgentDisconnect = () => console.log("Live session stopped") return onAgentDisconnect } - ``` -Warning: it is possible for the same agent to be connected/disconnected several times during one session due to a bad network. Several agents may connect simultaneously. +- `onCallStart: () => (()=>void | void)`: This callback function is fired as soon as a call (webRTC) starts. It can also return `onCallEnd` which will be called when the call ends. In case of an unstable connection, this may be called several times. Below is an example: -A callback `onCallStart` will be fired when the end-user accepts the call. It can return another callback that will be called on the call end. -```ts +```js onCallStart: () => { - console.log("Allo!") - const onCallEnd = () => console.log("short beeps...") + console.log("Call started") + const onCallEnd = () => console.log("Call ended") return onCallEnd } - ``` +- `onRemoteControlStart: () => (()=>void | void)`: This callback function is fired as soon as a remote control session starts. It can also return `onRemoteControlEnd` which will be called when the remote control permissions are revoked. Below is an example: +```js +onCallStart: () => { + console.log("Remote control started") + const onCallEnd = () => console.log("Remote control ended") + return onCallEnd +} +``` +## Troubleshooting + +### Critical dependency: the request of a dependency is an expression + +Please apply this [workaround](https://github.com/peers/peerjs/issues/630#issuecomment-910028230) if you face the below error when compiling: + +```log +Failed to compile. + +./node_modules/peerjs/dist/peerjs.min.js +Critical dependency: the request of a dependency is an expression +``` + +If you encounter any other issue, please connect to our [Slack](https://slack.openreplay.com) and get help from our community. diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 437a86499..ad080d92f 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "3.5.9", + "version": "3.5.10", "keywords": [ "WebRTC", "assistance", diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index ab64ab9b2..c5bd43ea8 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -8,8 +8,9 @@ import RequestLocalStream from './LocalStream.js'; import RemoteControl from './RemoteControl.js'; import CallWindow from './CallWindow.js'; import AnnotationCanvas from './AnnotationCanvas.js'; -import ConfirmWindow, { callConfirmDefault, controlConfirmDefault } from './ConfirmWindow.js'; -import type { Options as ConfirmOptions } from './ConfirmWindow.js'; +import ConfirmWindow from './ConfirmWindow/ConfirmWindow.js'; +import { callConfirmDefault } from './ConfirmWindow/defaults.js'; +import type { Options as ConfirmOptions } from './ConfirmWindow/defaults.js'; // TODO: fully specified strict check (everywhere) @@ -145,17 +146,24 @@ export default class Assist { socket.onAny((...args) => app.debug.log("Socket:", ...args)) + const remoteControl = new RemoteControl( this.options, id => { this.agents[id].onControlReleased = this.options.onRemoteControlStart() this.emit("control_granted", id) + annot = new AnnotationCanvas() + annot.mount() }, id => { const cb = this.agents[id].onControlReleased delete this.agents[id].onControlReleased typeof cb === "function" && cb() this.emit("control_rejected", id) + if (annot != null) { + annot.remove() + annot = null + } }, ) @@ -231,7 +239,7 @@ export default class Assist { peerOptions['config'] = this.options.config } const peer = this.peer = new Peer(peerID, peerOptions); - app.debug.log('Peer created: ', peer) + // app.debug.log('Peer created: ', peer) peer.on('error', e => app.debug.warn("Peer error: ", e.type, e)) peer.on('disconnect', () => peer.reconnect()) peer.on('call', (call) => { diff --git a/tracker/tracker-assist/src/ConfirmWindow.ts b/tracker/tracker-assist/src/ConfirmWindow/ConfirmWindow.ts similarity index 68% rename from tracker/tracker-assist/src/ConfirmWindow.ts rename to tracker/tracker-assist/src/ConfirmWindow/ConfirmWindow.ts index 6cfabbca9..02d9dd9c6 100644 --- a/tracker/tracker-assist/src/ConfirmWindow.ts +++ b/tracker/tracker-assist/src/ConfirmWindow/ConfirmWindow.ts @@ -1,12 +1,6 @@ import type { Properties } from 'csstype'; -import { declineCall, acceptCall, cross, remoteControl } from './icons.js' - -const TEXT_GRANT_REMORTE_ACCESS = "Grant Remote Access"; -const TEXT_REJECT = "Reject"; -const TEXT_ANSWER_CALL = `${acceptCall}   Answer`; - -type ButtonOptions = +export type ButtonOptions = | HTMLButtonElement | string | { @@ -15,48 +9,15 @@ type ButtonOptions = }; // TODO: common strategy for InputOptions/defaultOptions merging -interface ConfirmWindowOptions { +export interface ConfirmWindowOptions { text: string; style?: Properties; confirmBtn: ButtonOptions; declineBtn: ButtonOptions; } -export type Options = string | Partial; -function confirmDefault( - opts: Options, - confirmBtn: ButtonOptions, - declineBtn: ButtonOptions, - text: string -): ConfirmWindowOptions { - const isStr = typeof opts === "string"; - return Object.assign( - { - text: isStr ? opts : text, - confirmBtn, - declineBtn - }, - isStr ? undefined : opts - ); -} - -export const callConfirmDefault = (opts: Options) => - confirmDefault( - opts, - TEXT_ANSWER_CALL, - TEXT_REJECT, - "You have an incoming call. Do you want to answer?" - ); -export const controlConfirmDefault = (opts: Options) => - confirmDefault( - opts, - TEXT_GRANT_REMORTE_ACCESS, - TEXT_REJECT, - "Allow remote control?" - ); - -function makeButton(options: ButtonOptions): HTMLButtonElement { +function makeButton(options: ButtonOptions, defaultStyle?: Properties): HTMLButtonElement { if (options instanceof HTMLButtonElement) { return options; } @@ -71,7 +32,7 @@ function makeButton(options: ButtonOptions): HTMLButtonElement { alignItems: "center", textTransform: "uppercase", marginRight: "10px" - }); + }, defaultStyle); if (typeof options === "string") { btn.innerHTML = options; } else { @@ -90,22 +51,19 @@ export default class ConfirmWindow { const p = document.createElement("p"); p.innerText = options.text; const buttons = document.createElement("div"); - const confirmBtn = makeButton(options.confirmBtn); - const declineBtn = makeButton(options.declineBtn); - buttons.appendChild(confirmBtn); - buttons.appendChild(declineBtn); - popup.appendChild(p); - popup.appendChild(buttons); - - Object.assign(confirmBtn.style, { + const confirmBtn = makeButton(options.confirmBtn, { background: "rgba(0, 167, 47, 1)", color: "white" - }); - - Object.assign(declineBtn.style, { + }) + const declineBtn = makeButton(options.declineBtn, { background: "#FFE9E9", color: "#CC0000" - }); + }) + buttons.appendChild(confirmBtn) + buttons.appendChild(declineBtn) + popup.appendChild(p) + popup.appendChild(buttons) + Object.assign(buttons.style, { marginTop: "10px", diff --git a/tracker/tracker-assist/src/ConfirmWindow/defaults.ts b/tracker/tracker-assist/src/ConfirmWindow/defaults.ts new file mode 100644 index 000000000..8f84cbe89 --- /dev/null +++ b/tracker/tracker-assist/src/ConfirmWindow/defaults.ts @@ -0,0 +1,42 @@ +import { declineCall, acceptCall, cross, remoteControl } from '../icons.js' +import type { ButtonOptions, ConfirmWindowOptions } from './ConfirmWindow.js' + + +const TEXT_GRANT_REMORTE_ACCESS = "Grant Remote Control"; +const TEXT_REJECT = "Reject"; +const TEXT_ANSWER_CALL = `${acceptCall}   Answer`; + +export type Options = string | Partial; + +function confirmDefault( + opts: Options, + confirmBtn: ButtonOptions, + declineBtn: ButtonOptions, + text: string +): ConfirmWindowOptions { + const isStr = typeof opts === "string"; + return Object.assign( + { + text: isStr ? opts : text, + confirmBtn, + declineBtn + }, + isStr ? undefined : opts + ); +} + +export const callConfirmDefault = (opts: Options) => + confirmDefault( + opts, + TEXT_ANSWER_CALL, + TEXT_REJECT, + "You have an incoming call. Do you want to answer?" + ) + +export const controlConfirmDefault = (opts: Options) => + confirmDefault( + opts, + TEXT_GRANT_REMORTE_ACCESS, + TEXT_REJECT, + "Agent requested remote control. Allow?" + ) diff --git a/tracker/tracker-assist/src/RemoteControl.ts b/tracker/tracker-assist/src/RemoteControl.ts index 02137d99f..d2903f9c4 100644 --- a/tracker/tracker-assist/src/RemoteControl.ts +++ b/tracker/tracker-assist/src/RemoteControl.ts @@ -1,6 +1,7 @@ import Mouse from './Mouse.js'; -import ConfirmWindow, { controlConfirmDefault } from './ConfirmWindow.js'; -import type { Options as AssistOptions } from './Assist' +import ConfirmWindow from './ConfirmWindow/ConfirmWindow.js'; +import { controlConfirmDefault } from './ConfirmWindow/defaults.js'; +import type { Options as AssistOptions } from './Assist'; enum RCStatus { Disabled,