jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'assist-fixes' into dev

Browse source
This commit is contained in:
ShiKhu 2022-05-18 17:55:25 +02:00
commit babe654329
10 changed files with 207 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
frontend/app/components/Assist/components/AssistActions/AssistActions.tsx
View file

@ -82,16 +82,16 @@ function AssistActions({ toggleChatWindow, userId, calling, annotating, peerConn
return (
<div className="flex items-center">
{onCall && (
{(onCall || remoteActive) && (
<>
<div
className={
cn(
'cursor-pointer p-2 flex items-center',
{[stl.disabled]: !onCall}
{[stl.disabled]: cannotCall}
)
}
onClick={ toggleAnnotation }
onClick={ () => toggleAnnotation(!annotating) }
role="button"
>
<IconButton label={`Annotate`} icon={ annotating ? "pencil-stop" : "pencil"} primaryText redText={annotating} />

2
frontend/app/player/MessageDistributor/MessageDistributor.ts
View file

@ -436,7 +436,7 @@ export default class MessageDistributor extends StatedScreen {
}
getFirstMessageTime(): number {
return 0;
return this.pagesManager.minTime;
}
// TODO: clean managers?

17
frontend/app/player/MessageDistributor/managers/AssistManager.ts
View file

@ -251,8 +251,8 @@ export default class AssistManager {
this.socket.emit("click", [ data.x, data.y ]);
}
private toggleRemoteControl(newState: boolean){
if (newState) {
private toggleRemoteControl(enable: boolean){
if (enable) {
this.md.overlay.addEventListener("mousemove", this.onMouseMove)
this.md.overlay.addEventListener("click", this.onMouseClick)
this.md.overlay.addEventListener("wheel", this.onWheel)
@ -262,6 +262,7 @@ export default class AssistManager {
this.md.overlay.removeEventListener("click", this.onMouseClick)
this.md.overlay.removeEventListener("wheel", this.onWheel)
update({ remoteControl: RemoteControlStatus.Disabled })
this.toggleAnnotation(false)
}
}
@ -335,10 +336,9 @@ export default class AssistManager {
private handleCallEnd() {
this.callArgs && this.callArgs.onCallEnd()
this.callConnection && this.callConnection.close()
update({ calling: CallingState.NoCall, annotating: false })
update({ calling: CallingState.NoCall })
this.callArgs = null
this.annot?.remove()
this.annot = null
this.toggleAnnotation(false)
}
private initiateCallEnd = () => {
@ -352,6 +352,7 @@ export default class AssistManager {
this.callConnection && this.callConnection.close()
update({ calling: CallingState.NoCall })
this.callArgs = null
this.toggleAnnotation(false)
} else {
this.handleCallEnd()
}
@ -385,11 +386,11 @@ export default class AssistManager {
}
toggleAnnotation(enable?: boolean) {
if (getState().calling !== CallingState.OnCall) { return }
// if (getState().calling !== CallingState.OnCall) { return }
if (typeof enable !== "boolean") {
enable = !!getState().annotating
}
if (!enable && !this.annot) {
if (enable && !this.annot) {
const annot = this.annot = new AnnotationCanvas()
annot.mount(this.md.overlay)
annot.canvas.addEventListener("mousedown", e => {
@ -416,7 +417,7 @@ export default class AssistManager {
this.socket.emit("moveAnnotation", [ data.x, data.y ])
})
update({ annotating: true })
} else if (enable && !!this.annot) {
} else if (!enable && !!this.annot) {
this.annot.remove()
this.annot = null
update({ annotating: false })

73
frontend/app/player/MessageDistributor/managers/DOMManager.ts
View file

@ -149,11 +149,11 @@ export default class DOMManager extends ListWalker<Message> {
//this.screen.setDisconnected(false);
this.stylesManager.reset();
break;
return
case "create_text_node":
this.nl[ msg.id ] = document.createTextNode('');
this.insertNode(msg);
break;
return
case "create_element_node":
// console.log('elementnode', msg)
if (msg.svg) {
@ -168,20 +168,20 @@ export default class DOMManager extends ListWalker<Message> {
}
this.removeBodyScroll(msg.id);
this.removeAutocomplete(msg);
break;
return
case "move_node":
this.insertNode(msg);
break;
return
case "remove_node":
node = this.nl[ msg.id ]
if (!node) { logger.error("Node not found", msg); break; }
if (!node.parentElement) { logger.error("Parent node not found", msg); break; }
if (!node) { logger.error("Node not found", msg); return }
if (!node.parentElement) { logger.error("Parent node not found", msg); return }
node.parentElement.removeChild(node);
break;
return
case "set_node_attribute":
let { id, name, value } = msg;
node = this.nl[ id ];
if (!node) { logger.error("Node not found", msg); break; }
if (!node) { logger.error("Node not found", msg); return }
if (this.isLink[ id ] && name === "href") {
// @ts-ignore TODO: global ENV type
if (value.startsWith(window.ENV.ASSETS_HOST)) { // Hack for queries in rewrited urls
@ -198,43 +198,54 @@ export default class DOMManager extends ListWalker<Message> {
logger.error(e, msg);
}
this.removeBodyScroll(msg.id);
break;
return
case "remove_node_attribute":
if (!this.nl[ msg.id ]) { logger.error("Node not found", msg); break; }
if (!this.nl[ msg.id ]) { logger.error("Node not found", msg); return }
try {
(this.nl[ msg.id ] as HTMLElement).removeAttribute(msg.name);
} catch(e) {
logger.error(e, msg);
}
break;
return
case "set_input_value":
if (!this.nl[ msg.id ]) { logger.error("Node not found", msg); break; }
const val = msg.mask > 0 ? '*'.repeat(msg.mask) : msg.value;
(this.nl[ msg.id ] as HTMLInputElement).value = val;
break;
node = this.nl[ msg.id ]
if (!node) { logger.error("Node not found", msg); return }
if (!(node instanceof HTMLInputElement || node instanceof HTMLTextAreaElement)) {
logger.error("Trying to set value of non-Input element", msg)
return
}
const val = msg.mask > 0 ? '*'.repeat(msg.mask) : msg.value
doc = this.screen.document
if (doc && node === doc.activeElement) {
// For the case of Remote Control
node.onblur = () => { node.value = val }
return
}
node.value = val
return
case "set_input_checked":
node = this.nl[ msg.id ];
if (!node) { logger.error("Node not found", msg); break; }
if (!node) { logger.error("Node not found", msg); return }
(node as HTMLInputElement).checked = msg.checked;
break;
return
case "set_node_data":
case "set_css_data":
node = this.nl[ msg.id ]
if (!node) { logger.error("Node not found", msg); break; }
if (!node) { logger.error("Node not found", msg); return }
// @ts-ignore
node.data = msg.data;
if (node instanceof HTMLStyleElement) {
doc = this.screen.document
doc && rewriteNodeStyleSheet(doc, node)
}
break;
return
case "css_insert_rule":
node = this.nl[ msg.id ];
if (!node) { logger.error("Node not found", msg); break; }
if (!node) { logger.error("Node not found", msg); return }
if (!(node instanceof HTMLStyleElement) // link or null
|| node.sheet == null) {
logger.warn("Non-style node in CSS rules message (or sheet is null)", msg);
break;
return
}
try {
node.sheet.insertRule(msg.rule, msg.index)
@ -246,21 +257,21 @@ export default class DOMManager extends ListWalker<Message> {
logger.warn("Cannot insert rule.", e, msg)
}
}
break;
return
case "css_delete_rule":
node = this.nl[ msg.id ];
if (!node) { logger.error("Node not found", msg); break; }
if (!node) { logger.error("Node not found", msg); return }
if (!(node instanceof HTMLStyleElement) // link or null
|| node.sheet == null) {
logger.warn("Non-style node in CSS rules message (or sheet is null)", msg);
break;
return
}
try {
node.sheet.deleteRule(msg.index)
} catch (e) {
logger.warn(e, msg)
}
break;
return
case "create_i_frame_document":
node = this.nl[ msg.frameID ];
// console.log('ifr', msg, node)
@ -282,17 +293,7 @@ export default class DOMManager extends ListWalker<Message> {
} else {
logger.warn("Context message host is not Element", msg)
}
break;
//not sure what to do with this one
//case "disconnected":
//setTimeout(() => {
// if last one
//if (this.msgs[ this.msgs.length - 1 ] === msg) {
// this.setDisconnected(true);
// }
//}, 10000);
//break;
return
}
}

129
tracker/tracker-assist/README.md
View file

@ -1,86 +1,131 @@
# OpenReplay Tracker Assist plugin
# OpenReplay Assist
Tracker plugin for WebRTC video support at your site.
OpenReplay Assist Plugin allows you to support your users by seeing their live screen and instantly hopping on call (WebRTC) with them without requiring any 3rd-party screen sharing software.
## Installation
```bash
npm i @openreplay/tracker-assist
```
OR
```bash
yarn add @openreplay/tracker-assist
```
## Usage
Initialize the `@openreplay/tracker` package as usual and load the plugin into it.
### With NPM
Initialize the tracker then load the `@openreplay/tracker-assist` plugin.
#### If your website is a Single Page Application (SPA)
```js
import Tracker from '@openreplay/tracker';
import trackerAssist from '@openreplay/tracker-assist';
const tracker = new Tracker({
projectKey: YOUR_PROJECT_KEY,
projectKey: PROJECT_KEY,
});
tracker.use(trackerAssist(options)); // check the list of available options below
tracker.start();
tracker.use(trackerAssist());
```
Options:
#### If your web app is Server-Side-Rendered (SSR)
```ts
{
confirmText: string,
confirmStyle: Object,
config: RTCConfiguration,
onAgentConnect: () => (()=>void | void),
onCallStart: () => (()=>void | void),
Follow the below example if your app is SSR. Ensure `tracker.start()` is called once the app is started (in `useEffect` or `componentDidMount`).
```js
import OpenReplay from '@openreplay/tracker/cjs';
import trackerFetch from '@openreplay/tracker-assist/cjs';
const tracker = new OpenReplay({
projectKey: PROJECT_KEY
});
tracker.use(trackerAssist(options)); // check the list of available options below
//...
function MyApp() {
useEffect(() => { // use componentDidMount in case of React Class Component
tracker.start();
}, [])
//...
}
```
Use `confirmText` option to specify a text in the call confirmation popup.
You can specify its styles as well with `confirmStyle` style object.
```ts
{
background: "#555"
color: "orange"
}
#### Options
```js
trackerAssist({
callConfirm?: string|ConfirmOptions;
controlConfirm?: string|ConfirmOptions;
config?: object;
onAgentConnect?: () => (()=>void | void);
onCallStart?: () => (()=>void | void);
onRemoteControlStart?: () => (()=>void | void);
})
```
It is possible to pass `config` RTCConfiguration object in order to configure TURN server or other parameters.
```ts
config: {
iceServers: [{
urls: "stun:stun.services.mozilla.com",
username: "louis@mozilla.com",
credential: "webrtcdemo"
}, {
urls: ["stun:stun.example.com", "stun:stun-1.example.com"]
}]
```js
type ConfirmOptions = {
text?:string,
style?: StyleObject, // style object (i.e {color: 'red', borderRadius: '10px'})
confirmBtn?: ButtonOptions,
declineBtn?: ButtonOptions
}
type ButtonOptions = HTMLButtonElement | string | {
innerHTML?: string, // to pass an svg string or text
style?: StyleObject, // style object (i.e {color: 'red', borderRadius: '10px'})
}
```
You can pass `onAgentConnect` callback. It will be called when someone from OpenReplay UI connects to the current live session. It can return another function. In this case, returned callback will be called when the same agent connection gets closed.
```ts
onAgentConnect: () => {
console.log("Hello!")
const onAgentDisconnect = () => console.log("Bye!")
- `callConfirm`: Customize the text and/or layout of the call request popup.
- `controlConfirm`: Customize the text and/or layout of the remote control request popup.
- `config`: Contains any custom ICE/TURN server configuration. Defaults to `{ 'iceServers': [{ 'urls': 'stun:stun.l.google.com:19302' }], 'sdpSemantics': 'unified-plan' }`.
- `onAgentConnect: () => (()=>void | void)`: This callback function is fired when someone from OpenReplay UI connects to the current live session. It can return another function. In this case, returned callback will be called when the same agent connection gets closed.
```js
onAgentConnect = () => {
console.log("Live session started")
const onAgentDisconnect = () => console.log("Live session stopped")
return onAgentDisconnect
}
```
Warning: it is possible for the same agent to be connected/disconnected several times during one session due to a bad network. Several agents may connect simultaneously.
- `onCallStart: () => (()=>void | void)`: This callback function is fired as soon as a call (webRTC) starts. It can also return `onCallEnd` which will be called when the call ends. In case of an unstable connection, this may be called several times. Below is an example:
A callback `onCallStart` will be fired when the end-user accepts the call. It can return another callback that will be called on the call end.
```ts
```js
onCallStart: () => {
console.log("Allo!")
const onCallEnd = () => console.log("short beeps...")
console.log("Call started")
const onCallEnd = () => console.log("Call ended")
return onCallEnd
}
```
- `onRemoteControlStart: () => (()=>void | void)`: This callback function is fired as soon as a remote control session starts. It can also return `onRemoteControlEnd` which will be called when the remote control permissions are revoked. Below is an example:
```js
onCallStart: () => {
console.log("Remote control started")
const onCallEnd = () => console.log("Remote control ended")
return onCallEnd
}
```
## Troubleshooting
### Critical dependency: the request of a dependency is an expression
Please apply this [workaround](https://github.com/peers/peerjs/issues/630#issuecomment-910028230) if you face the below error when compiling:
```log
Failed to compile.
./node_modules/peerjs/dist/peerjs.min.js
Critical dependency: the request of a dependency is an expression
```
If you encounter any other issue, please connect to our [Slack](https://slack.openreplay.com) and get help from our community.

2
tracker/tracker-assist/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@openreplay/tracker-assist",
"description": "Tracker plugin for screen assistance through the WebRTC",
"version": "3.5.9",
"version": "3.5.10",
"keywords": [
"WebRTC",
"assistance",

14
tracker/tracker-assist/src/Assist.ts
View file

@ -8,8 +8,9 @@ import RequestLocalStream from './LocalStream.js';
import RemoteControl from './RemoteControl.js';
import CallWindow from './CallWindow.js';
import AnnotationCanvas from './AnnotationCanvas.js';
import ConfirmWindow, { callConfirmDefault, controlConfirmDefault } from './ConfirmWindow.js';
import type { Options as ConfirmOptions } from './ConfirmWindow.js';
import ConfirmWindow from './ConfirmWindow/ConfirmWindow.js';
import { callConfirmDefault } from './ConfirmWindow/defaults.js';
import type { Options as ConfirmOptions } from './ConfirmWindow/defaults.js';
// TODO: fully specified strict check (everywhere)
@ -145,17 +146,24 @@ export default class Assist {
socket.onAny((...args) => app.debug.log("Socket:", ...args))
const remoteControl = new RemoteControl(
this.options,
id => {
this.agents[id].onControlReleased = this.options.onRemoteControlStart()
this.emit("control_granted", id)
annot = new AnnotationCanvas()
annot.mount()
},
id => {
const cb = this.agents[id].onControlReleased
delete this.agents[id].onControlReleased
typeof cb === "function" && cb()
this.emit("control_rejected", id)
if (annot != null) {
annot.remove()
annot = null
}
},
)
@ -231,7 +239,7 @@ export default class Assist {
peerOptions['config'] = this.options.config
}
const peer = this.peer = new Peer(peerID, peerOptions);
app.debug.log('Peer created: ', peer)
// app.debug.log('Peer created: ', peer)
peer.on('error', e => app.debug.warn("Peer error: ", e.type, e))
peer.on('disconnect', () => peer.reconnect())
peer.on('call', (call) => {

68
tracker/tracker-assist/src/ConfirmWindow.ts → tracker/tracker-assist/src/ConfirmWindow/ConfirmWindow.ts
View file

@ -1,12 +1,6 @@
import type { Properties } from 'csstype';
import { declineCall, acceptCall, cross, remoteControl } from './icons.js'
const TEXT_GRANT_REMORTE_ACCESS = "Grant Remote Access";
const TEXT_REJECT = "Reject";
const TEXT_ANSWER_CALL = `${acceptCall} &#xa0 Answer`;
type ButtonOptions =
export type ButtonOptions =
| HTMLButtonElement
| string
| {
@ -15,48 +9,15 @@ type ButtonOptions =
};
// TODO: common strategy for InputOptions/defaultOptions merging
interface ConfirmWindowOptions {
export interface ConfirmWindowOptions {
text: string;
style?: Properties;
confirmBtn: ButtonOptions;
declineBtn: ButtonOptions;
}
export type Options = string | Partial<ConfirmWindowOptions>;
function confirmDefault(
opts: Options,
confirmBtn: ButtonOptions,
declineBtn: ButtonOptions,
text: string
): ConfirmWindowOptions {
const isStr = typeof opts === "string";
return Object.assign(
{
text: isStr ? opts : text,
confirmBtn,
declineBtn
},
isStr ? undefined : opts
);
}
export const callConfirmDefault = (opts: Options) =>
confirmDefault(
opts,
TEXT_ANSWER_CALL,
TEXT_REJECT,
"You have an incoming call. Do you want to answer?"
);
export const controlConfirmDefault = (opts: Options) =>
confirmDefault(
opts,
TEXT_GRANT_REMORTE_ACCESS,
TEXT_REJECT,
"Allow remote control?"
);
function makeButton(options: ButtonOptions): HTMLButtonElement {
function makeButton(options: ButtonOptions, defaultStyle?: Properties): HTMLButtonElement {
if (options instanceof HTMLButtonElement) {
return options;
}
@ -71,7 +32,7 @@ function makeButton(options: ButtonOptions): HTMLButtonElement {
alignItems: "center",
textTransform: "uppercase",
marginRight: "10px"
});
}, defaultStyle);
if (typeof options === "string") {
btn.innerHTML = options;
} else {
@ -90,22 +51,19 @@ export default class ConfirmWindow {
const p = document.createElement("p");
p.innerText = options.text;
const buttons = document.createElement("div");
const confirmBtn = makeButton(options.confirmBtn);
const declineBtn = makeButton(options.declineBtn);
buttons.appendChild(confirmBtn);
buttons.appendChild(declineBtn);
popup.appendChild(p);
popup.appendChild(buttons);
Object.assign(confirmBtn.style, {
const confirmBtn = makeButton(options.confirmBtn, {
background: "rgba(0, 167, 47, 1)",
color: "white"
});
Object.assign(declineBtn.style, {
})
const declineBtn = makeButton(options.declineBtn, {
background: "#FFE9E9",
color: "#CC0000"
});
})
buttons.appendChild(confirmBtn)
buttons.appendChild(declineBtn)
popup.appendChild(p)
popup.appendChild(buttons)
Object.assign(buttons.style, {
marginTop: "10px",

42
tracker/tracker-assist/src/ConfirmWindow/defaults.ts Normal file
View file

@ -0,0 +1,42 @@
import { declineCall, acceptCall, cross, remoteControl } from '../icons.js'
import type { ButtonOptions, ConfirmWindowOptions } from './ConfirmWindow.js'
const TEXT_GRANT_REMORTE_ACCESS = "Grant Remote Control";
const TEXT_REJECT = "Reject";
const TEXT_ANSWER_CALL = `${acceptCall} &#xa0 Answer`;
export type Options = string | Partial<ConfirmWindowOptions>;
function confirmDefault(
opts: Options,
confirmBtn: ButtonOptions,
declineBtn: ButtonOptions,
text: string
): ConfirmWindowOptions {
const isStr = typeof opts === "string";
return Object.assign(
{
text: isStr ? opts : text,
confirmBtn,
declineBtn
},
isStr ? undefined : opts
);
}
export const callConfirmDefault = (opts: Options) =>
confirmDefault(
opts,
TEXT_ANSWER_CALL,
TEXT_REJECT,
"You have an incoming call. Do you want to answer?"
)
export const controlConfirmDefault = (opts: Options) =>
confirmDefault(
opts,
TEXT_GRANT_REMORTE_ACCESS,
TEXT_REJECT,
"Agent requested remote control. Allow?"
)

5
tracker/tracker-assist/src/RemoteControl.ts
View file

@ -1,6 +1,7 @@
import Mouse from './Mouse.js';
import ConfirmWindow, { controlConfirmDefault } from './ConfirmWindow.js';
import type { Options as AssistOptions } from './Assist'
import ConfirmWindow from './ConfirmWindow/ConfirmWindow.js';
import { controlConfirmDefault } from './ConfirmWindow/defaults.js';
import type { Options as AssistOptions } from './Assist';
enum RCStatus {
Disabled,