jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(ui): fix remote control event

Browse source
This commit is contained in:
nick-delirium 2023-10-02 14:01:23 +02:00
parent cf20ac2b04
commit 85cc45889b
3 changed files with 173 additions and 138 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
frontend/app/components/Assist/components/AssistActions/AssistActions.tsx
View file

@ -178,17 +178,20 @@ function AssistActions({
}; };
const requestControl = () => { const requestControl = () => {
if (remoteActive) player.assistManager.ping(AssistActionsPing.control.end, agentId) const onStart = () => {
setRemoteControlCallbacks({ onReject: onControlReject }); player.assistManager.ping(AssistActionsPing.control.start, agentId)
if (callRequesting || remoteRequesting) return; }
const onEnd = () => {
player.assistManager.ping(AssistActionsPing.control.end, agentId)
}
setRemoteControlCallbacks({
onReject: onControlReject,
onStart: onStart,
onEnd: onEnd,
});
requestReleaseRemoteControl(); requestReleaseRemoteControl();
}; };
React.useEffect(() => {
if (remoteActive) {
player.assistManager.ping(AssistActionsPing.control.start, agentId)
}
}, [remoteActive])
React.useEffect(() => { React.useEffect(() => {
if (onCall) { if (onCall) {
player.assistManager.ping(AssistActionsPing.call.start, agentId) player.assistManager.ping(AssistActionsPing.call.start, agentId)

290
frontend/app/player/web/assist/RemoteControl.ts
View file

@ -1,7 +1,7 @@
import AnnotationCanvas from './AnnotationCanvas'; import AnnotationCanvas from './AnnotationCanvas';
import type { Socket } from './types' import type { Socket } from './types';
import type Screen from '../Screen/Screen' import type Screen from '../Screen/Screen';
import type { Store } from '../../common/types' import type { Store } from '../../common/types';
export enum RemoteControlStatus { export enum RemoteControlStatus {
Disabled = 0, Disabled = 0,
@ -10,180 +10,212 @@ export enum RemoteControlStatus {
} }
export interface State { export interface State {
annotating: boolean annotating: boolean;
remoteControl: RemoteControlStatus remoteControl: RemoteControlStatus;
currentTab?: string currentTab?: string;
} }
export default class RemoteControl { export default class RemoteControl {
private assistVersion = 1 private assistVersion = 1;
static readonly INITIAL_STATE: Readonly<State> = { static readonly INITIAL_STATE: Readonly<State> = {
remoteControl: RemoteControlStatus.Disabled, remoteControl: RemoteControlStatus.Disabled,
annotating: false, annotating: false,
} };
onReject: () => void = () => {} onReject: () => void = () => {};
onStart: () => void = () => {};
onEnd: () => void = () => {};
constructor( constructor(
private store: Store<State>, private store: Store<State>,
private socket: Socket, private socket: Socket,
private screen: Screen, private screen: Screen,
private agentInfo: Object, private agentInfo: Object,
private onToggle: (active: boolean) => void, private onToggle: (active: boolean) => void,
private getAssistVersion: () => number private getAssistVersion: () => number
){ ) {
socket.on("control_granted", ({ meta, data }) => { socket.on('control_granted', ({ meta, data }) => {
this.toggleRemoteControl(data === socket.id) if (data === socket.id) {
}) this.toggleRemoteControl(data === socket.id);
socket.on("control_rejected", ({ meta, data }) => { this.onStart();
data === socket.id && this.toggleRemoteControl(false)
this.onReject()
})
socket.on('SESSION_DISCONNECTED', () => {
if (this.store.get().remoteControl === RemoteControlStatus.Requesting) {
this.toggleRemoteControl(false) // else its remaining
} }
}) });
socket.on("disconnect", () => { socket.on('control_rejected', ({ meta, data }) => {
this.toggleRemoteControl(false) if (data === socket.id) {
}) this.toggleRemoteControl(false);
socket.on("error", () => { this.onEnd();
this.toggleRemoteControl(false) }
}) this.onReject();
this.assistVersion = getAssistVersion() if (this.store.get().remoteControl === RemoteControlStatus.Requesting) {
} return this.store.update({ remoteControl: RemoteControlStatus.Disabled });
}
private onMouseMove = (e: MouseEvent): void => { });
const data = this.screen.getInternalCoordinates(e) socket.on('SESSION_DISCONNECTED', () => {
this.emitData("move", [ data.x, data.y ]) if (this.store.get().remoteControl === RemoteControlStatus.Requesting) {
this.toggleRemoteControl(false); // else its remaining
}
});
socket.on('disconnect', () => {
this.toggleRemoteControl(false);
this.onEnd();
});
socket.on('error', () => {
this.toggleRemoteControl(false);
this.onEnd();
});
this.assistVersion = getAssistVersion();
} }
private onMouseMove = (e: MouseEvent): void => {
const data = this.screen.getInternalCoordinates(e);
this.emitData('move', [data.x, data.y]);
};
private emitData = (event: string, data?: any) => { private emitData = (event: string, data?: any) => {
if (this.getAssistVersion() === 1) { if (this.getAssistVersion() === 1) {
this.socket.emit(event, data) this.socket.emit(event, data);
} else { } else {
this.socket.emit(event, { meta: { tabId: this.store.get().currentTab }, data }) this.socket.emit(event, { meta: { tabId: this.store.get().currentTab }, data });
} }
} };
private onWheel = (e: WheelEvent): void => { private onWheel = (e: WheelEvent): void => {
e.preventDefault() e.preventDefault();
//throttling makes movements less smooth, so it is omitted //throttling makes movements less smooth, so it is omitted
//this.onMouseMove(e) //this.onMouseMove(e)
this.emitData("scroll", [ e.deltaX, e.deltaY ]) this.emitData('scroll', [e.deltaX, e.deltaY]);
} };
public setCallbacks = ({ onReject }: { onReject: () => void }) => { public setCallbacks = ({
this.onReject = onReject onReject,
} onStart,
onEnd,
}: {
onReject: () => void;
onStart: () => void;
onEnd: () => void;
}) => {
this.onReject = onReject;
this.onStart = onStart;
this.onEnd = onEnd;
};
private onMouseClick = (e: MouseEvent): void => { private onMouseClick = (e: MouseEvent): void => {
if (this.store.get().annotating) { return; } // ignore clicks while annotating if (this.store.get().annotating) {
return;
} // ignore clicks while annotating
const data = this.screen.getInternalViewportCoordinates(e) const data = this.screen.getInternalViewportCoordinates(e);
// const el = this.screen.getElementFromPoint(e); // requires requestiong node_id from domManager // const el = this.screen.getElementFromPoint(e); // requires requestiong node_id from domManager
const el = this.screen.getElementFromInternalPoint(data) const el = this.screen.getElementFromInternalPoint(data);
if (el instanceof HTMLElement) { if (el instanceof HTMLElement) {
el.focus() el.focus();
el.oninput = e => { el.oninput = (e) => {
if (el instanceof HTMLTextAreaElement if (el instanceof HTMLTextAreaElement || el instanceof HTMLInputElement) {
|| el instanceof HTMLInputElement this.socket && this.emitData('input', el.value);
) {
this.socket && this.emitData("input", el.value)
} else if (el.isContentEditable) { } else if (el.isContentEditable) {
this.socket && this.emitData("input", el.innerText) this.socket && this.emitData('input', el.innerText);
} }
} };
// TODO: send "focus" event to assist with the nodeID // TODO: send "focus" event to assist with the nodeID
el.onkeydown = e => { el.onkeydown = (e) => {
if (e.key == "Tab") { if (e.key == 'Tab') {
e.preventDefault() e.preventDefault();
} }
} };
el.onblur = () => { el.onblur = () => {
el.oninput = null el.oninput = null;
el.onblur = null el.onblur = null;
} };
} }
this.emitData("click", [ data.x, data.y ]); this.emitData('click', [data.x, data.y]);
} };
private toggleRemoteControl(enable: boolean){ private toggleRemoteControl(enable: boolean) {
if (enable) { if (enable) {
this.screen.overlay.addEventListener("mousemove", this.onMouseMove) this.screen.overlay.addEventListener('mousemove', this.onMouseMove);
this.screen.overlay.addEventListener("click", this.onMouseClick) this.screen.overlay.addEventListener('click', this.onMouseClick);
this.screen.overlay.addEventListener("wheel", this.onWheel) this.screen.overlay.addEventListener('wheel', this.onWheel);
this.store.update({ remoteControl: RemoteControlStatus.Enabled }) this.store.update({ remoteControl: RemoteControlStatus.Enabled });
} else { } else {
this.screen.overlay.removeEventListener("mousemove", this.onMouseMove) this.screen.overlay.removeEventListener('mousemove', this.onMouseMove);
this.screen.overlay.removeEventListener("click", this.onMouseClick) this.screen.overlay.removeEventListener('click', this.onMouseClick);
this.screen.overlay.removeEventListener("wheel", this.onWheel) this.screen.overlay.removeEventListener('wheel', this.onWheel);
this.store.update({ remoteControl: RemoteControlStatus.Disabled }) this.store.update({ remoteControl: RemoteControlStatus.Disabled });
this.toggleAnnotation(false) this.toggleAnnotation(false);
} }
this.onToggle(enable) this.onToggle(enable);
} }
requestReleaseRemoteControl = () => { requestReleaseRemoteControl = () => {
const remoteControl = this.store.get().remoteControl const remoteControl = this.store.get().remoteControl;
if (remoteControl === RemoteControlStatus.Requesting) { return } console.log(remoteControl);
if (remoteControl === RemoteControlStatus.Disabled) { if (remoteControl === RemoteControlStatus.Requesting) {
this.store.update({ remoteControl: RemoteControlStatus.Requesting }) return;
this.emitData("request_control", JSON.stringify({
...this.agentInfo,
query: document.location.search
}))
} else {
this.releaseRemoteControl()
} }
} if (remoteControl === RemoteControlStatus.Disabled) {
this.store.update({ remoteControl: RemoteControlStatus.Requesting });
this.emitData(
'request_control',
JSON.stringify({
...this.agentInfo,
query: document.location.search,
})
);
} else {
this.onEnd();
this.releaseRemoteControl();
}
};
releaseRemoteControl = () => { releaseRemoteControl = () => {
this.emitData("release_control",) this.emitData('release_control');
this.toggleRemoteControl(false) this.toggleRemoteControl(false);
} };
private annot: AnnotationCanvas | null = null private annot: AnnotationCanvas | null = null;
toggleAnnotation(enable?: boolean) { toggleAnnotation(enable?: boolean) {
if (typeof enable !== "boolean") { if (typeof enable !== 'boolean') {
enable = this.store.get().annotating enable = this.store.get().annotating;
} }
if (enable && !this.annot) { if (enable && !this.annot) {
const annot = this.annot = new AnnotationCanvas() const annot = (this.annot = new AnnotationCanvas());
annot.mount(this.screen.overlay) annot.mount(this.screen.overlay);
annot.canvas.addEventListener("mousedown", e => { annot.canvas.addEventListener('mousedown', (e) => {
const data = this.screen.getInternalViewportCoordinates(e) const data = this.screen.getInternalViewportCoordinates(e);
annot.start([ data.x, data.y ]) annot.start([data.x, data.y]);
this.emitData("startAnnotation", [ data.x, data.y ]) this.emitData('startAnnotation', [data.x, data.y]);
}) });
annot.canvas.addEventListener("mouseleave", () => { annot.canvas.addEventListener('mouseleave', () => {
annot.stop() annot.stop();
this.emitData("stopAnnotation") this.emitData('stopAnnotation');
}) });
annot.canvas.addEventListener("mouseup", () => { annot.canvas.addEventListener('mouseup', () => {
annot.stop() annot.stop();
this.emitData("stopAnnotation") this.emitData('stopAnnotation');
}) });
annot.canvas.addEventListener("mousemove", e => { annot.canvas.addEventListener('mousemove', (e) => {
if (!annot.isPainting()) { return } if (!annot.isPainting()) {
return;
}
const data = this.screen.getInternalViewportCoordinates(e) const data = this.screen.getInternalViewportCoordinates(e);
annot.move([ data.x, data.y ]) annot.move([data.x, data.y]);
this.emitData("moveAnnotation", [ data.x, data.y ]) this.emitData('moveAnnotation', [data.x, data.y]);
}) });
this.store.update({ annotating: true }) this.store.update({ annotating: true });
} else if (!enable && !!this.annot) { } else if (!enable && !!this.annot) {
this.annot.remove() this.annot.remove();
this.annot = null this.annot = null;
this.store.update({ annotating: false }) this.store.update({ annotating: false });
} }
} }
clean() { clean() {
this.toggleRemoteControl(false) this.toggleRemoteControl(false);
if (this.annot) { if (this.annot) {
this.annot.remove() this.annot.remove();
this.annot = null this.annot = null;
} }
} }
} }

2
tracker/tracker-assist/src/RemoteControl.ts
View file

@ -42,7 +42,7 @@ export default class RemoteControl {
this.releaseControl() this.releaseControl()
return return
} }
setTimeout(() =>{ setTimeout(() => {
if (this.status === RCStatus.Requesting) { if (this.status === RCStatus.Requesting) {
this.releaseControl() this.releaseControl()
} }