From 6188b385554e8a25f5af796f56ac33d4b3cda9a6 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 23 Mar 2022 16:48:02 +0100 Subject: [PATCH] feat(frontend-assist): annotations & iremote typing --- frontend/.gitignore | 1 + .../StatedScreen/Screen/BaseScreen.ts | 23 +++-- .../StatedScreen/Screen/screen.css | 1 + .../managers/AnnotationCanvas.ts | 84 +++++++++++++++++++ .../managers/AssistManager.ts | 66 ++++++++++++--- 5 files changed, 155 insertions(+), 20 deletions(-) create mode 100644 frontend/app/player/MessageDistributor/managers/AnnotationCanvas.ts diff --git a/frontend/.gitignore b/frontend/.gitignore index 92150a232..dfcb0fd79 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -8,3 +8,4 @@ app/components/ui/SVG.js *.DS_Store .env *css.d.ts +*.cache diff --git a/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts b/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts index e2cd635fd..fc4acdf03 100644 --- a/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts +++ b/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts @@ -70,10 +70,10 @@ export default abstract class BaseScreen { private boundingRect: DOMRect | null = null; private getBoundingClientRect(): DOMRect { - //if (this.boundingRect === null) { - return this.boundingRect = this.overlay.getBoundingClientRect(); // expensive operation? - //} - //return this.boundingRect; + if (this.boundingRect === null) { + return this.boundingRect = this.overlay.getBoundingClientRect() // expensive operation? + } + return this.boundingRect } getInternalViewportCoordinates({ x, y }: Point): Point { @@ -85,17 +85,22 @@ export default abstract class BaseScreen { const screenX = (x - overlayX) * scale; const screenY = (y - overlayY) * scale; - return { x: screenX, y: screenY }; + return { x: Math.round(screenX), y: Math.round(screenY) }; + } + + getCurrentScroll(): Point { + const docEl = this.document?.documentElement + const x = docEl ? docEl.scrollLeft : 0 + const y = docEl ? docEl.scrollTop : 0 + return { x, y } } getInternalCoordinates(p: Point): Point { const { x, y } = this.getInternalViewportCoordinates(p); - const docEl = this.document?.documentElement - const scrollX = docEl ? docEl.scrollLeft : 0 - const scrollY = docEl ? docEl.scrollTop : 0 + const sc = this.getCurrentScroll() - return { x: x+scrollX, y: y+scrollY }; + return { x: x+sc.x, y: y+sc.y }; } getElementFromInternalPoint({ x, y }: Point): Element | null { diff --git a/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css b/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css index b715986d2..39808f1db 100644 --- a/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css +++ b/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css @@ -12,6 +12,7 @@ background: white; } .overlay { + user-select: none; position: absolute; top: 0; left: 0; diff --git a/frontend/app/player/MessageDistributor/managers/AnnotationCanvas.ts b/frontend/app/player/MessageDistributor/managers/AnnotationCanvas.ts new file mode 100644 index 000000000..ad110c2c3 --- /dev/null +++ b/frontend/app/player/MessageDistributor/managers/AnnotationCanvas.ts @@ -0,0 +1,84 @@ +export default class AnnotationCanvas { + readonly canvas: HTMLCanvasElement + private ctx: CanvasRenderingContext2D | null = null + private painting: boolean = false + constructor() { + this.canvas = document.createElement('canvas') + Object.assign(this.canvas.style, { + position: "fixed", + cursor: "url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABMAAAATCAYAAAByUDbMAAAAAXNSR0IArs4c6QAAAWNJREFUOE+l1D1Lw1AUBuD35Catg5NzaCMRMilINnGok7sguLg4OlRcBTd/hqBVB0ed7KDgIPgXhJoaG10Kgk4a83EkhcYYktimd703z31zzuESSqwGIDs1bRvAIiRcWrZ9ETFUwhJ6XTsDsPH7Le1bz08H42JkGMa09+W2CVhKBmHC7jhYlOgUTPdUEa3Q86+SIDN/j4olf43BtJMFjoJl1AgMUJMUcRInZHT+w7KgYakGoDxVafmue0hBsJeLmaapvPffziFhraDjDMKWZdvHRaNRlCi2mUNHYl55dBwrDysFZWGloTQ2EZTEJoZiTFXVmaos34Ixn9e5qNgCaHR6vW7emcFozNVmN1ERbfb9myww3bVCTK9rPsDrpCh37HnXAC3Ek5lqf9ErM0im1zUG8BmGtCqq4mEIjppoeEESA5g/JIkaLMuv7AVHEgfNohqlU/7Fol3mPodiufvS7Yz7cP4ARjbPWyYPZSMAAAAASUVORK5CYII=') 0 20, crosshair", + left: 0, + top: 0, + //zIndex: 2147483647 - 2, + }) + } + + isPainting() { + return this.painting + } + + private resizeCanvas = () => { + if (!this.canvas.parentElement) { return } + this.canvas.width = this.canvas.parentElement.offsetWidth + this.canvas.height = this.canvas.parentElement.offsetHeight + } + + private lastPosition: [number, number] = [0,0] + start = (p: [number, number]) => { + this.painting = true + this.clrTmID && clearTimeout(this.clrTmID) + this.lastPosition = p + } + + stop = () => { + this.painting = false + this.fadeOut() + } + + move = (p: [number, number]) =>{ + if (!this.ctx || !this.painting) { return } + this.ctx.globalAlpha = 1.0 + this.ctx.beginPath() + this.ctx.moveTo(this.lastPosition[0], this.lastPosition[1]) + this.ctx.lineTo(p[0], p[1]) + this.ctx.lineWidth = 8 + this.ctx.lineCap = "round" + this.ctx.lineJoin = "round" + this.ctx.strokeStyle = "red" + this.ctx.stroke() + this.lastPosition = p + } + + clrTmID: ReturnType | null = null + private fadeOut() { + let timeoutID: ReturnType + const fadeStep = () => { + if (!this.ctx || this.painting ) { return } + this.ctx.globalCompositeOperation = 'destination-out' + this.ctx.fillStyle = "rgba(255, 255, 255, 0.1)" + this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height) + this.ctx.globalCompositeOperation = 'source-over' + timeoutID = setTimeout(fadeStep,100) + } + this.clrTmID = setTimeout(() => { + clearTimeout(timeoutID) + this.ctx && + this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height) + }, 3700) + fadeStep() + } + + mount(parent: HTMLElement) { + parent.appendChild(this.canvas) + this.ctx = this.canvas.getContext("2d") + window.addEventListener("resize", this.resizeCanvas) + this.resizeCanvas() + } + + remove() { + if (this.canvas.parentNode){ + this.canvas.parentNode.removeChild(this.canvas) + } + window.removeEventListener("resize", this.resizeCanvas) + } +} \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager.ts b/frontend/app/player/MessageDistributor/managers/AssistManager.ts index 0b570fd87..92756567d 100644 --- a/frontend/app/player/MessageDistributor/managers/AssistManager.ts +++ b/frontend/app/player/MessageDistributor/managers/AssistManager.ts @@ -7,8 +7,8 @@ import store from 'App/store'; import type { LocalStream } from './LocalStream'; import { update, getState } from '../../store'; import { iceServerConfigFromString } from 'App/utils' - -import MStreamReader from '../messages/MStreamReader';; +import AnnotationCanvas from './AnnotationCanvas'; +import MStreamReader from '../messages/MStreamReader'; import JSONRawMessageReader from '../messages/JSONRawMessageReader' export enum CallingState { @@ -136,12 +136,13 @@ export default class AssistManager { //socket.onAny((...args) => console.log(...args)) socket.on("connect", () => { waitingForMessages = true - this.setStatus(ConnectionStatus.WaitingMessages) + this.setStatus(ConnectionStatus.WaitingMessages) // TODO: happens frequently on bad network }) socket.on("disconnect", () => { this.toggleRemoteControl(false) }) socket.on('messages', messages => { + //console.log(messages.filter(m => m._id === 41 || m._id === 44)) showDisconnectTimeout && clearTimeout(showDisconnectTimeout); jmr.append(messages) // as RawMessage[] @@ -173,9 +174,8 @@ export default class AssistManager { this.setStatus(ConnectionStatus.Disconnected) }, 30000) - if (getState().remoteControl === RemoteControlStatus.Requesting || - getState().remoteControl === RemoteControlStatus.Enabled) { - this.toggleRemoteControl(false) + if (getState().remoteControl === RemoteControlStatus.Requesting) { + this.toggleRemoteControl(false) // else its remaining } // Call State @@ -200,7 +200,7 @@ export default class AssistManager { private onMouseMove = (e: MouseEvent): void => { if (!this.socket) { return } const data = this.md.getInternalCoordinates(e) - this.socket.emit("move", [ Math.round(data.x), Math.round(data.y) ]) + this.socket.emit("move", [ data.x, data.y ]) } private onWheel = (e: WheelEvent): void => { @@ -213,15 +213,23 @@ export default class AssistManager { private onMouseClick = (e: MouseEvent): void => { if (!this.socket) { return; } - const data = this.md.getInternalViewportCoordinates(e); + const data = this.md.getInternalViewportCoordinates(e) // const el = this.md.getElementFromPoint(e); // requires requestiong node_id from domManager const el = this.md.getElementFromInternalPoint(data) if (el instanceof HTMLElement) { el.focus() - el.oninput = e => e.preventDefault(); - el.onkeydown = e => e.preventDefault(); + el.oninput = e => { + if (el instanceof HTMLTextAreaElement + || el instanceof HTMLInputElement + ) { + this.socket && this.socket.emit("input", el.value) + } else if (el.isContentEditable) { + this.socket && this.socket.emit("input", el.innerText) + } + } + //el.onkeydown = e => e.preventDefault() } - this.socket.emit("click", [ Math.round(data.x), Math.round(data.y) ]); + this.socket.emit("click", [ data.x, data.y ]); } private toggleRemoteControl(newState: boolean){ @@ -310,6 +318,8 @@ export default class AssistManager { this.callConnection && this.callConnection.close() update({ calling: CallingState.NoCall }) this.callArgs = null + this.annot?.remove() + this.annot = null } private initiateCallEnd = () => { @@ -355,6 +365,8 @@ export default class AssistManager { } } + private annot: AnnotationCanvas | null = null + private _call() { if (![CallingState.NoCall, CallingState.Reconnecting].includes(getState().calling)) { return } update({ calling: CallingState.Connecting }) @@ -379,6 +391,34 @@ export default class AssistManager { call.on('stream', stream => { update({ calling: CallingState.OnCall }) this.callArgs && this.callArgs.onStream(stream) + + if (!this.annot) { + const annot = this.annot = new AnnotationCanvas() + annot.mount(this.md.overlay) + annot.canvas.addEventListener("mousedown", e => { + if (!this.socket) { return } + const data = this.md.getInternalViewportCoordinates(e) + annot.start([ data.x, data.y ]) + this.socket.emit("startAnnotation", [ data.x, data.y ]) + }) + annot.canvas.addEventListener("mouseleave", () => { + if (!this.socket) { return } + annot.stop() + this.socket.emit("stopAnnotation") + }) + annot.canvas.addEventListener("mouseup", () => { + if (!this.socket) { return } + annot.stop() + this.socket.emit("stopAnnotation") + }) + annot.canvas.addEventListener("mousemove", e => { + if (!this.socket || !annot.isPainting()) { return } + + const data = this.md.getInternalViewportCoordinates(e) + annot.move([ data.x, data.y ]) + this.socket.emit("moveAnnotation", [ data.x, data.y ]) + }) + } }); //call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) @@ -409,6 +449,10 @@ export default class AssistManager { this.socket.close() document.removeEventListener('visibilitychange', this.onVisChange) } + if (this.annot) { + this.annot.remove() + this.annot = null + } } }