From 018f618e127519e6e467e401f7291a0941999cfe Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 8 Feb 2023 15:18:00 +0100 Subject: [PATCH] change(tracker): add mouse shake and update input hesitation message --- backend/pkg/messages/messages.go | 10 +++-- backend/pkg/messages/read-message.go | 3 ++ ee/connectors/msgcodec/messages.py | 3 +- ee/connectors/msgcodec/msgcodec.py | 1 + .../web/messages/RawMessageReader.gen.ts | 2 + frontend/app/player/web/messages/raw.gen.ts | 1 + .../app/player/web/messages/tracker.gen.ts | 4 +- mobs/messages.rb | 1 + tracker/tracker/src/common/messages.gen.ts | 1 + tracker/tracker/src/main/app/messages.gen.ts | 2 + tracker/tracker/src/main/modules/input.ts | 7 ++-- tracker/tracker/src/main/modules/mouse.ts | 41 +++++++++++++++++++ .../src/webworker/MessageEncoder.gen.ts | 2 +- 13 files changed, 68 insertions(+), 10 deletions(-) diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index a96f98de8..f3c422af2 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -531,17 +531,19 @@ func (msg *SetInputTarget) TypeID() int { type SetInputValue struct { message - ID uint64 - Value string - Mask int64 + ID uint64 + Value string + HesitationTime int64 + Mask int64 } func (msg *SetInputValue) Encode() []byte { - buf := make([]byte, 31+len(msg.Value)) + buf := make([]byte, 41+len(msg.Value)) buf[0] = 18 p := 1 p = WriteUint(msg.ID, buf, p) p = WriteString(msg.Value, buf, p) + p = WriteInt(msg.HesitationTime, buf, p) p = WriteInt(msg.Mask, buf, p) return buf[:p] } diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index ecc00183f..56d0662aa 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -270,6 +270,9 @@ func DecodeSetInputValue(reader BytesReader) (Message, error) { if msg.Value, err = reader.ReadString(); err != nil { return nil, err } + if msg.HesitationTime, err = reader.ReadInt(); err != nil { + return nil, err + } if msg.Mask, err = reader.ReadInt(); err != nil { return nil, err } diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 54f8df955..53d993bd2 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -163,9 +163,10 @@ class SetInputTarget(Message): class SetInputValue(Message): __id__ = 18 - def __init__(self, id, value, mask): + def __init__(self, id, value, hesitation_time, mask): self.id = id self.value = value + self.hesitation_time = hesitation_time self.mask = mask diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 0ba21ea12..2b1d05a07 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -200,6 +200,7 @@ class MessageCodec(Codec): return SetInputValue( id=self.read_uint(reader), value=self.read_string(reader), + hesitation_time=self.read_int(reader), mask=self.read_int(reader) ) diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 793f609f5..50b5a1a68 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -172,11 +172,13 @@ export default class RawMessageReader extends PrimitiveReader { case 18: { const id = this.readUint(); if (id === null) { return resetPointer() } const value = this.readString(); if (value === null) { return resetPointer() } + const hesitationTime = this.readInt(); if (hesitationTime === null) { return resetPointer() } const mask = this.readInt(); if (mask === null) { return resetPointer() } return { tp: MType.SetInputValue, id, value, + hesitationTime, mask, }; } diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index b51edb40e..8e6d613a4 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -156,6 +156,7 @@ export interface RawSetInputValue { tp: MType.SetInputValue, id: number, value: string, + hesitationTime: number, mask: number, } diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index 2c171011c..13f1d82e7 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -98,6 +98,7 @@ type TrSetInputValue = [ type: 18, id: number, value: string, + hesitationTime: number, mask: number, ] @@ -549,7 +550,8 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { tp: MType.SetInputValue, id: tMsg[1], value: tMsg[2], - mask: tMsg[3], + hesitationTime: tMsg[3], + mask: tMsg[4], } } diff --git a/mobs/messages.rb b/mobs/messages.rb index ef36ebfa7..c5ba732b6 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -94,6 +94,7 @@ end message 18, 'SetInputValue' do uint 'ID' string 'Value' + int 'HesitationTime' int 'Mask' end message 19, 'SetInputChecked' do diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 8f61ab917..08461e6b8 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -159,6 +159,7 @@ export type SetInputValue = [ /*type:*/ Type.SetInputValue, /*id:*/ number, /*value:*/ string, + /*hesitationTime:*/ number, /*mask:*/ number, ] diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index bdec57a0f..e8005a430 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -172,12 +172,14 @@ export function SetInputTarget( export function SetInputValue( id: number, value: string, + hesitationTime: number, mask: number, ): Messages.SetInputValue { return [ Messages.Type.SetInputValue, id, value, + hesitationTime, mask, ] } diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 210a1717a..83a43313d 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -129,9 +129,10 @@ export default function (app: App, opts: Partial): void { value = '' break } - // @ts-ignore if hesitationTime > 150 add it ??? - console.log(node.or_inputHesitation) - app.send(SetInputValue(id, value, mask)) + + // @ts-ignore maybe if hesitationTime > 150 ? + const hesitationTime = node.or_inputHesitation || 0 + app.send(SetInputValue(id, value, hesitationTime, mask)) } const inputValues: Map = new Map() diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index b00d6d304..7ed64f1c4 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -107,12 +107,45 @@ export default function (app: App): void { let mouseTargetTime = 0 let selectorMap: { [id: number]: string } = {} + let velocity = 0 + let direction = 0 + let directionChangeCount = 0 + let distance = 0 + let checkIntervalId: NodeJS.Timer + const shakeThreshold = 0.005 + const shakeCheckInterval = 225 + + function checkMouseShaking() { + const nextVelocity = distance / shakeCheckInterval + + if (!velocity) { + velocity = nextVelocity + return + } + + const acceleration = (nextVelocity - velocity) / shakeCheckInterval + if (directionChangeCount && acceleration > shakeThreshold) { + console.log('Mouse shake detected!') + } + + distance = 0 + directionChangeCount = 0 + velocity = nextVelocity + } + + app.attachStartCallback(() => { + checkIntervalId = setInterval(() => checkMouseShaking(), shakeCheckInterval) + }) + app.attachStopCallback(() => { mousePositionX = -1 mousePositionY = -1 mousePositionChanged = false mouseTarget = null selectorMap = {} + if (checkIntervalId) { + clearInterval(checkIntervalId) + } }) const sendMouseMove = (): void => { @@ -146,6 +179,14 @@ export default function (app: App): void { mousePositionX = e.clientX + left mousePositionY = e.clientY + top mousePositionChanged = true + + const nextDirection = Math.sign(e.movementX) + distance += Math.abs(e.movementX) + Math.abs(e.movementY) + + if (nextDirection !== direction) { + direction = nextDirection + directionChangeCount++ + } }, false, ) diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index e6e522dd4..60079647f 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -67,7 +67,7 @@ export default class MessageEncoder extends PrimitiveEncoder { break case Messages.Type.SetInputValue: - return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) + return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) && this.int(msg[4]) break case Messages.Type.SetInputChecked: