change(tracker): add more events to input, fix typo and timestamp calc

backend/pkg/messages/filters.go

@@ -2,7 +2,7 @@
 package messages
 
 func IsReplayerType(id int) bool {
-	return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 83 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id
+	return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 125 != id && 126 != id && 127 != id && 112 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id
 }
 
 func IsIOSType(id int) bool {
@@ -10,5 +10,5 @@ func IsIOSType(id int) bool {
 }
 
 func IsDOMType(id int) bool {
-	return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 84 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id
+	return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id
 }

backend/pkg/messages/messages.go

@@ -79,11 +79,11 @@ const (
 	MsgBatchMeta                   = 80
 	MsgBatchMetadata               = 81
 	MsgPartitionedMessage          = 82
-	MsgInputChange                 = 83
-	MsgSelectionChange             = 84
 	MsgIssueEvent                  = 125
 	MsgSessionEnd                  = 126
 	MsgSessionSearch               = 127
+	MsgInputChange                 = 112
+	MsgSelectionChange             = 113
 	MsgIOSBatchMeta                = 107
 	MsgIOSSessionStart             = 90
 	MsgIOSSessionEnd               = 91
@@ -2119,56 +2119,6 @@ func (msg *PartitionedMessage) TypeID() int {
 	return 82
 }
 
-type InputChange struct {
-	message
-	ID             uint64
-	Label          string
-	HesitationTime int64
-}
-
-func (msg *InputChange) Encode() []byte {
-	buf := make([]byte, 31+len(msg.Label))
-	buf[0] = 83
-	p := 1
-	p = WriteUint(msg.ID, buf, p)
-	p = WriteString(msg.Label, buf, p)
-	p = WriteInt(msg.HesitationTime, buf, p)
-	return buf[:p]
-}
-
-func (msg *InputChange) Decode() Message {
-	return msg
-}
-
-func (msg *InputChange) TypeID() int {
-	return 83
-}
-
-type SelectionChange struct {
-	message
-	SelectionStart uint64
-	SelectionEnd   uint64
-	Selection      string
-}
-
-func (msg *SelectionChange) Encode() []byte {
-	buf := make([]byte, 31+len(msg.Selection))
-	buf[0] = 84
-	p := 1
-	p = WriteUint(msg.SelectionStart, buf, p)
-	p = WriteUint(msg.SelectionEnd, buf, p)
-	p = WriteString(msg.Selection, buf, p)
-	return buf[:p]
-}
-
-func (msg *SelectionChange) Decode() Message {
-	return msg
-}
-
-func (msg *SelectionChange) TypeID() int {
-	return 84
-}
-
 type IssueEvent struct {
 	message
 	MessageID     uint64
@@ -2248,6 +2198,62 @@ func (msg *SessionSearch) TypeID() int {
 	return 127
 }
 
+type InputChange struct {
+	message
+	ID             uint64
+	Value          string
+	ValueMasked    bool
+	Label          string
+	HesitationTime int64
+	InputDuration  int64
+}
+
+func (msg *InputChange) Encode() []byte {
+	buf := make([]byte, 61+len(msg.Value)+len(msg.Label))
+	buf[0] = 112
+	p := 1
+	p = WriteUint(msg.ID, buf, p)
+	p = WriteString(msg.Value, buf, p)
+	p = WriteBoolean(msg.ValueMasked, buf, p)
+	p = WriteString(msg.Label, buf, p)
+	p = WriteInt(msg.HesitationTime, buf, p)
+	p = WriteInt(msg.InputDuration, buf, p)
+	return buf[:p]
+}
+
+func (msg *InputChange) Decode() Message {
+	return msg
+}
+
+func (msg *InputChange) TypeID() int {
+	return 112
+}
+
+type SelectionChange struct {
+	message
+	SelectionStart uint64
+	SelectionEnd   uint64
+	Selection      string
+}
+
+func (msg *SelectionChange) Encode() []byte {
+	buf := make([]byte, 31+len(msg.Selection))
+	buf[0] = 113
+	p := 1
+	p = WriteUint(msg.SelectionStart, buf, p)
+	p = WriteUint(msg.SelectionEnd, buf, p)
+	p = WriteString(msg.Selection, buf, p)
+	return buf[:p]
+}
+
+func (msg *SelectionChange) Decode() Message {
+	return msg
+}
+
+func (msg *SelectionChange) TypeID() int {
+	return 113
+}
+
 type IOSBatchMeta struct {
 	message
 	Timestamp  uint64

backend/pkg/messages/read-message.go

@@ -1293,36 +1293,6 @@ func DecodePartitionedMessage(reader BytesReader) (Message, error) {
 	return msg, err
 }
 
-func DecodeInputChange(reader BytesReader) (Message, error) {
-	var err error = nil
-	msg := &InputChange{}
-	if msg.ID, err = reader.ReadUint(); err != nil {
-		return nil, err
-	}
-	if msg.Label, err = reader.ReadString(); err != nil {
-		return nil, err
-	}
-	if msg.HesitationTime, err = reader.ReadInt(); err != nil {
-		return nil, err
-	}
-	return msg, err
-}
-
-func DecodeSelectionChange(reader BytesReader) (Message, error) {
-	var err error = nil
-	msg := &SelectionChange{}
-	if msg.SelectionStart, err = reader.ReadUint(); err != nil {
-		return nil, err
-	}
-	if msg.SelectionEnd, err = reader.ReadUint(); err != nil {
-		return nil, err
-	}
-	if msg.Selection, err = reader.ReadString(); err != nil {
-		return nil, err
-	}
-	return msg, err
-}
-
 func DecodeIssueEvent(reader BytesReader) (Message, error) {
 	var err error = nil
 	msg := &IssueEvent{}
@@ -1374,6 +1344,45 @@ func DecodeSessionSearch(reader BytesReader) (Message, error) {
 	return msg, err
 }
 
+func DecodeInputChange(reader BytesReader) (Message, error) {
+	var err error = nil
+	msg := &InputChange{}
+	if msg.ID, err = reader.ReadUint(); err != nil {
+		return nil, err
+	}
+	if msg.Value, err = reader.ReadString(); err != nil {
+		return nil, err
+	}
+	if msg.ValueMasked, err = reader.ReadBoolean(); err != nil {
+		return nil, err
+	}
+	if msg.Label, err = reader.ReadString(); err != nil {
+		return nil, err
+	}
+	if msg.HesitationTime, err = reader.ReadInt(); err != nil {
+		return nil, err
+	}
+	if msg.InputDuration, err = reader.ReadInt(); err != nil {
+		return nil, err
+	}
+	return msg, err
+}
+
+func DecodeSelectionChange(reader BytesReader) (Message, error) {
+	var err error = nil
+	msg := &SelectionChange{}
+	if msg.SelectionStart, err = reader.ReadUint(); err != nil {
+		return nil, err
+	}
+	if msg.SelectionEnd, err = reader.ReadUint(); err != nil {
+		return nil, err
+	}
+	if msg.Selection, err = reader.ReadString(); err != nil {
+		return nil, err
+	}
+	return msg, err
+}
+
 func DecodeIOSBatchMeta(reader BytesReader) (Message, error) {
 	var err error = nil
 	msg := &IOSBatchMeta{}
@@ -1932,16 +1941,16 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) {
 		return DecodeBatchMetadata(reader)
 	case 82:
 		return DecodePartitionedMessage(reader)
-	case 83:
-		return DecodeInputChange(reader)
-	case 84:
-		return DecodeSelectionChange(reader)
 	case 125:
 		return DecodeIssueEvent(reader)
 	case 126:
 		return DecodeSessionEnd(reader)
 	case 127:
 		return DecodeSessionSearch(reader)
+	case 112:
+		return DecodeInputChange(reader)
+	case 113:
+		return DecodeSelectionChange(reader)
 	case 107:
 		return DecodeIOSBatchMeta(reader)
 	case 90:

ee/connectors/msgcodec/messages.py

@@ -744,24 +744,6 @@ class PartitionedMessage(Message):
         self.part_total = part_total
 
 
-class InputChange(Message):
-    __id__ = 83
-
-    def __init__(self, id, label, hesitation_time):
-        self.id = id
-        self.label = label
-        self.hesitation_time = hesitation_time
-
-
-class SelectionChange(Message):
-    __id__ = 84
-
-    def __init__(self, selection_start, selection_end, selection):
-        self.selection_start = selection_start
-        self.selection_end = selection_end
-        self.selection = selection
-
-
 class IssueEvent(Message):
     __id__ = 125
 
@@ -791,6 +773,27 @@ class SessionSearch(Message):
         self.partition = partition
 
 
+class InputChange(Message):
+    __id__ = 112
+
+    def __init__(self, id, value, value_masked, label, hesitation_time, input_duration):
+        self.id = id
+        self.value = value
+        self.value_masked = value_masked
+        self.label = label
+        self.hesitation_time = hesitation_time
+        self.input_duration = input_duration
+
+
+class SelectionChange(Message):
+    __id__ = 113
+
+    def __init__(self, selection_start, selection_end, selection):
+        self.selection_start = selection_start
+        self.selection_end = selection_end
+        self.selection = selection
+
+
 class IOSBatchMeta(Message):
     __id__ = 107
 

ee/connectors/msgcodec/msgcodec.py

@@ -660,20 +660,6 @@ class MessageCodec(Codec):
                 part_total=self.read_uint(reader)
             )
 
-        if message_id == 83:
-            return InputChange(
-                id=self.read_uint(reader),
-                label=self.read_string(reader),
-                hesitation_time=self.read_int(reader)
-            )
-
-        if message_id == 84:
-            return SelectionChange(
-                selection_start=self.read_uint(reader),
-                selection_end=self.read_uint(reader),
-                selection=self.read_string(reader)
-            )
-
         if message_id == 125:
             return IssueEvent(
                 message_id=self.read_uint(reader),
@@ -697,6 +683,23 @@ class MessageCodec(Codec):
                 partition=self.read_uint(reader)
             )
 
+        if message_id == 112:
+            return InputChange(
+                id=self.read_uint(reader),
+                value=self.read_string(reader),
+                value_masked=self.read_boolean(reader),
+                label=self.read_string(reader),
+                hesitation_time=self.read_int(reader),
+                input_duration=self.read_int(reader)
+            )
+
+        if message_id == 113:
+            return SelectionChange(
+                selection_start=self.read_uint(reader),
+                selection_end=self.read_uint(reader),
+                selection=self.read_string(reader)
+            )
+
         if message_id == 107:
             return IOSBatchMeta(
                 timestamp=self.read_uint(reader),

frontend/app/player/web/messages/RawMessageReader.gen.ts

@@ -627,7 +627,7 @@ export default class RawMessageReader extends PrimitiveReader {
       };
     }
 
-    case 84: {
+    case 113: {
       const selectionStart = this.readUint(); if (selectionStart === null) { return resetPointer() }
       const selectionEnd = this.readUint(); if (selectionEnd === null) { return resetPointer() }
       const selection = this.readString(); if (selection === null) { return resetPointer() }

frontend/app/player/web/messages/filters.gen.ts

@@ -3,7 +3,7 @@
 
 import { MType } from './raw.gen'
 
-const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,84,90,93,96,100,102,103,105]
+const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,113,90,93,96,100,102,103,105]
 export function isDOMType(t: MType) {
 	return DOM_TYPES.includes(t)
 }

frontend/app/player/web/messages/raw.gen.ts

@@ -53,7 +53,7 @@ export const enum MType {
   AdoptedSsAddOwner = 76,
   AdoptedSsRemoveOwner = 77,
   Zustand = 79,
-  SelectionChange = 84,
+  SelectionChange = 113,
   IosSessionStart = 90,
   IosCustomEvent = 93,
   IosScreenChanges = 96,

frontend/app/player/web/messages/tracker-legacy.gen.ts

@@ -54,7 +54,7 @@ export const TP_MAP = {
   76: MType.AdoptedSsAddOwner,
   77: MType.AdoptedSsRemoveOwner,
   79: MType.Zustand,
-  84: MType.SelectionChange,
+  113: MType.SelectionChange,
   90: MType.IosSessionStart,
   93: MType.IosCustomEvent,
   96: MType.IosScreenChanges,

frontend/app/player/web/messages/tracker.gen.ts

@@ -430,14 +430,17 @@ type TrPartitionedMessage = [
 ]
 
 type TrInputChange = [
-  type: 83,
+  type: 112,
   id: number,
+  value: string,
+  valueMasked: boolean,
   label: string,
   hesitationTime: number,
+  inputDuration: number,
 ]
 
 type TrSelectionChange = [
-  type: 84,
+  type: 113,
   selectionStart: number,
   selectionEnd: number,
   selection: string,
@@ -881,7 +884,7 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null {
       }
     }
     
-    case 84: {
+    case 113: {
       return {
         tp: MType.SelectionChange,
         selectionStart: tMsg[1],

mobs/messages.rb

@@ -475,17 +475,7 @@ message 82, 'PartitionedMessage', :replayer => false do
   uint 'PartTotal'
 end
 
-message 83, 'InputChange', :replayer => false do
+# 90-111 reserved iOS
-    uint 'ID'
-    string 'Label'
-    int 'HesitationTime'
-end
-
-message 84, 'SelectionChange' do
-    uint 'SelectionStart'
-    uint 'SelectionEnd'
-    string 'Selection'
-end
 
 ## Backend-only
 message 125, 'IssueEvent', :replayer => false, :tracker => false do
@@ -505,3 +495,20 @@ message 127, 'SessionSearch', :tracker => false, :replayer => false  do
   uint 'Timestamp'
   uint 'Partition'
 end
+
+# since tracker 4.1.10
+
+message 112, 'InputChange', :replayer => false do
+    uint 'ID'
+    string 'Value'
+    boolean 'ValueMasked'
+    string 'Label'
+    int 'HesitationTime'
+    int 'InputDuration'
+end
+
+message 113, 'SelectionChange' do
+    uint 'SelectionStart'
+    uint 'SelectionEnd'
+    string 'Selection'
+end

tracker/tracker/src/common/messages.gen.ts

@@ -63,8 +63,8 @@ export declare const enum Type {
   Zustand = 79,
   BatchMetadata = 81,
   PartitionedMessage = 82,
-  InputChange = 83,
+  InputChange = 112,
-  SelectionChange = 84,
+  SelectionChange = 113,
 }
 
 
@@ -495,8 +495,11 @@ export type PartitionedMessage = [
 export type InputChange = [
   /*type:*/ Type.InputChange,
   /*id:*/ number,
+  /*value:*/ string,
+  /*valueMasked:*/ boolean,
   /*label:*/ string,
   /*hesitationTime:*/ number,
+  /*inputDuration:*/ number,
 ]
 
 export type SelectionChange = [

tracker/tracker/src/main/app/messages.gen.ts

@@ -794,14 +794,20 @@ export function PartitionedMessage(
 
 export function InputChange(
   id: number,
+  value: string,
+  valueMasked: boolean,
   label: string,
   hesitationTime: number,
+  inputDuration: number,
 ): Messages.InputChange {
   return [
     Messages.Type.InputChange,
     id,
+    value,
+    valueMasked,
     label,
     hesitationTime,
+    inputDuration,
   ]
 }
 

tracker/tracker/src/main/modules/input.ts

@@ -6,9 +6,9 @@ import { InputChange, SetInputValue, SetInputChecked } from '../app/messages.gen
 const INPUT_TYPES = ['text', 'password', 'email', 'search', 'number', 'range', 'date', 'tel']
 
 // TODO: take into consideration "contenteditable" attribute
-type TextFeildElement = HTMLInputElement | HTMLTextAreaElement
+type TextFieldElement = HTMLInputElement | HTMLTextAreaElement
 
-function isTextFeildElement(node: Node): node is TextFeildElement {
+function isTextFieldElement(node: Node): node is TextFieldElement {
   if (hasTag(node, 'textarea')) {
     return true
   }
@@ -27,7 +27,7 @@ function isCheckbox(node: Node): node is HTMLInputElement & { type: 'checkbox' |
   return type === 'checkbox' || type === 'radio'
 }
 
-const labelElementFor: (element: TextFeildElement) => HTMLLabelElement | undefined =
+const labelElementFor: (element: TextFieldElement) => HTMLLabelElement | undefined =
   IN_BROWSER && 'labels' in HTMLInputElement.prototype
     ? (node) => {
         let p: Node | null = node
@@ -57,7 +57,7 @@ const labelElementFor: (element: TextFeildElement) => HTMLLabelElement | undefin
         }
       }
 
-export function getInputLabel(node: TextFeildElement): string {
+export function getInputLabel(node: TextFieldElement): string {
   let label = getLabelAttribute(node)
   if (label === null) {
     const labelElement = labelElementFor(node)
@@ -96,7 +96,7 @@ export default function (app: App, opts: Partial<Options>): void {
     opts,
   )
 
-  function sendInputValue(id: number, node: TextFeildElement | HTMLSelectElement): void {
+  function getInputValue(id: number, node: TextFieldElement | HTMLSelectElement) {
     let value = node.value
     let inputMode: InputMode = options.defaultInputMode
 
@@ -123,6 +123,11 @@ export default function (app: App, opts: Partial<Options>): void {
         break
     }
 
+    return { value, mask }
+  }
+  function sendInputValue(id: number, node: TextFieldElement | HTMLSelectElement): void {
+    const { value, mask } = getInputValue(id, node)
+
     app.send(SetInputValue(id, value, mask))
   }
 
@@ -134,7 +139,7 @@ export default function (app: App, opts: Partial<Options>): void {
     checkboxValues.clear()
   })
 
-  function trackInputValue(id: number, node: TextFeildElement) {
+  function trackInputValue(id: number, node: TextFieldElement) {
     if (inputValues.get(id) === node.value) {
       return
     }
@@ -150,7 +155,7 @@ export default function (app: App, opts: Partial<Options>): void {
     app.send(SetInputChecked(id, value))
   }
 
-  // The only way (to our knowladge) to track all kinds of input changes, including those made by JS
+  // The only way (to our knowledge) to track all kinds of input changes, including those made by JS
   app.ticker.attach(() => {
     inputValues.forEach((value, id) => {
       const node = app.nodes.getNode(id) as HTMLInputElement
@@ -162,12 +167,18 @@ export default function (app: App, opts: Partial<Options>): void {
       if (!node) return checkboxValues.delete(id)
       trackCheckboxValue(id, node.checked)
     })
-  }, 5)
+  }, 3)
 
-  function sendInputChange(id: number, node: TextFeildElement, hesitationTime: number) {
+  function sendInputChange(
-    trackInputValue(id, node)
+    id: number,
+    node: TextFieldElement,
+    hesitationTime: number,
+    inputTime: number,
+  ) {
+    const { value, mask } = getInputValue(id, node)
     const label = getInputLabel(node)
-    app.send(InputChange(id, label, hesitationTime))
+
+    app.send(InputChange(id, value, mask !== 0, label, hesitationTime, inputTime))
   }
 
   app.nodes.attachNodeCallback(
@@ -183,22 +194,27 @@ export default function (app: App, opts: Partial<Options>): void {
         app.nodes.attachNodeListener(node, 'change', () => sendInputValue(id, node))
       }
 
-      if (isTextFeildElement(node)) {
+      if (isTextFieldElement(node)) {
         trackInputValue(id, node)
         let nodeFocusTime = 0
         let nodeHesitationTime = 0
+        let inputTime = 0
+
         const onFocus = () => {
           nodeFocusTime = now()
         }
+
         const onInput = () => {
-          const value = node.value
           if (nodeHesitationTime === 0) {
-            nodeHesitationTime = nodeFocusTime - now()
+            nodeHesitationTime = now() - nodeFocusTime
           }
         }
+
         const onChange = () => {
-          sendInputChange(id, node, nodeHesitationTime)
+          inputTime = now() - nodeFocusTime
+          sendInputChange(id, node, nodeHesitationTime, inputTime)
           nodeHesitationTime = 0
+          inputTime = 0
         }
         app.nodes.attachNodeListener(node, 'focus', onFocus)
         app.nodes.attachNodeListener(node, 'input', onInput)
@@ -208,7 +224,7 @@ export default function (app: App, opts: Partial<Options>): void {
 
       if (isCheckbox(node)) {
         trackCheckboxValue(id, node.checked)
-        app.nodes.attachNodeListener(node, 'change', (e) => trackCheckboxValue(id, node.checked))
+        app.nodes.attachNodeListener(node, 'change', () => trackCheckboxValue(id, node.checked))
         return
       }
     }),

tracker/tracker/src/webworker/MessageEncoder.gen.ts

@@ -255,7 +255,7 @@ export default class MessageEncoder extends PrimitiveEncoder {
     break
 
     case Messages.Type.InputChange:
-      return  this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3])
+      return  this.uint(msg[1]) && this.string(msg[2]) && this.boolean(msg[3]) && this.string(msg[4]) && this.int(msg[5]) && this.int(msg[6])
     break
 
     case Messages.Type.SelectionChange: