jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add 'ee/quickwit/' from commit '2ad4f4ea124232274bc09ab1f1faf2581053e619'

Browse source 
git-subtree-dir: ee/quickwit
git-subtree-mainline: f1e4d60ea8
git-subtree-split: 2ad4f4ea12
This commit is contained in:
mauricio garcia suarez 2022-08-01 11:27:57 +02:00
commit 1da78afa01
18 changed files with 2113 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
ee/quickwit/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
.idea
Pip*

39
ee/quickwit/README.md Normal file
View file

@ -0,0 +1,39 @@
# Quickwit for kafka messages (S3 storage)
## index
1. [Setup](#setup)
2. [Deploy](#deploy)
## Setup
This setup is made using Docker, make changes to the files accordingly to run it locally.
In order to connect to AWS S3 service the aws credentials must be defined in the environment
```bash
export aws_access_key_id={your_aws_access_key_id}
export aws_secret_access_key={your_aws_secret_access_key}
export aws_region={bucket_region}
```
In the file kafka-source.yaml replace the bootstap.server with the address of your kafka service and uncomment to activate ssl protocol if needed.
## Deploy
To create the index 'quickwit-kafka' run the command:
```bash
bash create_kafka_index.sh
```
Having the topic 'quickwit-kafka' in the kafka server defined in the kafka-source.yaml, the connection between the created index and the topic can be achieved by running the command:
```bash
bash create_source.sh
```
To delete both the index and the source connection run the command:
```bash
bash clean.sh
```
To deploy the indexer, search and UI services run the command:
```bash
bash run_quickwit.sh
```
UI server will start at localhost:7280. The api can also be called through the url http://127.0.0.1:7280/api/v1/quickwit-kafka/search?query={your_query} for example
```bash
curl "http://127.0.0.1:7280/api/v1/quickwit-kafka/search?query=body:error"
```

8
ee/quickwit/clean.sh Normal file
View file

@ -0,0 +1,8 @@
docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source delete --index fetchevent --source fetch-kafka --config s3-config.yaml
docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index delete --index fetchevent --config s3-config.yaml
docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source delete --index graphql --source graphql-kafka --config s3-config.yaml
docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index delete --index graphql --config s3-config.yaml
docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source delete --index pageevent --source pageevent-kafka --config s3-config.yaml
docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index delete --index pageevent --config s3-config.yaml

3
ee/quickwit/create_kafka_index.sh Normal file
View file

@ -0,0 +1,3 @@
docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/index-config-fetch.yaml:/quickwit/index-config-fetch.yaml -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index create --index-config index-config-fetch.yaml --config s3-config.yaml
docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/index-config-graphql.yaml:/quickwit/index-config-graphql.yaml -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index create --index-config index-config-graphql.yaml --config s3-config.yaml
docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/index-config-pageevent.yaml:/quickwit/index-config-pageevent.yaml -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index create --index-config index-config-pageevent.yaml --config s3-config.yaml

3
ee/quickwit/create_sources.sh Normal file
View file

@ -0,0 +1,3 @@
docker run -v /etc/hosts:/etc/hosts:ro -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -v $(pwd)/source-fetch.yaml:/quickwit/source-fetch.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source create --index fetchevent --source-config source-fetch.yaml --config s3-config.yaml
docker run -v /etc/hosts:/etc/hosts:ro -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -v $(pwd)/source-graphql.yaml:/quickwit/source-graphql.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source create --index graphql --source-config source-graphql.yaml --config s3-config.yaml
docker run -v /etc/hosts:/etc/hosts:ro -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -v $(pwd)/source-pageevent.yaml:/quickwit/source-pageevent.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source create --index pageevent --source-config source-pageevent.yaml --config s3-config.yaml

40
ee/quickwit/index-config-fetch.yaml Normal file
View file

@ -0,0 +1,40 @@
#
# Index config file for gh-archive dataset.
#
version: 0
index_id: fetchevent
doc_mapping:
mode: strict
field_mappings:
- name: method
type: text
tokenizer: default
record: position
- name: url
type: text
tokenizer: default
record: position
- name: request
type: text
tokenizer: default
record: position
- name: response
type: text
tokenizer: default
record: position
- name: status
type: i64
indexed: true
fast: true
- name: timestamp
type: i64
fast: true
- name: duration
type: i64
fast: true
search_settings:
default_search_fields: [url, request, response]

30
ee/quickwit/index-config-graphql.yaml Normal file
View file

@ -0,0 +1,30 @@
#
# Index config file for gh-archive dataset.
#
version: 0
index_id: graphql
doc_mapping:
mode: strict
field_mappings:
- name: operation_kind
type: text
tokenizer: default
record: position
- name: operation_name
type: text
tokenizer: default
record: position
- name: variables
type: text
tokenizer: default
record: position
- name: response
type: text
tokenizer: default
record: position
search_settings:
default_search_fields: [operation_kind, operation_name, variables]

68
ee/quickwit/index-config-pageevent.yaml Normal file
View file

@ -0,0 +1,68 @@
#
# Index config file for gh-archive dataset.
#
version: 0
index_id: pageevent
doc_mapping:
mode: strict
field_mappings:
- name: message_id
type: i64
indexed: true
fast: true
- name: timestamp
type: i64
fast: true
- name: url
type: text
tokenizer: default
record: position
- name: referrer
type: text
tokenizer: default
record: position
- name: loaded
type: i64
fast: true
- name: request_start
type: i64
fast: true
- name: response_start
type: i64
fast: true
- name: response_end
type: i64
fast: true
- name: dom_content_loaded_event_start
type: i64
fast: true
- name: dom_content_loaded_event_end
type: i64
fast: true
- name: load_event_start
type: i64
fast: true
- name: load_event_end
type: i64
fast: true
- name: first_paint
type: i64
fast: true
- name: first_contentful_paint
type: i64
fast: true
- name: speed_index
type: i64
fast: true
- name: visually_complete
type: i64
fast: true
- name: time_to_interactive
type: i64
fast: true
search_settings:
default_search_fields: [url, referrer, visually_complete]

132
ee/quickwit/kafka_sample.py Normal file
View file

@ -0,0 +1,132 @@
import os
from time import sleep
from kafka import KafkaConsumer, KafkaProducer
from datetime import datetime
from collections import defaultdict
from msgcodec.codec import MessageCodec
from msgcodec.messages import Fetch, FetchEvent, PageEvent, GraphQL
import json
import getopt, sys
n = 0
def transform_fetch(data):
global n
n += 1
return {
'method': data.method, 'url': data.url, 'request': data.url, 'response': data.request,
'status': data.status, 'timestamp': data.timestamp, 'duration': data.duration
}
def transform_graphql(data):
global n
n += 1
return {
'operation_kind': data.operation_kind, 'operation_name': data.operation_name,
'variables': data.variables, 'response': data.response
}
def transform_pageevent(data):
global n
n += 1
return {'massage_id': data.message_id, 'timestamp': data.timestamp, 'url': data.timestamp,
'referrer': data.referrer, 'loaded': data.loaded, 'request_start': data.request_start,
'response_start': data.response_start, 'response_end': data.response_end,
'dom_content_loaded_event_start': data.dom_content_loaded_event_start,
'dom_content_loaded_event_end': data.dom_content_loaded_event_end,
'load_event_start': data.load_event_start, 'load_event_end': data.load_event_end,
'first_paint': data.first_paint, 'first_contentful_paint': data.first_contentful_paint,
'speed_index': data.speed_index, 'visually_complete': data.visually_complete,
'time_to_interactive': data.time_to_interactive
}
def create_producer():
producer = KafkaProducer(#security_protocol="SSL",
bootstrap_servers=os.environ['KAFKA_SERVER_2'],
# os.environ['KAFKA_SERVER_1']],
#ssl_cafile="./ca.pem",
#ssl_certfile="./service.cert",
#ssl_keyfile="./service.key",
value_serializer=lambda v: json.dumps(v).encode('ascii')
)
return producer
def create_consumer():
consumer = KafkaConsumer(#security_protocol="SSL",
bootstrap_servers=os.environ['KAFKA_SERVER_2'],
# os.environ['KAFKA_SERVER_1']],
group_id=f"quickwit_connector2",
auto_offset_reset="earliest",
enable_auto_commit=False
)
return consumer
def consumer_producer_end():
global n
codec = MessageCodec()
consumer = create_consumer()
producer = create_producer()
consumer.subscribe(topics=["raw", "raw_ios"])
print("Kafka consumer subscribed")
escape = 0
for msg in consumer:
messages = codec.decode_detailed(msg.value)
session_id = codec.decode_key(msg.key)
if messages is None:
print('-')
for message in messages:
send = False
if isinstance(message, Fetch) or isinstance(message, FetchEvent):
producer.send('quickwit-kafka', value=transform_fetch(message))
print(f'added message {n} type Fetch')
sleep(5)
if isinstance(message, GraphQL):
producer.send('quickwit-kafka', value=transform_graphql(message))
print(f'added message {n} type GraphQL')
sleep(5)
if isinstance(message, PageEvent):
producer.send('quickwit-kafka', value=transform_pageevent(message))
print(f'added message {n} type PageEvent')
sleep(5)
def consumer_end():
consumer = create_consumer()
consumer.subscribe(topics=['quickwit-kafka'])
for msg in consumer:
print(msg)
def handle_args():
arguments = len(sys.argv)-1
argument_list = sys.argv[1:]
pos = 1
short_options = 'hm:'
long_options = ['help', 'method=']
try:
arguments, values = getopt.getopt(argument_list, short_options, long_options)
except getopt.error as err:
print(str(err))
sys.exit(2)
for arg, argv in arguments:
if arg in ('-h', '--help'):
print(""" Methods
--method, -m available methods: consumer, producer
--help, -h show help
""")
elif arg in ('-m', '--method'):
if argv == 'consumer':
consumer_end()
elif argv == 'producer':
consumer_producer_end()
else:
print('Method not found. Available methods: consumer, producer')
if __name__ == '__main__':
handle_args()

819
ee/quickwit/msgcodec/codec.py Normal file
View file

@ -0,0 +1,819 @@
import io
from typing import List
from msgcodec.messages import *
class Codec:
"""
Implements encode/decode primitives
"""
@staticmethod
def read_boolean(reader: io.BytesIO):
b = reader.read(1)
return b == 1
@staticmethod
def read_uint(reader: io.BytesIO):
"""
The ending "big" doesn't play any role here,
since we're dealing with data per one byte
"""
x = 0 # the result
s = 0 # the shift (our result is big-ending)
i = 0 # n of byte (max 9 for uint64)
while True:
b = reader.read(1)
if len(b) == 0:
raise IndexError('bytes out of range')
num = int.from_bytes(b, "big", signed=False)
# print(i, x)
if num < 0x80:
if i > 9 | i == 9 & num > 1:
raise OverflowError()
return int(x | num << s)
x |= (num & 0x7f) << s
s += 7
i += 1
@staticmethod
def read_int(reader: io.BytesIO) -> int:
"""
ux, err := ReadUint(reader)
x := int64(ux >> 1)
if err != nil {
return x, err
}
if ux&1 != 0 {
x = ^x
}
return x, err
"""
ux = Codec.read_uint(reader)
x = int(ux >> 1)
if ux & 1 != 0:
x = - x - 1
return x
@staticmethod
def read_string(reader: io.BytesIO) -> str:
length = Codec.read_uint(reader)
s = reader.read(length)
try:
return s.decode("utf-8", errors="replace").replace("\x00", "\uFFFD")
except UnicodeDecodeError:
return None
class MessageCodec(Codec):
def encode(self, m: Message) -> bytes:
...
def decode(self, b: bytes) -> Message:
reader = io.BytesIO(b)
return self.read_head_message(reader)
def decode_detailed(self, b: bytes) -> List[Message]:
reader = io.BytesIO(b)
messages_list = list()
while True:
try:
messages_list.append(self.read_head_message(reader))
except IndexError:
break
return messages_list
def read_head_message(self, reader: io.BytesIO) -> Message:
message_id = self.read_message_id(reader)
if message_id == 0:
return Timestamp(
timestamp=self.read_uint(reader)
)
if message_id == 1:
return SessionStart(
timestamp=self.read_uint(reader),
project_id=self.read_uint(reader),
tracker_version=self.read_string(reader),
rev_id=self.read_string(reader),
user_uuid=self.read_string(reader),
user_agent=self.read_string(reader),
user_os=self.read_string(reader),
user_os_version=self.read_string(reader),
user_browser=self.read_string(reader),
user_browser_version=self.read_string(reader),
user_device=self.read_string(reader),
user_device_type=self.read_string(reader),
user_device_memory_size=self.read_uint(reader),
user_device_heap_size=self.read_uint(reader),
user_country=self.read_string(reader)
)
if message_id == 2:
return SessionDisconnect(
timestamp=self.read_uint(reader)
)
if message_id == 3:
return SessionEnd(
timestamp=self.read_uint(reader)
)
if message_id == 4:
return SetPageLocation(
url=self.read_string(reader),
referrer=self.read_string(reader),
navigation_start=self.read_uint(reader)
)
if message_id == 5:
return SetViewportSize(
width=self.read_uint(reader),
height=self.read_uint(reader)
)
if message_id == 6:
return SetViewportScroll(
x=self.read_int(reader),
y=self.read_int(reader)
)
if message_id == 7:
return CreateDocument()
if message_id == 8:
return CreateElementNode(
id=self.read_uint(reader),
parent_id=self.read_uint(reader),
index=self.read_uint(reader),
tag=self.read_string(reader),
svg=self.read_boolean(reader),
)
if message_id == 9:
return CreateTextNode(
id=self.read_uint(reader),
parent_id=self.read_uint(reader),
index=self.read_uint(reader)
)
if message_id == 10:
return MoveNode(
id=self.read_uint(reader),
parent_id=self.read_uint(reader),
index=self.read_uint(reader)
)
if message_id == 11:
return RemoveNode(
id=self.read_uint(reader)
)
if message_id == 12:
return SetNodeAttribute(
id=self.read_uint(reader),
name=self.read_string(reader),
value=self.read_string(reader)
)
if message_id == 13:
return RemoveNodeAttribute(
id=self.read_uint(reader),
name=self.read_string(reader)
)
if message_id == 14:
return SetNodeData(
id=self.read_uint(reader),
data=self.read_string(reader)
)
if message_id == 15:
return SetCSSData(
id=self.read_uint(reader),
data=self.read_string(reader)
)
if message_id == 16:
return SetNodeScroll(
id=self.read_uint(reader),
x=self.read_int(reader),
y=self.read_int(reader),
)
if message_id == 17:
return SetInputTarget(
id=self.read_uint(reader),
label=self.read_string(reader)
)
if message_id == 18:
return SetInputValue(
id=self.read_uint(reader),
value=self.read_string(reader),
mask=self.read_int(reader),
)
if message_id == 19:
return SetInputChecked(
id=self.read_uint(reader),
checked=self.read_boolean(reader)
)
if message_id == 20:
return MouseMove(
x=self.read_uint(reader),
y=self.read_uint(reader)
)
if message_id == 21:
return MouseClickDepricated(
id=self.read_uint(reader),
hesitation_time=self.read_uint(reader),
label=self.read_string(reader)
)
if message_id == 22:
return ConsoleLog(
level=self.read_string(reader),
value=self.read_string(reader)
)
if message_id == 23:
return PageLoadTiming(
request_start=self.read_uint(reader),
response_start=self.read_uint(reader),
response_end=self.read_uint(reader),
dom_content_loaded_event_start=self.read_uint(reader),
dom_content_loaded_event_end=self.read_uint(reader),
load_event_start=self.read_uint(reader),
load_event_end=self.read_uint(reader),
first_paint=self.read_uint(reader),
first_contentful_paint=self.read_uint(reader)
)
if message_id == 24:
return PageRenderTiming(
speed_index=self.read_uint(reader),
visually_complete=self.read_uint(reader),
time_to_interactive=self.read_uint(reader),
)
if message_id == 25:
return JSException(
name=self.read_string(reader),
message=self.read_string(reader),
payload=self.read_string(reader)
)
if message_id == 26:
return RawErrorEvent(
timestamp=self.read_uint(reader),
source=self.read_string(reader),
name=self.read_string(reader),
message=self.read_string(reader),
payload=self.read_string(reader)
)
if message_id == 27:
return RawCustomEvent(
name=self.read_string(reader),
payload=self.read_string(reader)
)
if message_id == 28:
return UserID(
id=self.read_string(reader)
)
if message_id == 29:
return UserAnonymousID(
id=self.read_string(reader)
)
if message_id == 30:
return Metadata(
key=self.read_string(reader),
value=self.read_string(reader)
)
if message_id == 31:
return PageEvent(
message_id=self.read_uint(reader),
timestamp=self.read_uint(reader),
url=self.read_string(reader),
referrer=self.read_string(reader),
loaded=self.read_boolean(reader),
request_start=self.read_uint(reader),
response_start=self.read_uint(reader),
response_end=self.read_uint(reader),
dom_content_loaded_event_start=self.read_uint(reader),
dom_content_loaded_event_end=self.read_uint(reader),
load_event_start=self.read_uint(reader),
load_event_end=self.read_uint(reader),
first_paint=self.read_uint(reader),
first_contentful_paint=self.read_uint(reader),
speed_index=self.read_uint(reader),
visually_complete=self.read_uint(reader),
time_to_interactive=self.read_uint(reader)
)
if message_id == 32:
return InputEvent(
message_id=self.read_uint(reader),
timestamp=self.read_uint(reader),
value=self.read_string(reader),
value_masked=self.read_boolean(reader),
label=self.read_string(reader),
)
if message_id == 33:
return ClickEvent(
message_id=self.read_uint(reader),
timestamp=self.read_uint(reader),
hesitation_time=self.read_uint(reader),
label=self.read_string(reader)
)
if message_id == 34:
return ErrorEvent(
message_id=self.read_uint(reader),
timestamp=self.read_uint(reader),
source=self.read_string(reader),
name=self.read_string(reader),
message=self.read_string(reader),
payload=self.read_string(reader)
)
if message_id == 35:
message_id = self.read_uint(reader)
ts = self.read_uint(reader)
if ts > 9999999999999:
ts = None
return ResourceEvent(
message_id=message_id,
timestamp=ts,
duration=self.read_uint(reader),
ttfb=self.read_uint(reader),
header_size=self.read_uint(reader),
encoded_body_size=self.read_uint(reader),
decoded_body_size=self.read_uint(reader),
url=self.read_string(reader),
type=self.read_string(reader),
success=self.read_boolean(reader),
method=self.read_string(reader),
status=self.read_uint(reader)
)
if message_id == 36:
return CustomEvent(
message_id=self.read_uint(reader),
timestamp=self.read_uint(reader),
name=self.read_string(reader),
payload=self.read_string(reader)
)
if message_id == 37:
return CSSInsertRule(
id=self.read_uint(reader),
rule=self.read_string(reader),
index=self.read_uint(reader)
)
if message_id == 38:
return CSSDeleteRule(
id=self.read_uint(reader),
index=self.read_uint(reader)
)
if message_id == 39:
return Fetch(
method=self.read_string(reader),
url=self.read_string(reader),
request=self.read_string(reader),
response=self.read_string(reader),
status=self.read_uint(reader),
timestamp=self.read_uint(reader),
duration=self.read_uint(reader)
)
if message_id == 40:
return Profiler(
name=self.read_string(reader),
duration=self.read_uint(reader),
args=self.read_string(reader),
result=self.read_string(reader)
)
if message_id == 41:
return OTable(
key=self.read_string(reader),
value=self.read_string(reader)
)
if message_id == 42:
return StateAction(
type=self.read_string(reader)
)
if message_id == 43:
return StateActionEvent(
message_id=self.read_uint(reader),
timestamp=self.read_uint(reader),
type=self.read_string(reader)
)
if message_id == 44:
return Redux(
action=self.read_string(reader),
state=self.read_string(reader),
duration=self.read_uint(reader)
)
if message_id == 45:
return Vuex(
mutation=self.read_string(reader),
state=self.read_string(reader),
)
if message_id == 46:
return MobX(
type=self.read_string(reader),
payload=self.read_string(reader),
)
if message_id == 47:
return NgRx(
action=self.read_string(reader),
state=self.read_string(reader),
duration=self.read_uint(reader)
)
if message_id == 48:
return GraphQL(
operation_kind=self.read_string(reader),
operation_name=self.read_string(reader),
variables=self.read_string(reader),
response=self.read_string(reader)
)
if message_id == 49:
return PerformanceTrack(
frames=self.read_int(reader),
ticks=self.read_int(reader),
total_js_heap_size=self.read_uint(reader),
used_js_heap_size=self.read_uint(reader)
)
if message_id == 50:
return GraphQLEvent(
message_id=self.read_uint(reader),
timestamp=self.read_uint(reader),
name=self.read_string(reader)
)
if message_id == 51:
return FetchEvent(
message_id=self.read_uint(reader),
timestamp=self.read_uint(reader),
method=self.read_string(reader),
url=self.read_string(reader),
request=self.read_string(reader),
response=self.read_string(reader),
status=self.read_uint(reader),
duration=self.read_uint(reader)
)
if message_id == 52:
return DomDrop(
timestamp=self.read_uint(reader)
)
if message_id == 53:
return ResourceTiming(
timestamp=self.read_uint(reader),
duration=self.read_uint(reader),
ttfb=self.read_uint(reader),
header_size=self.read_uint(reader),
encoded_body_size=self.read_uint(reader),
decoded_body_size=self.read_uint(reader),
url=self.read_string(reader),
initiator=self.read_string(reader)
)
if message_id == 54:
return ConnectionInformation(
downlink=self.read_uint(reader),
type=self.read_string(reader)
)
if message_id == 55:
return SetPageVisibility(
hidden=self.read_boolean(reader)
)
if message_id == 56:
return PerformanceTrackAggr(
timestamp_start=self.read_uint(reader),
timestamp_end=self.read_uint(reader),
min_fps=self.read_uint(reader),
avg_fps=self.read_uint(reader),
max_fps=self.read_uint(reader),
min_cpu=self.read_uint(reader),
avg_cpu=self.read_uint(reader),
max_cpu=self.read_uint(reader),
min_total_js_heap_size=self.read_uint(reader),
avg_total_js_heap_size=self.read_uint(reader),
max_total_js_heap_size=self.read_uint(reader),
min_used_js_heap_size=self.read_uint(reader),
avg_used_js_heap_size=self.read_uint(reader),
max_used_js_heap_size=self.read_uint(reader)
)
if message_id == 59:
return LongTask(
timestamp=self.read_uint(reader),
duration=self.read_uint(reader),
context=self.read_uint(reader),
container_type=self.read_uint(reader),
container_src=self.read_string(reader),
container_id=self.read_string(reader),
container_name=self.read_string(reader)
)
if message_id == 60:
return SetNodeURLBasedAttribute(
id=self.read_uint(reader),
name=self.read_string(reader),
value=self.read_string(reader),
base_url=self.read_string(reader)
)
if message_id == 61:
return SetStyleData(
id=self.read_uint(reader),
data=self.read_string(reader),
base_url=self.read_string(reader)
)
if message_id == 62:
return IssueEvent(
message_id=self.read_uint(reader),
timestamp=self.read_uint(reader),
type=self.read_string(reader),
context_string=self.read_string(reader),
context=self.read_string(reader),
payload=self.read_string(reader)
)
if message_id == 63:
return TechnicalInfo(
type=self.read_string(reader),
value=self.read_string(reader)
)
if message_id == 64:
return CustomIssue(
name=self.read_string(reader),
payload=self.read_string(reader)
)
if message_id == 65:
return PageClose()
if message_id == 66:
return AssetCache(
url=self.read_string(reader)
)
if message_id == 67:
return CSSInsertRuleURLBased(
id=self.read_uint(reader),
rule=self.read_string(reader),
index=self.read_uint(reader),
base_url=self.read_string(reader)
)
if message_id == 69:
return MouseClick(
id=self.read_uint(reader),
hesitation_time=self.read_uint(reader),
label=self.read_string(reader),
selector=self.read_string(reader)
)
if message_id == 70:
return CreateIFrameDocument(
frame_id=self.read_uint(reader),
id=self.read_uint(reader)
)
if message_id == 80:
return BatchMeta(
page_no=self.read_uint(reader),
first_index=self.read_uint(reader),
timestamp=self.read_int(reader)
)
if message_id == 90:
return IOSSessionStart(
timestamp=self.read_uint(reader),
project_id=self.read_uint(reader),
tracker_version=self.read_string(reader),
rev_id=self.read_string(reader),
user_uuid=self.read_string(reader),
user_os=self.read_string(reader),
user_os_version=self.read_string(reader),
user_device=self.read_string(reader),
user_device_type=self.read_string(reader),
user_country=self.read_string(reader)
)
if message_id == 91:
return IOSSessionEnd(
timestamp=self.read_uint(reader)
)
if message_id == 92:
return IOSMetadata(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
key=self.read_string(reader),
value=self.read_string(reader)
)
if message_id == 93:
return IOSCustomEvent(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
name=self.read_string(reader),
payload=self.read_string(reader)
)
if message_id == 94:
return IOSUserID(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
value=self.read_string(reader)
)
if message_id == 95:
return IOSUserAnonymousID(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
value=self.read_string(reader)
)
if message_id == 96:
return IOSScreenChanges(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
x=self.read_uint(reader),
y=self.read_uint(reader),
width=self.read_uint(reader),
height=self.read_uint(reader)
)
if message_id == 97:
return IOSCrash(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
name=self.read_string(reader),
reason=self.read_string(reader),
stacktrace=self.read_string(reader)
)
if message_id == 98:
return IOSScreenEnter(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
title=self.read_string(reader),
view_name=self.read_string(reader)
)
if message_id == 99:
return IOSScreenLeave(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
title=self.read_string(reader),
view_name=self.read_string(reader)
)
if message_id == 100:
return IOSClickEvent(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
label=self.read_string(reader),
x=self.read_uint(reader),
y=self.read_uint(reader)
)
if message_id == 101:
return IOSInputEvent(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
value=self.read_string(reader),
value_masked=self.read_boolean(reader),
label=self.read_string(reader)
)
if message_id == 102:
return IOSPreformanceEvent(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
name=self.read_string(reader),
value=self.read_uint(reader)
)
if message_id == 103:
return IOSLog(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
severity=self.read_string(reader),
content=self.read_string(reader)
)
if message_id == 104:
return IOSInternalError(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
content=self.read_string(reader)
)
if message_id == 105:
return IOSNetworkCall(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
duration=self.read_uint(reader),
headers=self.read_string(reader),
body=self.read_string(reader),
url=self.read_string(reader),
success=self.read_boolean(reader),
method=self.read_string(reader),
status=self.read_uint(reader)
)
if message_id == 107:
return IOSBatchMeta(
timestamp=self.read_uint(reader),
length=self.read_uint(reader),
first_index=self.read_uint(reader)
)
if message_id == 110:
return IOSPerformanceAggregated(
timestamp_start=self.read_uint(reader),
timestamp_end=self.read_uint(reader),
min_fps=self.read_uint(reader),
avg_fps=self.read_uint(reader),
max_fps=self.read_uint(reader),
min_cpu=self.read_uint(reader),
avg_cpu=self.read_uint(reader),
max_cpu=self.read_uint(reader),
min_memory=self.read_uint(reader),
avg_memory=self.read_uint(reader),
max_memory=self.read_uint(reader),
min_battery=self.read_uint(reader),
avg_battery=self.read_uint(reader),
max_battery=self.read_uint(reader)
)
if message_id == 111:
return IOSIssueEvent(
timestamp=self.read_uint(reader),
type=self.read_string(reader),
context_string=self.read_string(reader),
context=self.read_string(reader),
payload=self.read_string(reader)
)
def read_message_id(self, reader: io.BytesIO) -> int:
"""
Read and return the first byte where the message id is encoded
"""
id_ = self.read_uint(reader)
return id_
@staticmethod
def check_message_id(b: bytes) -> int:
"""
todo: make it static and without reader. It's just the first byte
Read and return the first byte where the message id is encoded
"""
reader = io.BytesIO(b)
id_ = Codec.read_uint(reader)
return id_
@staticmethod
def decode_key(b) -> int:
"""
Decode the message key (encoded with little endian)
"""
try:
decoded = int.from_bytes(b, "little", signed=False)
except Exception as e:
raise UnicodeDecodeError(f"Error while decoding message key (SessionID) from {b}\n{e}")
return decoded

919
ee/quickwit/msgcodec/messages.py Normal file
View file

@ -0,0 +1,919 @@
"""
Representations of Kafka messages
"""
from abc import ABC
class Message(ABC):
pass
class Timestamp(Message):
__id__ = 0
def __init__(self, timestamp):
self.timestamp = timestamp
class SessionStart(Message):
__id__ = 1
def __init__(self, timestamp, project_id, tracker_version, rev_id, user_uuid,
user_agent, user_os, user_os_version, user_browser, user_browser_version,
user_device, user_device_type, user_device_memory_size, user_device_heap_size,
user_country):
self.timestamp = timestamp
self.project_id = project_id
self.tracker_version = tracker_version
self.rev_id = rev_id
self.user_uuid = user_uuid
self.user_agent = user_agent
self.user_os = user_os
self.user_os_version = user_os_version
self.user_browser = user_browser
self.user_browser_version = user_browser_version
self.user_device = user_device
self.user_device_type = user_device_type
self.user_device_memory_size = user_device_memory_size
self.user_device_heap_size = user_device_heap_size
self.user_country = user_country
class SessionDisconnect(Message):
__id__ = 2
def __init__(self, timestamp):
self.timestamp = timestamp
class SessionEnd(Message):
__id__ = 3
__name__ = 'SessionEnd'
def __init__(self, timestamp):
self.timestamp = timestamp
class SetPageLocation(Message):
__id__ = 4
def __init__(self, url, referrer, navigation_start):
self.url = url
self.referrer = referrer
self.navigation_start = navigation_start
class SetViewportSize(Message):
__id__ = 5
def __init__(self, width, height):
self.width = width
self.height = height
class SetViewportScroll(Message):
__id__ = 6
def __init__(self, x, y):
self.x = x
self.y = y
class CreateDocument(Message):
__id__ = 7
class CreateElementNode(Message):
__id__ = 8
def __init__(self, id, parent_id, index, tag, svg):
self.id = id
self.parent_id = parent_id,
self.index = index
self.tag = tag
self.svg = svg
class CreateTextNode(Message):
__id__ = 9
def __init__(self, id, parent_id, index):
self.id = id
self.parent_id = parent_id
self.index = index
class MoveNode(Message):
__id__ = 10
def __init__(self, id, parent_id, index):
self.id = id
self.parent_id = parent_id
self.index = index
class RemoveNode(Message):
__id__ = 11
def __init__(self, id):
self.id = id
class SetNodeAttribute(Message):
__id__ = 12
def __init__(self, id, name: str, value: str):
self.id = id
self.name = name
self.value = value
class RemoveNodeAttribute(Message):
__id__ = 13
def __init__(self, id, name: str):
self.id = id
self.name = name
class SetNodeData(Message):
__id__ = 14
def __init__(self, id, data: str):
self.id = id
self.data = data
class SetCSSData(Message):
__id__ = 15
def __init__(self, id, data: str):
self.id = id
self.data = data
class SetNodeScroll(Message):
__id__ = 16
def __init__(self, id, x: int, y: int):
self.id = id
self.x = x
self.y = y
class SetInputTarget(Message):
__id__ = 17
def __init__(self, id, label: str):
self.id = id
self.label = label
class SetInputValue(Message):
__id__ = 18
def __init__(self, id, value: str, mask: int):
self.id = id
self.value = value
self.mask = mask
class SetInputChecked(Message):
__id__ = 19
def __init__(self, id, checked: bool):
self.id = id
self.checked = checked
class MouseMove(Message):
__id__ = 20
def __init__(self, x, y):
self.x = x
self.y = y
class MouseClickDepricated(Message):
__id__ = 21
def __init__(self, id, hesitation_time, label: str):
self.id = id
self.hesitation_time = hesitation_time
self.label = label
class ConsoleLog(Message):
__id__ = 22
def __init__(self, level: str, value: str):
self.level = level
self.value = value
class PageLoadTiming(Message):
__id__ = 23
def __init__(self, request_start, response_start, response_end, dom_content_loaded_event_start,
dom_content_loaded_event_end, load_event_start, load_event_end,
first_paint, first_contentful_paint):
self.request_start = request_start
self.response_start = response_start
self.response_end = response_end
self.dom_content_loaded_event_start = dom_content_loaded_event_start
self.dom_content_loaded_event_end = dom_content_loaded_event_end
self.load_event_start = load_event_start
self.load_event_end = load_event_end
self.first_paint = first_paint
self.first_contentful_paint = first_contentful_paint
class PageRenderTiming(Message):
__id__ = 24
def __init__(self, speed_index, visually_complete, time_to_interactive):
self.speed_index = speed_index
self.visually_complete = visually_complete
self.time_to_interactive = time_to_interactive
class JSException(Message):
__id__ = 25
def __init__(self, name: str, message: str, payload: str):
self.name = name
self.message = message
self.payload = payload
class RawErrorEvent(Message):
__id__ = 26
def __init__(self, timestamp, source: str, name: str, message: str,
payload: str):
self.timestamp = timestamp
self.source = source
self.name = name
self.message = message
self.payload = payload
class RawCustomEvent(Message):
__id__ = 27
def __init__(self, name: str, payload: str):
self.name = name
self.payload = payload
class UserID(Message):
__id__ = 28
def __init__(self, id: str):
self.id = id
class UserAnonymousID(Message):
__id__ = 29
def __init__(self, id: str):
self.id = id
class Metadata(Message):
__id__ = 30
def __init__(self, key: str, value: str):
self.key = key
self.value = value
class PerformanceTrack(Message):
__id__ = 49
def __init__(self, frames: int, ticks: int, total_js_heap_size,
used_js_heap_size):
self.frames = frames
self.ticks = ticks
self.total_js_heap_size = total_js_heap_size
self.used_js_heap_size = used_js_heap_size
class PageEvent(Message):
__id__ = 31
def __init__(self, message_id, timestamp, url: str, referrer: str,
loaded: bool, request_start, response_start, response_end,
dom_content_loaded_event_start, dom_content_loaded_event_end,
load_event_start, load_event_end, first_paint, first_contentful_paint,
speed_index, visually_complete, time_to_interactive):
self.message_id = message_id
self.timestamp = timestamp
self.url = url
self.referrer = referrer
self.loaded = loaded
self.request_start = request_start
self.response_start = response_start
self.response_end = response_end
self.dom_content_loaded_event_start = dom_content_loaded_event_start
self.dom_content_loaded_event_end = dom_content_loaded_event_end
self.load_event_start = load_event_start
self.load_event_end = load_event_end
self.first_paint = first_paint
self.first_contentful_paint = first_contentful_paint
self.speed_index = speed_index
self.visually_complete = visually_complete
self.time_to_interactive = time_to_interactive
class InputEvent(Message):
__id__ = 32
def __init__(self, message_id, timestamp, value: str, value_masked: bool, label: str):
self.message_id = message_id
self.timestamp = timestamp
self.value = value
self.value_masked = value_masked
self.label = label
class ClickEvent(Message):
__id__ = 33
def __init__(self, message_id, timestamp, hesitation_time, label: str):
self.message_id = message_id
self.timestamp = timestamp
self.hesitation_time = hesitation_time
self.label = label
class ErrorEvent(Message):
__id__ = 34
def __init__(self, message_id, timestamp, source: str, name: str, message: str,
payload: str):
self.message_id = message_id
self.timestamp = timestamp
self.source = source
self.name = name
self.message = message
self.payload = payload
class ResourceEvent(Message):
__id__ = 35
def __init__(self, message_id, timestamp, duration, ttfb, header_size, encoded_body_size,
decoded_body_size, url: str, type: str, success: bool, method: str, status):
self.message_id = message_id
self.timestamp = timestamp
self.duration = duration
self.ttfb = ttfb
self.header_size = header_size
self.encoded_body_size = encoded_body_size
self.decoded_body_size = decoded_body_size
self.url = url
self.type = type
self.success = success
self.method = method
self.status = status
class CustomEvent(Message):
__id__ = 36
def __init__(self, message_id, timestamp, name: str, payload: str):
self.message_id = message_id
self.timestamp = timestamp
self.name = name
self.payload = payload
class CSSInsertRule(Message):
__id__ = 37
def __init__(self, id, rule: str, index):
self.id = id
self.rule = rule
self.index = index
class CSSDeleteRule(Message):
__id__ = 38
def __init__(self, id, index):
self.id = id
self.index = index
class Fetch(Message):
__id__ = 39
def __init__(self, method: str, url: str, request: str, response: str, status,
timestamp, duration):
self.method = method
self.url = url
self.request = request
self.response = response
self.status = status
self.timestamp = timestamp
self.duration = duration
class Profiler(Message):
__id__ = 40
def __init__(self, name: str, duration, args: str, result: str):
self.name = name
self.duration = duration
self.args = args
self.result = result
class OTable(Message):
__id__ = 41
def __init__(self, key: str, value: str):
self.key = key
self.value = value
class StateAction(Message):
__id__ = 42
def __init__(self, type: str):
self.type = type
class StateActionEvent(Message):
__id__ = 43
def __init__(self, message_id, timestamp, type: str):
self.message_id = message_id
self.timestamp = timestamp
self.type = type
class Redux(Message):
__id__ = 44
def __init__(self, action: str, state: str, duration):
self.action = action
self.state = state
self.duration = duration
class Vuex(Message):
__id__ = 45
def __init__(self, mutation: str, state: str):
self.mutation = mutation
self.state = state
class MobX(Message):
__id__ = 46
def __init__(self, type: str, payload: str):
self.type = type
self.payload = payload
class NgRx(Message):
__id__ = 47
def __init__(self, action: str, state: str, duration):
self.action = action
self.state = state
self.duration = duration
class GraphQL(Message):
__id__ = 48
def __init__(self, operation_kind: str, operation_name: str,
variables: str, response: str):
self.operation_kind = operation_kind
self.operation_name = operation_name
self.variables = variables
self.response = response
class PerformanceTrack(Message):
__id__ = 49
def __init__(self, frames: int, ticks: int,
total_js_heap_size, used_js_heap_size):
self.frames = frames
self.ticks = ticks
self.total_js_heap_size = total_js_heap_size
self.used_js_heap_size = used_js_heap_size
class GraphQLEvent(Message):
__id__ = 50
def __init__(self, message_id, timestamp, name: str):
self.message_id = message_id
self.timestamp = timestamp
self.name = name
class FetchEvent(Message):
__id__ = 51
def __init__(self, message_id, timestamp, method: str, url, request, response: str,
status, duration):
self.message_id = message_id
self.timestamp = timestamp
self.method = method
self.url = url
self.request = request
self.response = response
self.status = status
self.duration = duration
class DomDrop(Message):
__id__ = 52
def __init__(self, timestamp):
self.timestamp = timestamp
class ResourceTiming(Message):
__id__ = 53
def __init__(self, timestamp, duration, ttfb, header_size, encoded_body_size,
decoded_body_size, url, initiator):
self.timestamp = timestamp
self.duration = duration
self.ttfb = ttfb
self.header_size = header_size
self.encoded_body_size = encoded_body_size
self.decoded_body_size = decoded_body_size
self.url = url
self.initiator = initiator
class ConnectionInformation(Message):
__id__ = 54
def __init__(self, downlink, type: str):
self.downlink = downlink
self.type = type
class SetPageVisibility(Message):
__id__ = 55
def __init__(self, hidden: bool):
self.hidden = hidden
class PerformanceTrackAggr(Message):
__id__ = 56
def __init__(self, timestamp_start, timestamp_end, min_fps, avg_fps,
max_fps, min_cpu, avg_cpu, max_cpu,
min_total_js_heap_size, avg_total_js_heap_size,
max_total_js_heap_size, min_used_js_heap_size,
avg_used_js_heap_size, max_used_js_heap_size
):
self.timestamp_start = timestamp_start
self.timestamp_end = timestamp_end
self.min_fps = min_fps
self.avg_fps = avg_fps
self.max_fps = max_fps
self.min_cpu = min_cpu
self.avg_cpu = avg_cpu
self.max_cpu = max_cpu
self.min_total_js_heap_size = min_total_js_heap_size
self.avg_total_js_heap_size = avg_total_js_heap_size
self.max_total_js_heap_size = max_total_js_heap_size
self.min_used_js_heap_size = min_used_js_heap_size
self.avg_used_js_heap_size = avg_used_js_heap_size
self.max_used_js_heap_size = max_used_js_heap_size
class LongTask(Message):
__id__ = 59
def __init__(self, timestamp, duration, context, container_type, container_src: str,
container_id: str, container_name: str):
self.timestamp = timestamp
self.duration = duration
self.context = context
self.container_type = container_type
self.container_src = container_src
self.container_id = container_id
self.container_name = container_name
class SetNodeURLBasedAttribute(Message):
__id__ = 60
def __init__(self, id, name: str, value: str, base_url: str):
self.id = id
self.name = name
self.value = value
self.base_url = base_url
class SetStyleData(Message):
__id__ = 61
def __init__(self, id, data: str, base_url: str):
self.id = id
self.data = data
self.base_url = base_url
class IssueEvent(Message):
__id__ = 62
def __init__(self, message_id, timestamp, type: str, context_string: str,
context: str, payload: str):
self.message_id = message_id
self.timestamp = timestamp
self.type = type
self.context_string = context_string
self.context = context
self.payload = payload
class TechnicalInfo(Message):
__id__ = 63
def __init__(self, type: str, value: str):
self.type = type
self.value = value
class CustomIssue(Message):
__id__ = 64
def __init__(self, name: str, payload: str):
self.name = name
self.payload = payload
class PageClose(Message):
__id__ = 65
class AssetCache(Message):
__id__ = 66
def __init__(self, url):
self.url = url
class CSSInsertRuleURLBased(Message):
__id__ = 67
def __init__(self, id, rule, index, base_url):
self.id = id
self.rule = rule
self.index = index
self.base_url = base_url
class MouseClick(Message):
__id__ = 69
def __init__(self, id, hesitation_time, label: str, selector):
self.id = id
self.hesitation_time = hesitation_time
self.label = label
self.selector = selector
class CreateIFrameDocument(Message):
__id__ = 70
def __init__(self, frame_id, id):
self.frame_id = frame_id
self.id = id
class BatchMeta(Message):
__id__ = 80
def __init__(self, page_no, first_index, timestamp):
self.page_no = page_no
self.first_index = first_index
self.timestamp = timestamp
class IOSSessionStart(Message):
__id__ = 90
def __init__(self, timestamp, project_id, tracker_version: str,
rev_id: str, user_uuid: str, user_os: str, user_os_version: str,
user_device: str, user_device_type: str, user_country: str):
self.timestamp = timestamp
self.project_id = project_id
self.tracker_version = tracker_version
self.rev_id = rev_id
self.user_uuid = user_uuid
self.user_os = user_os
self.user_os_version = user_os_version
self.user_device = user_device
self.user_device_type = user_device_type
self.user_country = user_country
class IOSSessionEnd(Message):
__id__ = 91
def __init__(self, timestamp):
self.timestamp = timestamp
class IOSMetadata(Message):
__id__ = 92
def __init__(self, timestamp, length, key: str, value: str):
self.timestamp = timestamp
self.length = length
self.key = key
self.value = value
class IOSCustomEvent(Message):
__id__ = 93
def __init__(self, timestamp, length, name: str, payload: str):
self.timestamp = timestamp
self.length = length
self.name = name
self.payload = payload
class IOSUserID(Message):
__id__ = 94
def __init__(self, timestamp, length, value: str):
self.timestamp = timestamp
self.length = length
self.value = value
class IOSUserAnonymousID(Message):
__id__ = 95
def __init__(self, timestamp, length, value: str):
self.timestamp = timestamp
self.length = length
self.value = value
class IOSScreenChanges(Message):
__id__ = 96
def __init__(self, timestamp, length, x, y, width, height):
self.timestamp = timestamp
self.length = length
self.x = x
self.y = y
self.width = width
self.height = height
class IOSCrash(Message):
__id__ = 97
def __init__(self, timestamp, length, name: str, reason: str, stacktrace):
self.timestamp = timestamp
self.length = length
self.name = name
self.reason = reason
self.stacktrace = stacktrace
class IOSScreenEnter(Message):
__id__ = 98
def __init__(self, timestamp, length, title, view_name):
self.timestamp = timestamp
self.length = length
self.title = title
self.view_name = view_name
class IOSScreenLeave(Message):
__id__ = 99
def __init__(self, timestamp, length, title: str, view_name: str):
self.timestamp = timestamp
self.length = length
self.title = title
self.view_name = view_name
class IOSClickEvent(Message):
__id__ = 100
def __init__(self, timestamp, length, label, x, y):
self.timestamp = timestamp
self.length = length
self.label = label
self.x = x
self.y = y
class IOSInputEvent(Message):
__id__ = 101
def __init__(self, timestamp, length, value: str, value_masked: bool, label: str):
self.timestamp = timestamp
self.length = length
self.value_masked = value_masked
self.label = label
class IOSPerformanceEvent(Message):
__id__ = 102
def __init__(self, timestamp, length, name: str, value):
self.timestamp = timestamp
self.length = length
self.name = name
self.value = value
class IOSLog(Message):
__id__ = 103
def __init__(self, timestamp, length, severity: str, content: str):
self.timestamp = timestamp
self.length = length
self.severity = severity
self.content = content
class IOSInternalError(Message):
__id__ = 104
def __init__(self, timestamp, length, content: str):
self.timestamp = timestamp
self.length = length
self.content = content
class IOSNetworkCall(Message):
__id__ = 105
def __init__(self, timestamp, length, duration, headers, body, url, success: bool, method: str, status):
self.timestamp = timestamp
self.length = length
self.duration = duration
self.headers = headers
self.body = body
self.url = url
self.success = success
self.method = method
self.status = status
class IOSBatchMeta(Message):
__id__ = 107
def __init__(self, timestamp, length, first_index):
self.timestamp = timestamp
self.length = length
self.first_index = first_index
class IOSPerformanceAggregated(Message):
__id__ = 110
def __init__(self, timestamp_start, timestamp_end, min_fps, avg_fps,
max_fps, min_cpu, avg_cpu, max_cpu,
min_memory, avg_memory, max_memory,
min_battery, avg_battery, max_battery
):
self.timestamp_start = timestamp_start
self.timestamp_end = timestamp_end
self.min_fps = min_fps
self.avg_fps = avg_fps
self.max_fps = max_fps
self.min_cpu = min_cpu
self.avg_cpu = avg_cpu
self.max_cpu = max_cpu
self.min_memory = min_memory
self.avg_memory = avg_memory
self.max_memory = max_memory
self.min_battery = min_battery
self.avg_battery = avg_battery
self.max_battery = max_battery
class IOSIssueEvent(Message):
__id__ = 111
def __init__(self, timestamp, type: str, context_string: str, context: str, payload: str):
self.timestamp = timestamp
self.type = type
self.context_string = context_string
self.context = context
self.payload = payload

1
ee/quickwit/requirements.txt Normal file
View file

@ -0,0 +1 @@
kafka-python

1
ee/quickwit/run_quickwit.sh Normal file
View file

@ -0,0 +1 @@
docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config-listen.yaml:/quickwit/s3-config-listen.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region -p 127.0.0.1:7280:7280 quickwit/quickwit run --config s3-config-listen.yaml

6
ee/quickwit/s3-config-listen.yaml Normal file
View file

@ -0,0 +1,6 @@
## In order to save data into S3
# metastore also accepts s3://{bucket/path}#pooling_interval={seconds}s
version: 0
metastore_uri: s3://quickwit/quickwit-indexes
default_index_root_uri: s3://quickwit/quickwit-indexes
listen_address: 0.0.0.0

5
ee/quickwit/s3-config.yaml Normal file
View file

@ -0,0 +1,5 @@
## In order to save data into S3
# metastore also accepts s3://{bucket/path}#pooling_interval={seconds}s
version: 0
metastore_uri: s3://quickwit/quickwit-indexes
default_index_root_uri: s3://quickwit/quickwit-indexes

12
ee/quickwit/source-fetch.yaml Normal file
View file

@ -0,0 +1,12 @@
#
# Source config file.
#
source_id: fetch-kafka
source_type: kafka
params:
topic: quickwit
client_params:
bootstrap.servers: localhost:9092
group.id: fetch-consumer
security.protocol: SSL

12
ee/quickwit/source-graphql.yaml Normal file
View file

@ -0,0 +1,12 @@
#
# Source config file.
#
source_id: graphql-kafka
source_type: kafka
params:
topic: quickwit
client_params:
bootstrap.servers: localhost:9092
group.id: graphql-consumer
security.protocol: SSL

12
ee/quickwit/source-pageevent.yaml Normal file
View file

@ -0,0 +1,12 @@
#
# Source config file.
#
source_id: pageevent-kafka
source_type: kafka
params:
topic: quickwit
client_params:
bootstrap.servers: localhost:9092
group.id: pageevent-consumer
security.protocol: SSL