diff --git a/ee/quickwit/consumer.py b/ee/quickwit/consumer.py index be83d00e9..563dde634 100644 --- a/ee/quickwit/consumer.py +++ b/ee/quickwit/consumer.py @@ -92,9 +92,9 @@ class KafkaFilter(): "enable.auto.commit": False }) self.consumer.subscribe([topic]) - self.queues = {'fetchevent': Queue(self.fetchevent_maxsize), - 'graphql': Queue(self.graphql_maxsize), - 'pageevent': Queue(self.pageevent_maxsize) + self.queues = {'Fetchevent': Queue(self.fetchevent_maxsize), + 'Graphql': Queue(self.graphql_maxsize), + 'Pageevent': Queue(self.pageevent_maxsize) } async def add_to_queue(self, message): diff --git a/ee/quickwit/entrypoint.sh b/ee/quickwit/entrypoint.sh index 284faf3c7..37161a010 100755 --- a/ee/quickwit/entrypoint.sh +++ b/ee/quickwit/entrypoint.sh @@ -9,4 +9,13 @@ find /quickwit/ -type f -name "*.yaml" -exec sed -i "s/{{QUICKWIT_TOPIC}}/${QUIC find /quickwit/ -type f -name "*.yaml" -exec sed -i "s/{{QUICKWIT_PORT}}/${QUICKWIT_PORT}/g" {} \; find /quickwit/ -type f -name "*.yaml" -exec sed -i "s#{{data_dir_path}}#${data_dir_path}#g" {} \; -./quickwit_start_task.sh & ./setup_indexes_and_worker.sh && fg +./quickwit_start_task.sh & pid1=$! +sleep 120 +echo "Creating indexes.." +quickwit index create --index-config index-config-fetch.yaml +quickwit index create --index-config index-config-graphql.yaml +quickwit index create --index-config index-config-pageevent.yaml +echo "Running kafka reader.." +python3 -u consumer.py & pid2=$! +wait $pid1 $pid2 + diff --git a/ee/quickwit/index-config-fetch.yaml b/ee/quickwit/index-config-fetch.yaml index c4a3dfebb..03751fab2 100644 --- a/ee/quickwit/index-config-fetch.yaml +++ b/ee/quickwit/index-config-fetch.yaml @@ -4,7 +4,7 @@ version: 0.6 -index_id: "fetchevent" +index_id: "Fetchevent" index_uri: "s3://openreplay-quickwit/quickwit-indexes/fetchevent" doc_mapping: @@ -63,3 +63,9 @@ search_settings: retention: period: 30 days schedule: hourly + +indexing_settings: + merge_policy: + type: "stable_log" + min_level_num_docs: 40000 + maturation_period: 12h diff --git a/ee/quickwit/index-config-graphql.yaml b/ee/quickwit/index-config-graphql.yaml index c71ce2dbf..c838ba13d 100644 --- a/ee/quickwit/index-config-graphql.yaml +++ b/ee/quickwit/index-config-graphql.yaml @@ -4,7 +4,7 @@ version: 0.6 -index_id: "graphql" +index_id: "Graphql" index_uri: "s3://openreplay-quickwit/quickwit-indexes/graphql" doc_mapping: @@ -50,3 +50,9 @@ search_settings: retention: period: 30 days schedule: hourly + +indexing_settings: + merge_policy: + type: "stable_log" + min_level_num_docs: 40000 + maturation_period: 12h diff --git a/ee/quickwit/index-config-pageevent.yaml b/ee/quickwit/index-config-pageevent.yaml index 870efff29..578a41510 100644 --- a/ee/quickwit/index-config-pageevent.yaml +++ b/ee/quickwit/index-config-pageevent.yaml @@ -4,7 +4,7 @@ version: 0.6 -index_id: "pageevent" +index_id: "Pageevent" index_uri: "s3://openreplay-quickwit/quickwit-indexes/pageevent" doc_mapping: @@ -84,3 +84,9 @@ search_settings: retention: period: 30 days schedule: hourly + +indexing_settings: + merge_policy: + type: "stable_log" + min_level_num_docs: 40000 + maturation_period: 12h