Fixed Docker run issues

ee/quickwit/consumer.py

@@ -92,9 +92,9 @@ class KafkaFilter():
             "enable.auto.commit": False
         })
         self.consumer.subscribe([topic])
-        self.queues = {'fetchevent': Queue(self.fetchevent_maxsize),
-                'graphql': Queue(self.graphql_maxsize),
-                'pageevent': Queue(self.pageevent_maxsize)
+        self.queues = {'Fetchevent': Queue(self.fetchevent_maxsize),
+                'Graphql': Queue(self.graphql_maxsize),
+                'Pageevent': Queue(self.pageevent_maxsize)
                 }
 
     async def add_to_queue(self, message):

ee/quickwit/entrypoint.sh

@@ -9,4 +9,13 @@ find /quickwit/ -type f -name "*.yaml" -exec sed -i "s/{{QUICKWIT_TOPIC}}/${QUIC
 find /quickwit/ -type f -name "*.yaml" -exec sed -i "s/{{QUICKWIT_PORT}}/${QUICKWIT_PORT}/g" {} \;
 find /quickwit/ -type f -name "*.yaml" -exec sed -i "s#{{data_dir_path}}#${data_dir_path}#g" {} \;
 
-./quickwit_start_task.sh & ./setup_indexes_and_worker.sh && fg
+./quickwit_start_task.sh & pid1=$!
+sleep 120
+echo "Creating indexes.."
+quickwit index create --index-config index-config-fetch.yaml
+quickwit index create --index-config index-config-graphql.yaml
+quickwit index create --index-config index-config-pageevent.yaml
+echo "Running kafka reader.."
+python3 -u consumer.py & pid2=$!
+wait $pid1 $pid2
+

ee/quickwit/index-config-fetch.yaml

@@ -4,7 +4,7 @@
 
 version: 0.6
 
-index_id: "fetchevent"
+index_id: "Fetchevent"
 index_uri: "s3://openreplay-quickwit/quickwit-indexes/fetchevent"
 
 doc_mapping:
@@ -63,3 +63,9 @@ search_settings:
 retention:
   period: 30 days
   schedule: hourly
+
+indexing_settings:
+  merge_policy:
+    type: "stable_log"
+    min_level_num_docs: 40000
+    maturation_period: 12h

ee/quickwit/index-config-graphql.yaml

@@ -4,7 +4,7 @@
 
 version: 0.6
 
-index_id: "graphql"
+index_id: "Graphql"
 index_uri: "s3://openreplay-quickwit/quickwit-indexes/graphql"
 
 doc_mapping:
@@ -50,3 +50,9 @@ search_settings:
 retention:
   period: 30 days
   schedule: hourly
+
+indexing_settings:
+  merge_policy:
+    type: "stable_log"
+    min_level_num_docs: 40000
+    maturation_period: 12h

ee/quickwit/index-config-pageevent.yaml

@@ -4,7 +4,7 @@
 
 version: 0.6
 
-index_id: "pageevent"
+index_id: "Pageevent"
 index_uri: "s3://openreplay-quickwit/quickwit-indexes/pageevent"
 
 doc_mapping:
@@ -84,3 +84,9 @@ search_settings:
 retention:
   period: 30 days
   schedule: hourly
+
+indexing_settings:
+  merge_policy:
+    type: "stable_log"
+    min_level_num_docs: 40000
+    maturation_period: 12h