From 125e5289be880b5d64f1b2668bb1d93fe0f98c24 Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Fri, 29 Jul 2022 16:46:49 +0200 Subject: [PATCH] Updated methods. New indexes created in quickwit reading from same topic in kafka with different group id --- clean.sh | 10 +++++++-- create_kafka_index.sh | 4 +++- create_source.sh | 1 - create_sources.sh | 3 +++ index-config-fetch.yaml | 4 ++-- index-config-pageevent.yaml | 2 +- run_quickwit.sh | 2 +- kafka-source.yaml => source-fetch.yaml | 6 ++++-- source-graphql.yaml | 12 +++++++++++ source-pageevent.yaml | 12 +++++++++++ sources.yaml | 29 -------------------------- 11 files changed, 46 insertions(+), 39 deletions(-) delete mode 100644 create_source.sh create mode 100644 create_sources.sh rename kafka-source.yaml => source-fetch.yaml (69%) create mode 100644 source-graphql.yaml create mode 100644 source-pageevent.yaml delete mode 100644 sources.yaml diff --git a/clean.sh b/clean.sh index 502db2a2e..9ec3a5f7e 100644 --- a/clean.sh +++ b/clean.sh @@ -1,2 +1,8 @@ -docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source delete --index quickwit-kafka --source kafka-source --config s3-config.yaml -docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index delete --index quickwit-kafka --config s3-config.yaml +docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source delete --index fetchevent --source fetch-kafka --config s3-config.yaml +docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index delete --index fetchevent --config s3-config.yaml + +docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source delete --index graphql --source graphql-kafka --config s3-config.yaml +docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index delete --index graphql --config s3-config.yaml + +docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source delete --index pageevent --source pageevent-kafka --config s3-config.yaml +docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index delete --index pageevent --config s3-config.yaml diff --git a/create_kafka_index.sh b/create_kafka_index.sh index 24e76803a..efb165db6 100644 --- a/create_kafka_index.sh +++ b/create_kafka_index.sh @@ -1 +1,3 @@ -docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/index-config.yaml:/quickwit/index-config.yaml -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index create --index-config index-config.yaml --config s3-config.yaml +docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/index-config-fetch.yaml:/quickwit/index-config-fetch.yaml -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index create --index-config index-config-fetch.yaml --config s3-config.yaml +docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/index-config-graphql.yaml:/quickwit/index-config-graphql.yaml -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index create --index-config index-config-graphql.yaml --config s3-config.yaml +docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/index-config-pageevent.yaml:/quickwit/index-config-pageevent.yaml -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit index create --index-config index-config-pageevent.yaml --config s3-config.yaml diff --git a/create_source.sh b/create_source.sh deleted file mode 100644 index 77aab1937..000000000 --- a/create_source.sh +++ /dev/null @@ -1 +0,0 @@ -docker run -v /etc/hosts:/etc/hosts:ro -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -v $(pwd)/kafka-source.yaml:/quickwit/kafka-source.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source create --index quickwit-kafka --source-config kafka-source.yaml --config s3-config.yaml diff --git a/create_sources.sh b/create_sources.sh new file mode 100644 index 000000000..c854899c3 --- /dev/null +++ b/create_sources.sh @@ -0,0 +1,3 @@ +docker run -v /etc/hosts:/etc/hosts:ro -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -v $(pwd)/source-fetch.yaml:/quickwit/source-fetch.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source create --index fetchevent --source-config source-fetch.yaml --config s3-config.yaml +docker run -v /etc/hosts:/etc/hosts:ro -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -v $(pwd)/source-graphql.yaml:/quickwit/source-graphql.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source create --index graphql --source-config source-graphql.yaml --config s3-config.yaml +docker run -v /etc/hosts:/etc/hosts:ro -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config.yaml:/quickwit/s3-config.yaml -v $(pwd)/source-pageevent.yaml:/quickwit/source-pageevent.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region quickwit/quickwit source create --index pageevent --source-config source-pageevent.yaml --config s3-config.yaml diff --git a/index-config-fetch.yaml b/index-config-fetch.yaml index 1a7577450..1d89f72c9 100644 --- a/index-config-fetch.yaml +++ b/index-config-fetch.yaml @@ -4,7 +4,7 @@ version: 0 -index_id: fetch +index_id: fetchevent doc_mapping: mode: strict @@ -27,8 +27,8 @@ doc_mapping: record: position - name: status type: i64 + indexed: true fast: true - record: position - name: timestamp type: i64 fast: true diff --git a/index-config-pageevent.yaml b/index-config-pageevent.yaml index 90d8d152f..36a0a69fe 100644 --- a/index-config-pageevent.yaml +++ b/index-config-pageevent.yaml @@ -11,8 +11,8 @@ doc_mapping: field_mappings: - name: message_id type: i64 + indexed: true fast: true - record: position - name: timestamp type: i64 fast: true diff --git a/run_quickwit.sh b/run_quickwit.sh index a00b1046d..b2afeb011 100644 --- a/run_quickwit.sh +++ b/run_quickwit.sh @@ -1 +1 @@ -docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/qwdata:/quickwit/qwdata -v $(pwd)/s3-config-listen.yaml:/quickwit/s3-config-listen.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=eu-central-1 -e AWS_REGION=eu-central-1 -p 127.0.0.1:7280:7280 quickwit/quickwit run --config s3-config-listen.yaml +docker run -v /etc/hosts:/etc/hosts:ro -v $(pwd)/s3-config-listen.yaml:/quickwit/s3-config-listen.yaml -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key -e AWS_DEFAULT_REGION=$aws_region -e AWS_REGION=$aws_region -p 127.0.0.1:7280:7280 quickwit/quickwit run --config s3-config-listen.yaml diff --git a/kafka-source.yaml b/source-fetch.yaml similarity index 69% rename from kafka-source.yaml rename to source-fetch.yaml index 12f667dee..506c234f6 100644 --- a/kafka-source.yaml +++ b/source-fetch.yaml @@ -1,10 +1,12 @@ # -# Kafka source config file. +# Source config file. # -source_id: kafka-source + +source_id: fetch-kafka source_type: kafka params: topic: quickwit-kafka client_params: bootstrap.servers: kafka-1.kafka-headless.db.svc.cluster.local:9092 + group.id: fetch-consumer # security.protocol: SSL diff --git a/source-graphql.yaml b/source-graphql.yaml new file mode 100644 index 000000000..900e0f92b --- /dev/null +++ b/source-graphql.yaml @@ -0,0 +1,12 @@ +# +# Source config file. +# + +source_id: graphql-kafka +source_type: kafka +params: + topic: quickwit-kafka + client_params: + bootstrap.servers: kafka-1.kafka-headless.db.svc.cluster.local:9092 + group.id: graphql-consumer + # security.protocol: SSL diff --git a/source-pageevent.yaml b/source-pageevent.yaml new file mode 100644 index 000000000..6c5582a96 --- /dev/null +++ b/source-pageevent.yaml @@ -0,0 +1,12 @@ +# +# Source config file. +# + +source_id: pageevent-kafka +source_type: kafka +params: + topic: quickwit-kafka + client_params: + bootstrap.servers: kafka-1.kafka-headless.db.svc.cluster.local:9092 + group.id: pageevent-consumer + # security.protocol: SSL diff --git a/sources.yaml b/sources.yaml deleted file mode 100644 index 7dad76619..000000000 --- a/sources.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Source config file. -# - -sources: - - fetch: fetch-kafka - source_type: kafka - params: - topic: quickwit-kafka - client_params: - bootstrap.servers: kafka-1.kafka-headless.db.svc.cluster.local:9092 - group.id: fetch-consumer - # security.protocol: SSL - - graphql: graphql-kafka - source_type: kafka - params: - topic: quickwit-kafka - client_params: - bootstrap.servers: kafka-1.kafka-headless.db.svc.cluster.local:9092 - group.id: graphql-consumer - # security.protocol: SSL - - graphql: graphql-pageevent - source_type: kafka - params: - topic: quickwit-kafka - client_params: - bootstrap.servers: kafka-1.kafka-headless.db.svc.cluster.local:9092 - group.id: pageevent-consumer - # security.protocol: SSL