Quickwit Helm Updates (#708)

* chore(helm): adding quickwit helm chart

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* chore(helm): disabling ingress quickwit

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* fix(helm): quickwit image tag

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* fix(helm): quickwit run

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* chore(helm): quickwit health check

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* chore(helm): Adding quickwit indexer config

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* Adding quickwit config files

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* chore(helm): Adding quickwit bucket

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* chore(helm): creating quickwit config from files

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* chore(helm): quickwit mouting configs

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* chore(helm): updating quickwit

* chore(helm): updating quickwit chart

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

* chore(helm): Updating quickwit version

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>
This commit is contained in:
Rajesh Rajendran 2022-09-02 05:28:44 +00:00 committed by GitHub
parent 3769f16f86
commit 76e4ad9ad2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 681 additions and 1 deletions

View file

@ -29,3 +29,7 @@ dependencies:
version: "4.x.x"
repository: "https://kubernetes.github.io/ingress-nginx"
condition: ingress-nginx.enabled
- name: quickwit
version: "0.3.1"
repository: "file://charts/quickwit"
condition: quickwit.enabled

View file

@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

View file

@ -0,0 +1,24 @@
apiVersion: v2
name: quickwit
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful quickwit or functions for the chart developer. They're included as
# a dependency of application charts to inject those quickwit and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.3.1
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
AppVersion: "v1.8.0"

View file

@ -0,0 +1,40 @@
#
# Index config file for gh-archive dataset.
#
version: 0
index_id: fetchevent
doc_mapping:
mode: strict
field_mappings:
- name: method
type: text
tokenizer: default
record: position
- name: url
type: text
tokenizer: default
record: position
- name: request
type: text
tokenizer: default
record: position
- name: response
type: text
tokenizer: default
record: position
- name: status
type: i64
indexed: true
fast: true
- name: timestamp
type: i64
fast: true
- name: duration
type: i64
fast: true
search_settings:
default_search_fields: [url, request, response]

View file

@ -0,0 +1,30 @@
#
# Index config file for gh-archive dataset.
#
version: 0
index_id: graphql
doc_mapping:
mode: strict
field_mappings:
- name: operation_kind
type: text
tokenizer: default
record: position
- name: operation_name
type: text
tokenizer: default
record: position
- name: variables
type: text
tokenizer: default
record: position
- name: response
type: text
tokenizer: default
record: position
search_settings:
default_search_fields: [operation_kind, operation_name, variables]

View file

@ -0,0 +1,68 @@
#
# Index config file for gh-archive dataset.
#
version: 0
index_id: pageevent
doc_mapping:
mode: strict
field_mappings:
- name: message_id
type: i64
indexed: true
fast: true
- name: timestamp
type: i64
fast: true
- name: url
type: text
tokenizer: default
record: position
- name: referrer
type: text
tokenizer: default
record: position
- name: loaded
type: bool
fast: true
- name: request_start
type: i64
fast: true
- name: response_start
type: i64
fast: true
- name: response_end
type: i64
fast: true
- name: dom_content_loaded_event_start
type: i64
fast: true
- name: dom_content_loaded_event_end
type: i64
fast: true
- name: load_event_start
type: i64
fast: true
- name: load_event_end
type: i64
fast: true
- name: first_paint
type: i64
fast: true
- name: first_contentful_paint
type: i64
fast: true
- name: speed_index
type: i64
fast: true
- name: visually_complete
type: i64
fast: true
- name: time_to_interactive
type: i64
fast: true
search_settings:
default_search_fields: [url, referrer, visually_complete]

View file

@ -0,0 +1,6 @@
## In order to save data into S3
# metastore also accepts s3://{bucket/path}#pooling_interval={seconds}s
version: 0
metastore_uri: s3://{{.Values.global.s3.quickwitBucket}}/quickwit-indexes
default_index_root_uri: s3://{{.Values.global.s3.quickwitBucket}}/quickwit-indexes
listen_address: 0.0.0.0

View file

@ -0,0 +1,5 @@
## In order to save data into S3
# metastore also accepts s3://{bucket/path}#pooling_interval={seconds}s
version: 0
metastore_uri: s3://{{.Values.global.s3.quickwitBucket}}/quickwit-indexes
default_index_root_uri: s3://{{.Values.global.s3.quickwitBucket}}/quickwit-indexes

View file

@ -0,0 +1,14 @@
#
# Source config file.
#
source_id: fetch-kafka
source_type: kafka
params:
topic: quickwit
client_params:
group.id: fetch-consumer
bootstrap.servers: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}'
{{- if eq .Values.global.kafka.kafkaUseSsl "true" }}
security.protocol: SSL
{{- end}}

View file

@ -0,0 +1,14 @@
#
# Source config file.
#
source_id: graphql-kafka
source_type: kafka
params:
topic: quickwit
client_params:
group.id: graphql-consumer
bootstrap.servers: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}'
{{- if eq .Values.global.kafka.kafkaUseSsl "true" }}
security.protocol: SSL
{{- end}}

View file

@ -0,0 +1,14 @@
#
# Source config file.
#
source_id: pageevent-kafka
source_type: kafka
params:
topic: quickwit
client_params:
group.id: pageevent-consumer
bootstrap.servers: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}'
{{- if eq .Values.global.kafka.kafkaUseSsl "true" }}
security.protocol: SSL
{{- end}}

View file

@ -0,0 +1,22 @@
1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
{{- range .paths }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
{{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "quickwit.fullname" . }})
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "quickwit.fullname" . }}'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "quickwit.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "quickwit.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}

View file

@ -0,0 +1,62 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "quickwit.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "quickwit.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "quickwit.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "quickwit.labels" -}}
helm.sh/chart: {{ include "quickwit.chart" . }}
{{ include "quickwit.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "quickwit.selectorLabels" -}}
app.kubernetes.io/name: {{ include "quickwit.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "quickwit.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "quickwit.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,18 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "quickwit.fullname" . }}
labels:
{{- include "quickwit.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": pre-install, pre-upgrade
"helm.sh/hook-weight": "-4" # Higher precidence, so the first the config map will get created.
data:
{{/* https://helm.sh/docs/chart_template_guide/accessing_files/ */}}
{{ $currentScope := .}}
{{- range $path, $_ := .Files.Glob (printf "files/**.yaml") }}
{{- with $currentScope}}
{{- base $path | nindent 2}}: |-
{{- tpl (.Files.Get $path) . | nindent 4 }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,84 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "quickwit.fullname" . }}
labels:
{{- include "quickwit.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "quickwit.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "quickwit.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "quickwit.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- if .Values.healthCheck}}
{{- .Values.healthCheck | toYaml | nindent 10}}
{{- end}}
command:
- /bin/sh
- -c
args:
- |
set -x
ls -laR /quickwit/
quickwit run --config config/s3-config-listen.yaml
env:
- name: AWS_DEFAULT_REGION
value: "{{ .Values.global.s3.region }}"
{{- if eq .Values.global.s3.endpoint "http://minio.db.svc.cluster.local:9000" }}
- name: QW_S3_ENDPOINT
value: 'http://minio.db.svc.cluster.local:9000'
{{- end}}
- name: AWS_ACCESS_KEY_ID
value: {{ .Values.global.s3.accessKey }}
- name: AWS_SECRET_ACCESS_KEY
value: {{ .Values.global.s3.secretKey }}
ports:
{{- range $key, $val := .Values.service.ports }}
- name: {{ $key }}
containerPort: {{ $val }}
protocol: TCP
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: configs
mountPath: /quickwit/config/
volumes:
- name: configs
configMap:
name: {{ include "quickwit.fullname" . }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}

View file

@ -0,0 +1,28 @@
{{- if .Values.autoscaling.enabled }}
apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "quickwit.fullname" . }}
labels:
{{- include "quickwit.labels" . | nindent 4 }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "quickwit.fullname" . }}
minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
metrics:
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,35 @@
{{- if .Values.ingress.enabled }}
{{- $fullName := include "quickwit.fullname" . -}}
{{- $socketioSvcPort := .Values.service.ports.socketio -}}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ $fullName }}
labels:
{{- include "quickwit.labels" . | nindent 4 }}
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /$1
nginx.ingress.kubernetes.io/upstream-hash-by: $http_x_forwarded_for
{{- with .Values.ingress.annotations }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
ingressClassName: "{{ tpl .Values.ingress.className . }}"
tls:
- hosts:
- {{ .Values.global.domainName }}
{{- if .Values.ingress.tls.secretName}}
secretName: {{ .Values.ingress.tls.secretName }}
{{- end}}
rules:
- host: {{ .Values.global.domainName }}
http:
paths:
- pathType: Prefix
backend:
service:
name: {{ $fullName }}
port:
number: {{ $socketioSvcPort }}
path: /ws-quickwit/(.*)
{{- end }}

View file

@ -0,0 +1,52 @@
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "quickwit.fullname" . }}
labels:
{{- include "quickwit.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": pre-install, pre-upgrade
"helm.sh/hook-weight": "-3"
spec:
backoffLimit: 0 # Don't restart failing containers
template:
metadata:
name: quickwitInitialization
spec:
containers:
- name: quickwit
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
env:
- name: AWS_DEFAULT_REGION
value: "{{ .Values.global.s3.region }}"
{{- if eq .Values.global.s3.endpoint "http://minio.db.svc.cluster.local:9000" }}
- name: QW_S3_ENDPOINT
value: 'http://minio.db.svc.cluster.local:9000'
{{- end}}
- name: AWS_ACCESS_KEY_ID
value: {{ .Values.global.s3.accessKey }}
- name: AWS_SECRET_ACCESS_KEY
value: {{ .Values.global.s3.secretKey }}
command:
- /bin/sh
- -c
args:
- |
set -x
configs=""
configs="${configs} fetch"
configs="${configs} graphql"
configs="${configs} pageevent"
for config in ${configs};do
quickwit index create --index-config config/index-config-${config}.yaml --config config/s3-config.yaml || true
quickwit source create --index fetchevent --source-config config/source-${config}.yaml --config config/s3-config.yaml || true
done
volumeMounts:
- name: dbmigrationscript
mountPath: /quickwit/config/
restartPolicy: Never
volumes:
- name: dbmigrationscript
configMap:
name: {{ include "quickwit.fullname" . }}

View file

@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "quickwit.fullname" . }}
labels:
{{- include "quickwit.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
ports:
{{- range $key, $val := .Values.service.ports }}
- port: {{ $val }}
targetPort: {{ $key }}
protocol: TCP
name: {{ $key }}
{{- end}}
selector:
{{- include "quickwit.selectorLabels" . | nindent 4 }}

View file

@ -0,0 +1,12 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "quickwit.serviceAccountName" . }}
labels:
{{- include "quickwit.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,101 @@
# Default values for openreplay.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
repository: quickwit/quickwit
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: "latest"
# tag: "v0.2.1"
imagePullSecrets: []
nameOverride: "quickwit"
fullnameOverride: "quickwit-openreplay"
indexConfig: |-
# Version of the index config file format
version: 0
# Sources
sources:
- source_id: openreplay-kafka-source
source_type: kafka
params:
topic: quickwit-indexer
client_params:
bootstrap.servers: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}'
{{- if eq .Values.global.kafka.kafkaUseSsl "true" }}
security.protocol: SSL
{{- end}}
serviceAccount:
# Specifies whether a service account should be created
create: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
podAnnotations: {}
podSecurityContext: {}
# fsGroup: 2000
securityContext: {}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
service:
type: ClusterIP
ports:
search: 7280
ingress:
enabled: false
className: "{{ .Values.global.ingress.controller.ingressClassResource.name }}"
annotations: {}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
tls:
secretName: openreplay-ssl
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 5
targetCPUUtilizationPercentage: 80
# targetMemoryUtilizationPercentage: 80
env:
REDIS_URL: "redis://redis-master.db.svc.cluster.local:6379"
debug: 0
uws: false
redis: false
nodeSelector: {}
tolerations: []
affinity: {}

View file

@ -12,6 +12,7 @@ topics=(
"cache"
"analytics"
"storage-failover"
"quickwit"
)
touch /tmp/config.txt

View file

@ -5,7 +5,7 @@ set -e
cd /tmp
buckets=("mobs" "sessions-assets" "static" "sourcemaps" "sessions-mobile-assets")
buckets=("mobs" "sessions-assets" "static" "sourcemaps" "sessions-mobile-assets" "quickwit")
mc alias set minio http://minio.db.svc.cluster.local:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY

View file

@ -21,6 +21,10 @@ clickhouse:
# For enterpriseEdition
enabled: false
quickwit:
# For enterpriseEdition
enabled: false
kafka: &kafka
# For enterpriseEdition
# enabled: true
@ -100,6 +104,8 @@ global:
assetsBucket: "sessions-assets"
recordingsBucket: "mobs"
sourcemapsBucket: "sourcemaps"
# This is only for enterpriseEdition
quickwitBucket: "quickwit"
# if you're using one node installation, where
# you're using local s3, make sure these variables
# are same as minio.global.minio.accesskey and secretKey