521 lines
15 KiB
YAML
521 lines
15 KiB
YAML
---
|
|
# Source: kafka/templates/serviceaccount.yaml
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: kafka
|
|
labels:
|
|
app.kubernetes.io/name: kafka
|
|
helm.sh/chart: kafka-11.8.6
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: kafka
|
|
---
|
|
# Source: kafka/templates/scripts-configmap.yaml
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: kafka-scripts
|
|
labels:
|
|
app.kubernetes.io/name: kafka
|
|
helm.sh/chart: kafka-11.8.6
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
data:
|
|
setup.sh: |-
|
|
#!/bin/bash
|
|
|
|
ID="${MY_POD_NAME#"kafka-"}"
|
|
export KAFKA_CFG_BROKER_ID="$ID"
|
|
|
|
exec /entrypoint.sh /run.sh
|
|
---
|
|
# Source: kafka/charts/zookeeper/templates/svc-headless.yaml
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: kafka-zookeeper-headless
|
|
namespace: db
|
|
labels:
|
|
app.kubernetes.io/name: zookeeper
|
|
helm.sh/chart: zookeeper-5.21.9
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: zookeeper
|
|
spec:
|
|
type: ClusterIP
|
|
clusterIP: None
|
|
publishNotReadyAddresses: true
|
|
ports:
|
|
|
|
- name: tcp-client
|
|
port: 2181
|
|
targetPort: client
|
|
|
|
|
|
- name: follower
|
|
port: 2888
|
|
targetPort: follower
|
|
- name: tcp-election
|
|
port: 3888
|
|
targetPort: election
|
|
selector:
|
|
app.kubernetes.io/name: zookeeper
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/component: zookeeper
|
|
---
|
|
# Source: kafka/charts/zookeeper/templates/svc.yaml
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: kafka-zookeeper
|
|
namespace: db
|
|
labels:
|
|
app.kubernetes.io/name: zookeeper
|
|
helm.sh/chart: zookeeper-5.21.9
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: zookeeper
|
|
spec:
|
|
type: ClusterIP
|
|
ports:
|
|
|
|
- name: tcp-client
|
|
port: 2181
|
|
targetPort: client
|
|
|
|
|
|
- name: follower
|
|
port: 2888
|
|
targetPort: follower
|
|
- name: tcp-election
|
|
port: 3888
|
|
targetPort: election
|
|
selector:
|
|
app.kubernetes.io/name: zookeeper
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/component: zookeeper
|
|
---
|
|
# Source: kafka/templates/kafka-metrics-svc.yaml
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: kafka-metrics
|
|
labels:
|
|
app.kubernetes.io/name: kafka
|
|
helm.sh/chart: kafka-11.8.6
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: metrics
|
|
annotations:
|
|
|
|
prometheus.io/path: /metrics
|
|
prometheus.io/port: '9308'
|
|
prometheus.io/scrape: "true"
|
|
spec:
|
|
type: ClusterIP
|
|
ports:
|
|
- name: http-metrics
|
|
port: 9308
|
|
protocol: TCP
|
|
targetPort: metrics
|
|
nodePort: null
|
|
selector:
|
|
app.kubernetes.io/name: kafka
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/component: metrics
|
|
---
|
|
# Source: kafka/templates/svc-headless.yaml
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: kafka-headless
|
|
labels:
|
|
app.kubernetes.io/name: kafka
|
|
helm.sh/chart: kafka-11.8.6
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: kafka
|
|
spec:
|
|
type: ClusterIP
|
|
clusterIP: None
|
|
ports:
|
|
- name: tcp-client
|
|
port: 9092
|
|
protocol: TCP
|
|
targetPort: kafka-client
|
|
- name: tcp-internal
|
|
port: 9093
|
|
protocol: TCP
|
|
targetPort: kafka-internal
|
|
selector:
|
|
app.kubernetes.io/name: kafka
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/component: kafka
|
|
---
|
|
# Source: kafka/templates/svc.yaml
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: kafka
|
|
labels:
|
|
app.kubernetes.io/name: kafka
|
|
helm.sh/chart: kafka-11.8.6
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: kafka
|
|
spec:
|
|
type: ClusterIP
|
|
ports:
|
|
- name: tcp-client
|
|
port: 9092
|
|
protocol: TCP
|
|
targetPort: kafka-client
|
|
nodePort: null
|
|
selector:
|
|
app.kubernetes.io/name: kafka
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/component: kafka
|
|
---
|
|
# Source: kafka/templates/kafka-metrics-deployment.yaml
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: kafka-exporter
|
|
labels:
|
|
app.kubernetes.io/name: kafka
|
|
helm.sh/chart: kafka-11.8.6
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: metrics
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: kafka
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/component: metrics
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: kafka
|
|
helm.sh/chart: kafka-11.8.6
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: metrics
|
|
spec:
|
|
containers:
|
|
- name: kafka-exporter
|
|
image: docker.io/bitnami/kafka-exporter:1.2.0-debian-10-r220
|
|
imagePullPolicy: "IfNotPresent"
|
|
command:
|
|
- /bin/bash
|
|
- -ec
|
|
- |
|
|
read -r -a sasl_passwords <<< "$(tr ',;' ' ' <<< "${SASL_USER_PASSWORD}")"
|
|
kafka_exporter \
|
|
--kafka.server=kafka-0.kafka-headless.db.svc.cluster.local:9092 \
|
|
--kafka.server=kafka-1.kafka-headless.db.svc.cluster.local:9092 \
|
|
--web.listen-address=:9308
|
|
ports:
|
|
- name: metrics
|
|
containerPort: 9308
|
|
resources:
|
|
limits: {}
|
|
requests: {}
|
|
---
|
|
# Source: kafka/charts/zookeeper/templates/statefulset.yaml
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: kafka-zookeeper
|
|
namespace: db
|
|
labels:
|
|
app.kubernetes.io/name: zookeeper
|
|
helm.sh/chart: zookeeper-5.21.9
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: zookeeper
|
|
role: zookeeper
|
|
spec:
|
|
serviceName: kafka-zookeeper-headless
|
|
replicas: 1
|
|
podManagementPolicy: Parallel
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: zookeeper
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/component: zookeeper
|
|
template:
|
|
metadata:
|
|
name: kafka-zookeeper
|
|
labels:
|
|
app.kubernetes.io/name: zookeeper
|
|
helm.sh/chart: zookeeper-5.21.9
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: zookeeper
|
|
spec:
|
|
|
|
serviceAccountName: default
|
|
securityContext:
|
|
fsGroup: 1001
|
|
containers:
|
|
- name: zookeeper
|
|
image: docker.io/bitnami/zookeeper:3.6.2-debian-10-r10
|
|
imagePullPolicy: "IfNotPresent"
|
|
securityContext:
|
|
runAsUser: 1001
|
|
command:
|
|
- bash
|
|
- -ec
|
|
- |
|
|
# Execute entrypoint as usual after obtaining ZOO_SERVER_ID based on POD hostname
|
|
HOSTNAME=`hostname -s`
|
|
if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then
|
|
ORD=${BASH_REMATCH[2]}
|
|
export ZOO_SERVER_ID=$((ORD+1))
|
|
else
|
|
echo "Failed to get index from hostname $HOST"
|
|
exit 1
|
|
fi
|
|
exec /entrypoint.sh /run.sh
|
|
resources:
|
|
requests:
|
|
cpu: 250m
|
|
memory: 256Mi
|
|
env:
|
|
- name: ZOO_DATA_LOG_DIR
|
|
value: ""
|
|
- name: ZOO_PORT_NUMBER
|
|
value: "2181"
|
|
- name: ZOO_TICK_TIME
|
|
value: "2000"
|
|
- name: ZOO_INIT_LIMIT
|
|
value: "10"
|
|
- name: ZOO_SYNC_LIMIT
|
|
value: "5"
|
|
- name: ZOO_MAX_CLIENT_CNXNS
|
|
value: "60"
|
|
- name: ZOO_4LW_COMMANDS_WHITELIST
|
|
value: "srvr, mntr, ruok"
|
|
- name: ZOO_LISTEN_ALLIPS_ENABLED
|
|
value: "no"
|
|
- name: ZOO_AUTOPURGE_INTERVAL
|
|
value: "0"
|
|
- name: ZOO_AUTOPURGE_RETAIN_COUNT
|
|
value: "3"
|
|
- name: ZOO_MAX_SESSION_TIMEOUT
|
|
value: "40000"
|
|
- name: ZOO_SERVERS
|
|
value: kafka-zookeeper-0.kafka-zookeeper-headless.db.svc.cluster.local:2888:3888
|
|
- name: ZOO_ENABLE_AUTH
|
|
value: "no"
|
|
- name: ZOO_HEAP_SIZE
|
|
value: "1024"
|
|
- name: ZOO_LOG_LEVEL
|
|
value: "ERROR"
|
|
- name: ALLOW_ANONYMOUS_LOGIN
|
|
value: "yes"
|
|
- name: POD_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
apiVersion: v1
|
|
fieldPath: metadata.name
|
|
ports:
|
|
|
|
- name: client
|
|
containerPort: 2181
|
|
|
|
|
|
- name: follower
|
|
containerPort: 2888
|
|
- name: election
|
|
containerPort: 3888
|
|
livenessProbe:
|
|
exec:
|
|
command: ['/bin/bash', '-c', 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok']
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
successThreshold: 1
|
|
failureThreshold: 6
|
|
readinessProbe:
|
|
exec:
|
|
command: ['/bin/bash', '-c', 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok']
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
successThreshold: 1
|
|
failureThreshold: 6
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /bitnami/zookeeper
|
|
volumes:
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
name: data
|
|
annotations:
|
|
spec:
|
|
accessModes:
|
|
- "ReadWriteOnce"
|
|
resources:
|
|
requests:
|
|
storage: "8Gi"
|
|
---
|
|
# Source: kafka/templates/statefulset.yaml
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: kafka
|
|
labels:
|
|
app.kubernetes.io/name: kafka
|
|
helm.sh/chart: kafka-11.8.6
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: kafka
|
|
spec:
|
|
podManagementPolicy: Parallel
|
|
replicas: 2
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: kafka
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/component: kafka
|
|
serviceName: kafka-headless
|
|
updateStrategy:
|
|
type: "RollingUpdate"
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: kafka
|
|
helm.sh/chart: kafka-11.8.6
|
|
app.kubernetes.io/instance: kafka
|
|
app.kubernetes.io/managed-by: Helm
|
|
app.kubernetes.io/component: kafka
|
|
spec:
|
|
securityContext:
|
|
fsGroup: 1001
|
|
runAsUser: 1001
|
|
serviceAccountName: kafka
|
|
containers:
|
|
- name: kafka
|
|
image: docker.io/bitnami/kafka:2.6.0-debian-10-r30
|
|
imagePullPolicy: "IfNotPresent"
|
|
command:
|
|
- /scripts/setup.sh
|
|
env:
|
|
- name: BITNAMI_DEBUG
|
|
value: "false"
|
|
- name: MY_POD_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.podIP
|
|
- name: MY_POD_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.name
|
|
- name: KAFKA_CFG_ZOOKEEPER_CONNECT
|
|
value: "kafka-zookeeper"
|
|
- name: KAFKA_INTER_BROKER_LISTENER_NAME
|
|
value: "INTERNAL"
|
|
- name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP
|
|
value: "INTERNAL:PLAINTEXT,CLIENT:PLAINTEXT"
|
|
- name: KAFKA_CFG_LISTENERS
|
|
value: "INTERNAL://:9093,CLIENT://:9092"
|
|
- name: KAFKA_CFG_ADVERTISED_LISTENERS
|
|
value: "INTERNAL://$(MY_POD_NAME).kafka-headless.db.svc.cluster.local:9093,CLIENT://$(MY_POD_NAME).kafka-headless.db.svc.cluster.local:9092"
|
|
- name: ALLOW_PLAINTEXT_LISTENER
|
|
value: "yes"
|
|
- name: KAFKA_CFG_DELETE_TOPIC_ENABLE
|
|
value: "false"
|
|
- name: KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE
|
|
value: "true"
|
|
- name: KAFKA_HEAP_OPTS
|
|
value: "-Xmx1024m -Xms1024m"
|
|
- name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MESSAGES
|
|
value: "10000"
|
|
- name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MS
|
|
value: "1000"
|
|
- name: KAFKA_CFG_LOG_RETENTION_BYTES
|
|
value: "1073741824"
|
|
- name: KAFKA_CFG_LOG_RETENTION_CHECK_INTERVALS_MS
|
|
value: "300000"
|
|
- name: KAFKA_CFG_LOG_RETENTION_HOURS
|
|
value: "168"
|
|
- name: KAFKA_CFG_MESSAGE_MAX_BYTES
|
|
value: "1000012"
|
|
- name: KAFKA_CFG_LOG_SEGMENT_BYTES
|
|
value: "1073741824"
|
|
- name: KAFKA_CFG_LOG_DIRS
|
|
value: "/bitnami/kafka/data"
|
|
- name: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR
|
|
value: "1"
|
|
- name: KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR
|
|
value: "1"
|
|
- name: KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
|
|
value: "1"
|
|
- name: KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR
|
|
value: "1"
|
|
- name: KAFKA_CFG_NUM_IO_THREADS
|
|
value: "8"
|
|
- name: KAFKA_CFG_NUM_NETWORK_THREADS
|
|
value: "3"
|
|
- name: KAFKA_CFG_NUM_PARTITIONS
|
|
value: "1"
|
|
- name: KAFKA_CFG_NUM_RECOVERY_THREADS_PER_DATA_DIR
|
|
value: "1"
|
|
- name: KAFKA_CFG_SOCKET_RECEIVE_BUFFER_BYTES
|
|
value: "102400"
|
|
- name: KAFKA_CFG_SOCKET_REQUEST_MAX_BYTES
|
|
value: "104857600"
|
|
- name: KAFKA_CFG_SOCKET_SEND_BUFFER_BYTES
|
|
value: "102400"
|
|
- name: KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS
|
|
value: "6000"
|
|
ports:
|
|
- name: kafka-client
|
|
containerPort: 9092
|
|
- name: kafka-internal
|
|
containerPort: 9093
|
|
livenessProbe:
|
|
tcpSocket:
|
|
port: kafka-client
|
|
initialDelaySeconds: 10
|
|
timeoutSeconds: 5
|
|
failureThreshold:
|
|
periodSeconds:
|
|
successThreshold:
|
|
readinessProbe:
|
|
tcpSocket:
|
|
port: kafka-client
|
|
initialDelaySeconds: 5
|
|
timeoutSeconds: 5
|
|
failureThreshold: 6
|
|
periodSeconds:
|
|
successThreshold:
|
|
resources:
|
|
limits: {}
|
|
requests: {}
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /bitnami/kafka
|
|
- name: scripts
|
|
mountPath: /scripts/setup.sh
|
|
subPath: setup.sh
|
|
volumes:
|
|
- name: scripts
|
|
configMap:
|
|
name: kafka-scripts
|
|
defaultMode: 0755
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
name: data
|
|
spec:
|
|
accessModes:
|
|
- "ReadWriteOnce"
|
|
resources:
|
|
requests:
|
|
storage: "8Gi"
|