jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #948 from openreplay/connectors_confluent

Browse source 
Changed python-kafka to confluent-kafka
This commit is contained in:
MauricioGarciaS 2023-01-20 16:11:42 +01:00 • committed by GitHub
commit 8c90ff8da9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 50 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
ee/connectors/consumer.py
View file

@ -1,5 +1,5 @@
import os
from kafka import KafkaConsumer
from confluent_kafka import Consumer
from datetime import datetime
from collections import defaultdict
@ -29,17 +29,21 @@ def main():
sessions_batch = []
codec = MessageCodec()
consumer = KafkaConsumer(security_protocol="SSL",
bootstrap_servers=[os.environ['KAFKA_SERVER_2'],
os.environ['KAFKA_SERVER_1']],
group_id=f"my_test3_connector_{DATABASE}",
auto_offset_reset="earliest",
enable_auto_commit=False
)
consumer = Consumer({
"security.protocol": "SSL",
"bootstrap.servers": ",".join([os.environ['KAFKA_SERVER_1'],
os.environ['KAFKA_SERVER_2']]),
"group.id": f"connector_{DATABASE}",
"auto.offset.reset": "earliest",
"enable.auto.commit": False
})
consumer.subscribe(topics=["raw", "raw_ios"])
consumer.subscribe(["raw", "raw_ios"])
print("Kafka consumer subscribed")
for msg in consumer:
while True:
msg.consumer.poll(1.0)
if msg is None:
continue
messages = codec.decode_detailed(msg.value)
session_id = codec.decode_key(msg.key)
if messages is None:

7
ee/connectors/deploy/Dockerfile_bigquery
View file

@ -1,8 +1,13 @@
FROM python:3.8-slim
FROM python:3.11
WORKDIR /usr/src/app
COPY . .
RUN apt update
RUN apt-get install -y libc-dev libffi-dev gcc
RUN apt update && apt -y install software-properties-common gcc
RUN git clone https://github.com/edenhill/librdkafka
RUN cd librdkafka && ./configure && make && make install && ldconfig
RUN pip install -r ./deploy/requirements_bigquery.txt

7
ee/connectors/deploy/Dockerfile_clickhouse
View file

@ -1,8 +1,13 @@
FROM python:3.8-slim
FROM python:3.11
WORKDIR /usr/src/app
COPY . .
RUN apt update
RUN apt-get install -y libc-dev libffi-dev gcc
RUN apt update && apt -y install software-properties-common gcc
RUN git clone https://github.com/edenhill/librdkafka
RUN cd librdkafka && ./configure && make && make install && ldconfig
RUN pip install -r ./deploy/requirements_clickhouse.txt

7
ee/connectors/deploy/Dockerfile_pg
View file

@ -1,8 +1,13 @@
FROM python:3.8-slim
FROM python:3.11
WORKDIR /usr/src/app
COPY . .
RUN apt update
RUN apt-get install -y libc-dev libffi-dev gcc
RUN apt update && apt -y install software-properties-common gcc
RUN git clone https://github.com/edenhill/librdkafka
RUN cd librdkafka && ./configure && make && make install && ldconfig
RUN pip install -r ./deploy/requirements_pg.txt

7
ee/connectors/deploy/Dockerfile_redshift
View file

@ -1,8 +1,13 @@
FROM python:3.8-slim
FROM python:3.11
WORKDIR /usr/src/app
COPY . .
RUN apt update
RUN apt-get install -y libc-dev libffi-dev gcc
RUN apt update && apt -y install software-properties-common gcc
RUN git clone https://github.com/edenhill/librdkafka
RUN cd librdkafka && ./configure && make && make install && ldconfig
RUN pip install -r ./deploy/requirements_redshift.txt

7
ee/connectors/deploy/Dockerfile_snowflake
View file

@ -1,8 +1,13 @@
FROM python:3.8-slim
FROM python:3.11
WORKDIR /usr/src/app
COPY . .
RUN apt update
RUN apt-get install -y libc-dev libffi-dev gcc
RUN apt update && apt -y install software-properties-common gcc
RUN git clone https://github.com/edenhill/librdkafka
RUN cd librdkafka && ./configure && make && make install && ldconfig
RUN pip install -r ./deploy/requirements_snowflake.txt

4
ee/connectors/deploy/requirements_bigquery.txt
View file

@ -1,7 +1,7 @@
kafka-python==2.0.2
confluent-kafka
psycopg2-binary==2.9.3
SQLAlchemy==1.4.43
google-cloud-bigquery
google-cloud-bigquery==3.4.2
pandas==1.5.1
PyYAML
pandas-gbq

2
ee/connectors/deploy/requirements_clickhouse.txt
View file

@ -3,7 +3,7 @@ chardet==5.0.0
clickhouse-driver==0.2.4
clickhouse-sqlalchemy==0.2.2
idna==3.4
kafka-python==2.0.2
confluent-kafka
pandas==1.5.1
pytz==2022.6
requests==2.28.1

2
ee/connectors/deploy/requirements_pg.txt
View file

@ -1,7 +1,7 @@
certifi==2022.09.24
chardet==5.0.0
idna==3.4
kafka-python==2.0.2
confluent-kafka
pandas==1.5.1
psycopg2-binary==2.9.3
pytz==2022.6

2
ee/connectors/deploy/requirements_redshift.txt
View file

@ -3,7 +3,7 @@ chardet==5.0.0
clickhouse-driver==0.2.4
clickhouse-sqlalchemy==0.2.2
idna==3.4
kafka-python==2.0.2
confluent-kafka
psycopg2-binary==2.9.3
pytz==2022.6
requests==2.28.1

2
ee/connectors/deploy/requirements_snowflake.txt
View file

@ -1,5 +1,5 @@
pandas==1.5.1
kafka-python==2.0.2
confluent-kafka
SQLAlchemy==1.4.43
snowflake-connector-python==2.8.2
snowflake-sqlalchemy==1.4.4