91921d622e
* Updated pg connector * fix(player): fix first 8 byte checker * fix(player): fix commit conflict * Added pylint * Removed pylint for incompatible license * change(ui): check for sessions records state * Patch/api v1.12.0 (#1299) * fix(chalice): include metadata in sessions exp search * fix(chalice): fixed sessions exp search wrong col name * fix(chalice): removed cookies * fix(chalice): changed base image to support SSO/xmlsec * fix(chalice): changed Dockerfile to support SSO/xmlsec * fix(chalice): changed Dockerfile to support SSO/xmlsec (cherry picked from commit4b8cf9742c) * fix(ui): project fallback to recorded variable * Patch/api v1.12.0 (#1301) * fix(chalice): changed base image to support SSO/xmlsec * fix(chalice): fixed exp search null metadata (cherry picked from commitab000751d2) * change(ui): assist no content message styles and icons * change(ui): revert menu disable * fix(connector): Added method to save state in s3 for redshift if sigterm arise * Rewriting python code in cython * Added pyx module for messages * Auto create pyx files * Updated and fixed msgcodec.pyx * Added new module to connector code * Updated kafka lib for base image * cleaned Docker and updated base image version for pandas * cleaned prints * Added code to fetch data from db and add it into redshift * Updated consumer reading method. Async multithreading over sessionId * Added split for country (Country,State,City) * Fixed decoding issue for uint * Created service able to fix data from redshift by reading from db * Handle when process ended because of lost connection to pg, country set to country value only
41 lines
1.3 KiB
Python
41 lines
1.3 KiB
Python
from decouple import config, Csv
|
|
import asyncio
|
|
from db.api import DBConnection
|
|
from utils import pg_client
|
|
from utils.worker import WorkerPool
|
|
|
|
|
|
def main():
|
|
DATABASE = config('CLOUD_SERVICE')
|
|
database_api = DBConnection(DATABASE)
|
|
|
|
allowed_projects = config('PROJECT_IDS', default=None, cast=Csv(int))
|
|
w_pool = WorkerPool(n_workers=60,
|
|
project_filter=allowed_projects)
|
|
try:
|
|
w_pool.load_checkpoint(database_api)
|
|
except Exception as e:
|
|
print('[WARN] Checkpoint not found')
|
|
print(repr(e))
|
|
# ssl_protocol = config('KAFKA_USE_SSL', default=True, cast=bool)
|
|
# consumer_settings = {
|
|
# "bootstrap.servers": config('KAFKA_SERVERS'),
|
|
# "group.id": f"connector_{DATABASE}",
|
|
# "auto.offset.reset": "earliest",
|
|
# "enable.auto.commit": False
|
|
# }
|
|
# if ssl_protocol:
|
|
# consumer_settings['security.protocol'] = 'SSL'
|
|
# consumer = Consumer(consumer_settings)
|
|
|
|
# consumer.subscribe(config("TOPICS", default="saas-raw").split(','))
|
|
print("[INFO] Kafka consumer subscribed")
|
|
|
|
# w_pool.run_workers(kafka_consumer=consumer, database_api=database_api)
|
|
w_pool.run_workers(database_api=database_api)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
asyncio.run(pg_client.init())
|
|
main()
|
|
|