diff --git a/api/build.sh b/api/build.sh index 8c735ff3c..ba160e6ee 100644 --- a/api/build.sh +++ b/api/build.sh @@ -40,3 +40,7 @@ check_prereq build_api $1 echo buil_complete IMAGE_TAG=$IMAGE_TAG PUSH_IMAGE=$PUSH_IMAGE DOCKER_REPO=$DOCKER_REPO bash build_alerts.sh $1 + +[[ $1 == "ee" ]] && { + IMAGE_TAG=$IMAGE_TAG PUSH_IMAGE=$PUSH_IMAGE DOCKER_REPO=$DOCKER_REPO bash build_crons.sh $1 +} \ No newline at end of file diff --git a/api/routers/crons/core_crons.py b/api/routers/crons/core_crons.py index 5643ce1a6..aa9ce100f 100644 --- a/api/routers/crons/core_crons.py +++ b/api/routers/crons/core_crons.py @@ -1,15 +1,3 @@ -from chalicelib.core import weekly_report, jobs - - -async def run_scheduled_jobs() -> None: - jobs.execute_jobs() - - -async def weekly_report2() -> None: - weekly_report.cron() - - cron_jobs = [ - {"func": run_scheduled_jobs, "trigger": "interval", "seconds": 60, "misfire_grace_time": 20}, - {"func": weekly_report2, "trigger": "cron", "day_of_week": "mon", "hour": 5, "misfire_grace_time": 60 * 60} + ] diff --git a/api/routers/crons/core_dynamic_crons.py b/api/routers/crons/core_dynamic_crons.py index 78d91856d..3e4df3825 100644 --- a/api/routers/crons/core_dynamic_crons.py +++ b/api/routers/crons/core_dynamic_crons.py @@ -1,10 +1,21 @@ from chalicelib.core import telemetry +from chalicelib.core import weekly_report, jobs -def telemetry_cron() -> None: +async def run_scheduled_jobs() -> None: + jobs.execute_jobs() + + +async def weekly_report2() -> None: + weekly_report.cron() + + +async def telemetry_cron() -> None: telemetry.compute() cron_jobs = [ - {"func": telemetry_cron, "trigger": "cron", "day_of_week": "*"} + {"func": telemetry_cron, "trigger": "cron", "day_of_week": "*"}, + {"func": run_scheduled_jobs, "trigger": "interval", "seconds": 60, "misfire_grace_time": 20}, + {"func": weekly_report2, "trigger": "cron", "day_of_week": "mon", "hour": 5, "misfire_grace_time": 60 * 60} ] diff --git a/ee/api/Dockerfile.crons b/ee/api/Dockerfile.crons new file mode 100644 index 000000000..2ba3c92c0 --- /dev/null +++ b/ee/api/Dockerfile.crons @@ -0,0 +1,27 @@ +FROM python:3.9.12-slim +LABEL Maintainer="Rajesh Rajendran" +LABEL Maintainer="KRAIEM Taha Yassine" +ENV APP_NAME crons +ENV pg_minconn 2 +ENV pg_maxconn 10 +RUN apt-get update && apt-get install -y pkg-config libxmlsec1-dev gcc && rm -rf /var/lib/apt/lists/* +# Add Tini +# Startup daemon +ENV TINI_VERSION v0.19.0 +ARG envarg +ENV ENTERPRISE_BUILD ${envarg} +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini + +WORKDIR /work_tmp +COPY requirements.txt /work_tmp/requirements.txt +RUN pip install -r /work_tmp/requirements.txt +COPY sourcemap-reader/*.json /work_tmp/ +RUN cd /work_tmp && npm install + +WORKDIR /work +COPY . . +RUN mv env.default .env && mv /work_tmp/node_modules sourcemap-reader/. + +ENTRYPOINT ["/tini", "--"] +CMD ./entrypoint.sh diff --git a/ee/api/app_crons.py b/ee/api/app_crons.py new file mode 100644 index 000000000..3dfea4fe4 --- /dev/null +++ b/ee/api/app_crons.py @@ -0,0 +1,19 @@ +print("============= CRONS =============") +import sys + +from routers.crons import core_dynamic_crons + + +def process(action): + { + "TELEMETRY": core_dynamic_crons.telemetry_cron, + "JOB": core_dynamic_crons.run_scheduled_jobs, + "REPORT": core_dynamic_crons.weekly_report2 + }.get(action.upper(), lambda: print(f"{action} not found in crons-definitions"))() + + +if __name__ == '__main__': + if len(sys.argv) < 2 or len(sys.argv[1]) < 1: + print("please provide actions as argument") + else: + process(sys.argv[1]) diff --git a/ee/api/build_crons.sh b/ee/api/build_crons.sh new file mode 100644 index 000000000..4f1cca1c6 --- /dev/null +++ b/ee/api/build_crons.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Script to build crons module +# flags to accept: +# envarg: build for enterprise edition. +# Default will be OSS build. + +# Usage: IMAGE_TAG=latest DOCKER_REPO=myDockerHubID bash build.sh +function make_submodule() { + # -- this part was generated by modules_lister.py -- + mkdir crons + cp -R ./{app_crons,schemas,schemas_ee}.py ./crons/ + mkdir -p ./crons/routers/crons/ + cp -R ./routers/crons/{__init__,core_dynamic_crons}.py ./crons/routers/crons/ + mkdir -p ./crons/chalicelib/ + cp -R ./chalicelib/__init__.py ./crons/chalicelib/ + mkdir -p ./crons/chalicelib/core/ + cp -R ../chalicelib/core/{__init__,telemetry,license,unlock,weekly_report,jobs,sessions,events,issues,sessions_metas,metadata,projects,users,authorizers,tenants,roles,assist,events_ios,sessions_mobs,errors,metrics,sourcemaps,sourcemaps_parser,resources,performance_event}.py ../crons/chalicelib/core/ + mkdir -p ./crons/chalicelib/utils/ + cp -R ../chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,email_helper,email_handler,smtp,s3,args_transformer,ch_client,SAML2_helper,metrics_helper}.py ../crons/chalicelib/utils/ + # -- end of generated part + + cp -R ./{Dockerfile.crons,requirements.txt,.env.default,entrypoint_crons.sh} ./crons/ + cp -R ./chalicelib/utils/html ./crons/chalicelib/utils/html +} + +git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)} +envarg="default-foss" +check_prereq() { + which docker || { + echo "Docker not installed, please install docker." + exit=1 + } + [[ exit -eq 1 ]] && exit 1 +} + +function build_api(){ + tag="" + # Copy enterprise code + + cp -rf ../ee/api/* ./ + envarg="default-ee" + tag="ee-" + + make_submodule $1 + cd crons + docker build -f ./Dockerfile.crons --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/crons:${git_sha1} . + cd .. + rm -rf crons + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/crons:${git_sha1} + docker tag ${DOCKER_REPO:-'local'}/crons:${git_sha1} ${DOCKER_REPO:-'local'}/crons:${tag}latest + docker push ${DOCKER_REPO:-'local'}/crons:${tag}latest + } +echo "completed crons build" +} + +check_prereq +build_api $1 diff --git a/ee/api/entrypoint_crons.sh b/ee/api/entrypoint_crons.sh new file mode 100755 index 000000000..71078a19d --- /dev/null +++ b/ee/api/entrypoint_crons.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python app_crons.py $ACTION diff --git a/ee/api/routers/crons/core_dynamic_crons.py b/ee/api/routers/crons/core_dynamic_crons.py index bdde42a15..35d92ebbe 100644 --- a/ee/api/routers/crons/core_dynamic_crons.py +++ b/ee/api/routers/crons/core_dynamic_crons.py @@ -1,7 +1,17 @@ from chalicelib.core import telemetry, unlock +from chalicelib.core import weekly_report, jobs +from decouple import config -def telemetry_cron() -> None: +async def run_scheduled_jobs() -> None: + jobs.execute_jobs() + + +async def weekly_report2() -> None: + weekly_report.cron() + + +async def telemetry_cron() -> None: telemetry.compute() @@ -13,6 +23,11 @@ def unlock_cron() -> None: cron_jobs = [ - {"func": telemetry_cron, "trigger": "cron", "day_of_week": "*"}, {"func": unlock_cron, "trigger": "cron", "hour": "*"} ] + +if config("LOCAL_CRONS", default=False, cast=bool): + cron_jobs += [{"func": telemetry_cron, "trigger": "cron", "day_of_week": "*"}, + {"func": run_scheduled_jobs, "trigger": "interval", "seconds": 60, "misfire_grace_time": 20}, + {"func": weekly_report2, "trigger": "cron", "day_of_week": "mon", "hour": 5, + "misfire_grace_time": 60 * 60}]