Merge remote-tracking branch 'origin/api-v1.7.0' into dev

api/build.sh

@@ -40,3 +40,7 @@ check_prereq
 build_api $1
 echo buil_complete
 IMAGE_TAG=$IMAGE_TAG PUSH_IMAGE=$PUSH_IMAGE DOCKER_REPO=$DOCKER_REPO bash build_alerts.sh $1
+
+[[ $1 == "ee" ]] && {
+  IMAGE_TAG=$IMAGE_TAG PUSH_IMAGE=$PUSH_IMAGE DOCKER_REPO=$DOCKER_REPO bash build_crons.sh $1
+}

api/routers/crons/core_crons.py

@@ -1,15 +1,3 @@
-from chalicelib.core import weekly_report, jobs
-
-
-async def run_scheduled_jobs() -> None:
-    jobs.execute_jobs()
-
-
-async def weekly_report2() -> None:
-    weekly_report.cron()
-
-
 cron_jobs = [
-    {"func": run_scheduled_jobs, "trigger": "interval", "seconds": 60, "misfire_grace_time": 20},
-    {"func": weekly_report2, "trigger": "cron", "day_of_week": "mon", "hour": 5, "misfire_grace_time": 60 * 60}
+
 ]

api/routers/crons/core_dynamic_crons.py

@@ -1,10 +1,21 @@
 from chalicelib.core import telemetry
+from chalicelib.core import weekly_report, jobs
 
 
-def telemetry_cron() -> None:
+async def run_scheduled_jobs() -> None:
+    jobs.execute_jobs()
+
+
+async def weekly_report2() -> None:
+    weekly_report.cron()
+
+
+async def telemetry_cron() -> None:
     telemetry.compute()
 
 
 cron_jobs = [
-    {"func": telemetry_cron, "trigger": "cron", "day_of_week": "*"}
+    {"func": telemetry_cron, "trigger": "cron", "day_of_week": "*"},
+    {"func": run_scheduled_jobs, "trigger": "interval", "seconds": 60, "misfire_grace_time": 20},
+    {"func": weekly_report2, "trigger": "cron", "day_of_week": "mon", "hour": 5, "misfire_grace_time": 60 * 60}
 ]

ee/api/Dockerfile.crons

@@ -0,0 +1,27 @@
+FROM python:3.9.12-slim
+LABEL Maintainer="Rajesh Rajendran<rjshrjndrn@gmail.com>"
+LABEL Maintainer="KRAIEM Taha Yassine<tahayk2@gmail.com>"
+ENV APP_NAME crons
+ENV pg_minconn 2
+ENV pg_maxconn 10
+RUN apt-get update && apt-get install -y pkg-config libxmlsec1-dev gcc && rm -rf /var/lib/apt/lists/*
+# Add Tini
+# Startup daemon
+ENV TINI_VERSION v0.19.0
+ARG envarg
+ENV ENTERPRISE_BUILD ${envarg}
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
+RUN chmod +x /tini
+
+WORKDIR /work_tmp
+COPY requirements.txt /work_tmp/requirements.txt
+RUN pip install -r /work_tmp/requirements.txt
+COPY sourcemap-reader/*.json /work_tmp/
+RUN cd /work_tmp && npm install
+
+WORKDIR /work
+COPY . .
+RUN mv env.default .env && mv /work_tmp/node_modules sourcemap-reader/.
+
+ENTRYPOINT ["/tini", "--"]
+CMD ./entrypoint.sh

ee/api/app_crons.py

@@ -0,0 +1,19 @@
+print("============= CRONS =============")
+import sys
+
+from routers.crons import core_dynamic_crons
+
+
+def process(action):
+    {
+        "TELEMETRY": core_dynamic_crons.telemetry_cron,
+        "JOB": core_dynamic_crons.run_scheduled_jobs,
+        "REPORT": core_dynamic_crons.weekly_report2
+    }.get(action.upper(), lambda: print(f"{action} not found in crons-definitions"))()
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2 or len(sys.argv[1]) < 1:
+        print("please provide actions as argument")
+    else:
+        process(sys.argv[1])

ee/api/build_crons.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Script to build crons module
+# flags to accept:
+# envarg: build for enterprise edition.
+# Default will be OSS build.
+
+# Usage: IMAGE_TAG=latest DOCKER_REPO=myDockerHubID bash build.sh <ee>
+function make_submodule() {
+    # -- this part was generated by modules_lister.py --
+    mkdir crons
+    cp -R ./{app_crons,schemas,schemas_ee}.py ./crons/
+    mkdir -p ./crons/routers/crons/
+    cp -R ./routers/crons/{__init__,core_dynamic_crons}.py ./crons/routers/crons/
+    mkdir -p ./crons/chalicelib/
+    cp -R ./chalicelib/__init__.py ./crons/chalicelib/
+    mkdir -p ./crons/chalicelib/core/
+    cp -R ../chalicelib/core/{__init__,telemetry,license,unlock,weekly_report,jobs,sessions,events,issues,sessions_metas,metadata,projects,users,authorizers,tenants,roles,assist,events_ios,sessions_mobs,errors,metrics,sourcemaps,sourcemaps_parser,resources,performance_event}.py ../crons/chalicelib/core/
+    mkdir -p ./crons/chalicelib/utils/
+    cp -R ../chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,email_helper,email_handler,smtp,s3,args_transformer,ch_client,SAML2_helper,metrics_helper}.py ../crons/chalicelib/utils/
+    # -- end of generated part
+
+    cp -R ./{Dockerfile.crons,requirements.txt,.env.default,entrypoint_crons.sh} ./crons/
+    cp -R ./chalicelib/utils/html ./crons/chalicelib/utils/html
+}
+
+git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)}
+envarg="default-foss"
+check_prereq() {
+    which docker || {
+        echo "Docker not installed, please install docker."
+        exit=1
+    }
+    [[ exit -eq 1 ]] && exit 1
+}
+
+function build_api(){
+    tag=""
+    # Copy enterprise code
+
+    cp -rf ../ee/api/* ./
+    envarg="default-ee"
+    tag="ee-"
+
+    make_submodule $1
+    cd crons
+    docker build -f ./Dockerfile.crons --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/crons:${git_sha1} .
+    cd ..
+    rm -rf crons
+    [[ $PUSH_IMAGE -eq 1 ]] && {
+        docker push ${DOCKER_REPO:-'local'}/crons:${git_sha1}
+        docker tag ${DOCKER_REPO:-'local'}/crons:${git_sha1} ${DOCKER_REPO:-'local'}/crons:${tag}latest
+        docker push ${DOCKER_REPO:-'local'}/crons:${tag}latest
+    }
+echo "completed crons build"
+}
+
+check_prereq
+build_api $1

ee/api/entrypoint_crons.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python app_crons.py $ACTION

ee/api/routers/crons/core_dynamic_crons.py

@@ -1,7 +1,17 @@
 from chalicelib.core import telemetry, unlock
+from chalicelib.core import weekly_report, jobs
+from decouple import config
 
 
-def telemetry_cron() -> None:
+async def run_scheduled_jobs() -> None:
+    jobs.execute_jobs()
+
+
+async def weekly_report2() -> None:
+    weekly_report.cron()
+
+
+async def telemetry_cron() -> None:
     telemetry.compute()
 
 
@@ -13,6 +23,11 @@ def unlock_cron() -> None:
 
 
 cron_jobs = [
-    {"func": telemetry_cron, "trigger": "cron", "day_of_week": "*"},
     {"func": unlock_cron, "trigger": "cron", "hour": "*"}
 ]
+
+if config("LOCAL_CRONS", default=False, cast=bool):
+    cron_jobs += [{"func": telemetry_cron, "trigger": "cron", "day_of_week": "*"},
+                  {"func": run_scheduled_jobs, "trigger": "interval", "seconds": 60, "misfire_grace_time": 20},
+                  {"func": weekly_report2, "trigger": "cron", "day_of_week": "mon", "hour": 5,
+                   "misfire_grace_time": 60 * 60}]