Merge remote-tracking branch 'origin/api-v1.10.0' into dev

.github/workflows/alerts-ee.yaml

@@ -10,8 +10,21 @@ on:
     branches:
       - api-v1.10.0
     paths:
-      - ee/api/**
-      - api/**
+      - "ee/api/**"
+      - "api/**"
+      - "!api/.gitignore"
+      - "!api/routers"
+      - "!api/app.py"
+      - "!api/*-dev.sh"
+      - "!api/requirements.txt"
+      - "!api/requirements-crons.txt"
+      - "!ee/api/.gitignore"
+      - "!ee/api/routers"
+      - "!ee/api/app.py"
+      - "!ee/api/*-dev.sh"
+      - "!ee/api/requirements.txt"
+      - "!ee/api/requirements-crons.txt"
+
 
 name: Build and Deploy Alerts EE
 
@@ -21,115 +34,115 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        # We need to diff with old commit 
-        # to see which workers got changed.
-        fetch-depth: 2
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          # We need to diff with old commit
+          # to see which workers got changed.
+          fetch-depth: 2
 
-    - name: Docker login
-      run: |
-        docker login ${{ secrets.EE_REGISTRY_URL }} -u ${{ secrets.EE_DOCKER_USERNAME }} -p "${{ secrets.EE_REGISTRY_TOKEN }}" 
+      - name: Docker login
+        run: |
+          docker login ${{ secrets.EE_REGISTRY_URL }} -u ${{ secrets.EE_DOCKER_USERNAME }} -p "${{ secrets.EE_REGISTRY_TOKEN }}" 
 
-    - uses: azure/k8s-set-context@v1
-      with:
-        method: kubeconfig
-        kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret.
-      id: setcontext
+      - uses: azure/k8s-set-context@v1
+        with:
+          method: kubeconfig
+          kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret.
+        id: setcontext
 
-    # Caching docker images
-    - uses: satackey/action-docker-layer-caching@v0.0.11
-      # Ignore the failure of a step and avoid terminating the job.
-      continue-on-error: true
+      # Caching docker images
+      - uses: satackey/action-docker-layer-caching@v0.0.11
+        # Ignore the failure of a step and avoid terminating the job.
+        continue-on-error: true
 
 
-    - name: Building and Pushing api image
-      id: build-image
-      env:
-        DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
-        IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee
-        ENVIRONMENT: staging
-      run: |
-        skip_security_checks=${{ github.event.inputs.skip_security_checks }}
-        cd api
-        PUSH_IMAGE=0 bash -x ./build_alerts.sh ee
-        [[ "x$skip_security_checks" == "xtrue" ]]  || {
-          curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ 
+      - name: Building and Pushing api image
+        id: build-image
+        env:
+          DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
+          IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee
+          ENVIRONMENT: staging
+        run: |
+          skip_security_checks=${{ github.event.inputs.skip_security_checks }}
+          cd api
+          PUSH_IMAGE=0 bash -x ./build_alerts.sh ee
+          [[ "x$skip_security_checks" == "xtrue" ]]  || {
+            curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ 
+            images=("alerts")
+            for image in ${images[*]};do
+              ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL"  --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG 
+            done
+            err_code=$?
+            [[ $err_code -ne 0 ]] && {
+              exit $err_code
+            }
+          } && {
+            echo "Skipping Security Checks"
+          }
           images=("alerts")
           for image in ${images[*]};do
-            ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL"  --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG 
+            docker push $DOCKER_REPO/$image:$IMAGE_TAG 
+          done
+      - name: Creating old image input
+        run: |
+          #
+          # Create yaml with existing image tags
+          #
+          kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\
+          tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt
+          
+          echo > /tmp/image_override.yaml
+          
+          for line in `cat /tmp/image_tag.txt`;
+          do
+              image_array=($(echo "$line" | tr ':' '\n'))
+              cat <<EOF >> /tmp/image_override.yaml
+          ${image_array[0]}:
+            image:
+              # We've to strip off the -ee, as helm will append it.
+              tag: `echo ${image_array[1]} | cut -d '-' -f 1`
+          EOF
           done
-          err_code=$?
-          [[ $err_code -ne 0 ]] && {
-            exit $err_code
-          }
-        } && {
-          echo "Skipping Security Checks"
-        }
-        images=("alerts")
-        for image in ${images[*]};do
-          docker push $DOCKER_REPO/$image:$IMAGE_TAG 
-        done
-    - name: Creating old image input
-      run: |
-        #
-        # Create yaml with existing image tags
-        #
-        kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\
-        tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt
 
-        echo > /tmp/image_override.yaml
+      - name: Deploy to kubernetes
+        run: |
+          cd scripts/helmcharts/
+          
+          ## Update secerts
+          sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml
+          sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"${{ secrets.EE_PG_PASSWORD }}\"/g" vars.yaml
+          sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"${{ secrets.EE_MINIO_ACCESS_KEY }}\"/g" vars.yaml
+          sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"${{ secrets.EE_MINIO_SECRET_KEY }}\"/g" vars.yaml
+          sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"${{ secrets.EE_JWT_SECRET }}\"/g" vars.yaml
+          sed -i "s/domainName: \"\"/domainName: \"${{ secrets.EE_DOMAIN_NAME }}\"/g" vars.yaml
+          sed -i "s/enterpriseEditionLicense: \"\"/enterpriseEditionLicense: \"${{ secrets.EE_LICENSE_KEY }}\"/g" vars.yaml
+          
+          # Update changed image tag
+          sed -i "/alerts/{n;n;n;s/.*/    tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml
+          
+          cat /tmp/image_override.yaml
+          # Deploy command
+          mv openreplay/charts/{ingress-nginx,alerts,quickwit} /tmp
+          rm -rf  openreplay/charts/*
+          mv /tmp/{ingress-nginx,alerts,quickwit} openreplay/charts/
+          helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks --kube-version=$k_version | kubectl apply -f -
+        env:
+          DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
+          # We're not passing -ee flag, because helm will add that.
+          IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
+          ENVIRONMENT: staging
 
-        for line in `cat /tmp/image_tag.txt`;
-        do
-            image_array=($(echo "$line" | tr ':' '\n'))
-            cat <<EOF >> /tmp/image_override.yaml
-        ${image_array[0]}:
-          image:
-            # We've to strip off the -ee, as helm will append it.
-            tag: `echo ${image_array[1]} | cut -d '-' -f 1`
-        EOF
-        done
-
-    - name: Deploy to kubernetes
-      run: |
-        cd scripts/helmcharts/
-
-        ## Update secerts
-        sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml
-        sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"${{ secrets.EE_PG_PASSWORD }}\"/g" vars.yaml
-        sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"${{ secrets.EE_MINIO_ACCESS_KEY }}\"/g" vars.yaml
-        sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"${{ secrets.EE_MINIO_SECRET_KEY }}\"/g" vars.yaml
-        sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"${{ secrets.EE_JWT_SECRET }}\"/g" vars.yaml
-        sed -i "s/domainName: \"\"/domainName: \"${{ secrets.EE_DOMAIN_NAME }}\"/g" vars.yaml
-        sed -i "s/enterpriseEditionLicense: \"\"/enterpriseEditionLicense: \"${{ secrets.EE_LICENSE_KEY }}\"/g" vars.yaml
-
-        # Update changed image tag
-        sed -i "/alerts/{n;n;n;s/.*/    tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml
-
-        cat /tmp/image_override.yaml
-        # Deploy command
-        mv openreplay/charts/{ingress-nginx,alerts,quickwit} /tmp
-        rm -rf  openreplay/charts/*
-        mv /tmp/{ingress-nginx,alerts,quickwit} openreplay/charts/
-        helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks --kube-version=$k_version | kubectl apply -f -
-      env:
-        DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
-        # We're not passing -ee flag, because helm will add that.
-        IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
-        ENVIRONMENT: staging
-
-    - name: Alert slack
-      if: ${{ failure() }}
-      uses: rtCamp/action-slack-notify@v2
-      env:
-        SLACK_CHANNEL: ee
-        SLACK_TITLE: "Failed ${{ github.workflow }}"
-        SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff'
-        SLACK_WEBHOOK: ${{ secrets.SLACK_WEB_HOOK }}
-        SLACK_USERNAME: "OR Bot"
-        SLACK_MESSAGE: 'Build failed :bomb:'
+      - name: Alert slack
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_CHANNEL: ee
+          SLACK_TITLE: "Failed ${{ github.workflow }}"
+          SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff'
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEB_HOOK }}
+          SLACK_USERNAME: "OR Bot"
+          SLACK_MESSAGE: 'Build failed :bomb:'
 
 #    - name: Debug Job
 #      # if: ${{ failure() }}

.github/workflows/alerts.yaml

@@ -10,7 +10,13 @@ on:
     branches:
       - api-v1.10.0
     paths:
-      - api/**
+      - "api/**"
+      - "!api/.gitignore"
+      - "!api/routers"
+      - "!api/app.py"
+      - "!api/*-dev.sh"
+      - "!api/requirements.txt"
+      - "!api/requirements-crons.txt"
 
 name: Build and Deploy Alerts
 
@@ -20,112 +26,112 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        # We need to diff with old commit 
-        # to see which workers got changed.
-        fetch-depth: 2
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          # We need to diff with old commit
+          # to see which workers got changed.
+          fetch-depth: 2
 
-    - name: Docker login
-      run: |
-        docker login ${{ secrets.OSS_REGISTRY_URL }} -u ${{ secrets.OSS_DOCKER_USERNAME }} -p "${{ secrets.OSS_REGISTRY_TOKEN }}" 
+      - name: Docker login
+        run: |
+          docker login ${{ secrets.OSS_REGISTRY_URL }} -u ${{ secrets.OSS_DOCKER_USERNAME }} -p "${{ secrets.OSS_REGISTRY_TOKEN }}" 
 
-    - uses: azure/k8s-set-context@v1
-      with:
-        method: kubeconfig
-        kubeconfig: ${{ secrets.OSS_KUBECONFIG }} # Use content of kubeconfig in secret.
-      id: setcontext
+      - uses: azure/k8s-set-context@v1
+        with:
+          method: kubeconfig
+          kubeconfig: ${{ secrets.OSS_KUBECONFIG }} # Use content of kubeconfig in secret.
+        id: setcontext
 
-    # Caching docker images
-    - uses: satackey/action-docker-layer-caching@v0.0.11
-      # Ignore the failure of a step and avoid terminating the job.
-      continue-on-error: true
+      # Caching docker images
+      - uses: satackey/action-docker-layer-caching@v0.0.11
+        # Ignore the failure of a step and avoid terminating the job.
+        continue-on-error: true
 
 
-    - name: Building and Pushing Alerts image
-      id: build-image
-      env:
-        DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}
-        IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
-        ENVIRONMENT: staging
-      run: |
-        skip_security_checks=${{ github.event.inputs.skip_security_checks }}
-        cd api
-        PUSH_IMAGE=0 bash -x ./build_alerts.sh
-        [[ "x$skip_security_checks" == "xtrue" ]]  || {
-          curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ 
+      - name: Building and Pushing Alerts image
+        id: build-image
+        env:
+          DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}
+          IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
+          ENVIRONMENT: staging
+        run: |
+          skip_security_checks=${{ github.event.inputs.skip_security_checks }}
+          cd api
+          PUSH_IMAGE=0 bash -x ./build_alerts.sh
+          [[ "x$skip_security_checks" == "xtrue" ]]  || {
+            curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ 
+            images=("alerts")
+            for image in ${images[*]};do
+              ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL"  --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG 
+            done
+            err_code=$?
+            [[ $err_code -ne 0 ]] && {
+              exit $err_code
+            }
+          } && {
+            echo "Skipping Security Checks"
+          }
           images=("alerts")
           for image in ${images[*]};do
-            ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL"  --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG 
+            docker push $DOCKER_REPO/$image:$IMAGE_TAG 
+          done
+      - name: Creating old image input
+        run: |
+          #
+          # Create yaml with existing image tags
+          #
+          kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\
+          tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt
+          
+          echo > /tmp/image_override.yaml
+          
+          for line in `cat /tmp/image_tag.txt`;
+          do
+              image_array=($(echo "$line" | tr ':' '\n'))
+              cat <<EOF >> /tmp/image_override.yaml
+          ${image_array[0]}:
+            image:
+              tag: ${image_array[1]}
+          EOF
           done
-          err_code=$?
-          [[ $err_code -ne 0 ]] && {
-            exit $err_code
-          }
-        } && {
-          echo "Skipping Security Checks"
-        }
-        images=("alerts")
-        for image in ${images[*]};do
-          docker push $DOCKER_REPO/$image:$IMAGE_TAG 
-        done
-    - name: Creating old image input
-      run: |
-        #
-        # Create yaml with existing image tags
-        #
-        kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\
-        tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt
 
-        echo > /tmp/image_override.yaml
+      - name: Deploy to kubernetes
+        run: |
+          cd scripts/helmcharts/
+          
+          ## Update secerts
+          sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml
+          sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"${{ secrets.OSS_PG_PASSWORD }}\"/g" vars.yaml
+          sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"${{ secrets.OSS_MINIO_ACCESS_KEY }}\"/g" vars.yaml
+          sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"${{ secrets.OSS_MINIO_SECRET_KEY }}\"/g" vars.yaml
+          sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"${{ secrets.OSS_JWT_SECRET }}\"/g" vars.yaml
+          sed -i "s/domainName: \"\"/domainName: \"${{ secrets.OSS_DOMAIN_NAME }}\"/g" vars.yaml
+          
+          # Update changed image tag
+          sed -i "/alerts/{n;n;s/.*/    tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml
+          
+          cat /tmp/image_override.yaml
+          # Deploy command
+          mv openreplay/charts/{ingress-nginx,alerts,quickwit} /tmp
+          rm -rf  openreplay/charts/*
+          mv /tmp/{ingress-nginx,alerts,quickwit} openreplay/charts/
+          helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks | kubectl apply -n app -f -
+        env:
+          DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}
+          IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
+          ENVIRONMENT: staging
 
-        for line in `cat /tmp/image_tag.txt`;
-        do
-            image_array=($(echo "$line" | tr ':' '\n'))
-            cat <<EOF >> /tmp/image_override.yaml
-        ${image_array[0]}:
-          image:
-            tag: ${image_array[1]}
-        EOF
-        done
-
-    - name: Deploy to kubernetes
-      run: |
-        cd scripts/helmcharts/
-
-        ## Update secerts
-        sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml
-        sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"${{ secrets.OSS_PG_PASSWORD }}\"/g" vars.yaml
-        sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"${{ secrets.OSS_MINIO_ACCESS_KEY }}\"/g" vars.yaml
-        sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"${{ secrets.OSS_MINIO_SECRET_KEY }}\"/g" vars.yaml
-        sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"${{ secrets.OSS_JWT_SECRET }}\"/g" vars.yaml
-        sed -i "s/domainName: \"\"/domainName: \"${{ secrets.OSS_DOMAIN_NAME }}\"/g" vars.yaml
-
-        # Update changed image tag
-        sed -i "/alerts/{n;n;s/.*/    tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml
-
-        cat /tmp/image_override.yaml
-        # Deploy command
-        mv openreplay/charts/{ingress-nginx,alerts,quickwit} /tmp
-        rm -rf  openreplay/charts/*
-        mv /tmp/{ingress-nginx,alerts,quickwit} openreplay/charts/
-        helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks | kubectl apply -n app -f -
-      env:
-        DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}
-        IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
-        ENVIRONMENT: staging
-
-    - name: Alert slack
-      if: ${{ failure() }}
-      uses: rtCamp/action-slack-notify@v2
-      env:
-        SLACK_CHANNEL: foss
-        SLACK_TITLE: "Failed ${{ github.workflow }}"
-        SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff'
-        SLACK_WEBHOOK: ${{ secrets.SLACK_WEB_HOOK }}
-        SLACK_USERNAME: "OR Bot"
-        SLACK_MESSAGE: 'Build failed :bomb:'
+      - name: Alert slack
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_CHANNEL: foss
+          SLACK_TITLE: "Failed ${{ github.workflow }}"
+          SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff'
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEB_HOOK }}
+          SLACK_USERNAME: "OR Bot"
+          SLACK_MESSAGE: 'Build failed :bomb:'
 
     # - name: Debug Job
     #   if: ${{ failure() }}

.github/workflows/api-ee.yaml

@@ -10,8 +10,18 @@ on:
     branches:
       - api-v1.10.0
     paths:
-      - ee/api/**
-      - api/**
+      - "ee/api/**"
+      - "api/**"
+      - "!api/.gitignore"
+      - "!api/app_alerts.py"
+      - "!api/*-dev.sh"
+      - "!api/requirements-*.txt"
+      - "!ee/api/.gitignore"
+      - "!ee/api/app_alerts.py"
+      - "!ee/api/app_crons.py"
+      - "!ee/api/*-dev.sh"
+      - "!ee/api/requirements-*.txt"
+
 
 name: Build and Deploy Chalice EE
 

.github/workflows/api.yaml

@@ -8,9 +8,13 @@ on:
         default: 'false'
   push:
     branches:
-      - dev
+      - api-v1.10.0
     paths:
-      - api/**
+      - "api/**"
+      - "!api/.gitignore"
+      - "!api/app_alerts.py"
+      - "!api/*-dev.sh"
+      - "!api/requirements-*.txt"
 
 name: Build and Deploy Chalice
 

.github/workflows/crons-ee.yaml

@@ -10,8 +10,20 @@ on:
     branches:
       - api-v1.10.0
     paths:
-      - ee/api/**
-      - api/**
+      - "ee/api/**"
+      - "api/**"
+      - "!api/.gitignore"
+      - "!api/app.py"
+      - "!api/app_alerts.py"
+      - "!api/*-dev.sh"
+      - "!api/requirements.txt"
+      - "!api/requirements-alerts.txt"
+      - "!ee/api/.gitignore"
+      - "!ee/api/app.py"
+      - "!ee/api/app_alerts.py"
+      - "!ee/api/*-dev.sh"
+      - "!ee/api/requirements.txt"
+      - "!ee/api/requirements-crons.txt"
 
 name: Build and Deploy Crons EE
 
@@ -110,9 +122,9 @@ jobs:
 
         cat /tmp/image_override.yaml
         # Deploy command
-        mv openreplay/charts/{ingress-nginx,crons,quickwit} /tmp
+        mv openreplay/charts/{ingress-nginx,utilities,quickwit} /tmp
         rm -rf  openreplay/charts/*
-        mv /tmp/{ingress-nginx,crons,quickwit} openreplay/charts/
+        mv /tmp/{ingress-nginx,utilities,quickwit} openreplay/charts/
         helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks --kube-version=$k_version | kubectl apply -f -
       env:
         DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}

.github/workflows/peers-ee.yaml

@@ -0,0 +1,69 @@
+# This action will push the peers changes to aws
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - dev
+    paths:
+      - "ee/peers/**"
+      - "peers/**"
+      - "!peers/.gitignore"
+      - "!peers/*-dev.sh"
+
+name: Build and Deploy Peers
+
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        # We need to diff with old commit 
+        # to see which workers got changed.
+        fetch-depth: 2
+
+    - name: Docker login
+      run: |
+        docker login ${{ secrets.EE_REGISTRY_URL }} -u ${{ secrets.EE_DOCKER_USERNAME }} -p "${{ secrets.EE_REGISTRY_TOKEN }}" 
+
+    - uses: azure/k8s-set-context@v1
+      with:
+        method: kubeconfig
+        kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret.
+      id: setcontext
+
+    - name: Building and Pushing api image
+      id: build-image
+      env:
+        DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
+        IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
+        ENVIRONMENT: staging
+      run: |
+        cd peers
+        PUSH_IMAGE=1 bash build.sh ee
+    - name: Deploy to kubernetes
+      run: |
+        cd scripts/helmcharts/
+        sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.EE_REGISTRY_URL }}\"#g" vars.yaml
+        sed -i "s#minio_access_key.*#minio_access_key: \"${{ secrets.EE_MINIO_ACCESS_KEY }}\" #g" vars.yaml
+        sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.EE_MINIO_SECRET_KEY }}\" #g" vars.yaml
+        sed -i "s#domain_name.*#domain_name: \"ee.openreplay.com\" #g" vars.yaml
+        sed -i "s#kubeconfig.*#kubeconfig_path: ${KUBECONFIG}#g" vars.yaml
+        sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml
+        bash kube-install.sh --app peers
+      env:
+        DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
+        IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
+        ENVIRONMENT: staging
+
+    # - name: Debug Job
+    #   if: ${{ failure() }}
+    #   uses: mxschmitt/action-tmate@v3
+    #   env:
+    #     DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
+    #     IMAGE_TAG: ${{ github.sha }}
+    #     ENVIRONMENT: staging
+    #

.github/workflows/peers.yaml

@@ -0,0 +1,68 @@
+# This action will push the peers changes to aws
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - dev
+    paths:
+      - "peers/**"
+      - "!peers/.gitignore"
+      - "!peers/*-dev.sh"
+
+name: Build and Deploy Peers
+
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        # We need to diff with old commit 
+        # to see which workers got changed.
+        fetch-depth: 2
+
+    - name: Docker login
+      run: |
+        docker login ${{ secrets.OSS_REGISTRY_URL }} -u ${{ secrets.OSS_DOCKER_USERNAME }} -p "${{ secrets.OSS_REGISTRY_TOKEN }}" 
+
+    - uses: azure/k8s-set-context@v1
+      with:
+        method: kubeconfig
+        kubeconfig: ${{ secrets.OSS_KUBECONFIG }} # Use content of kubeconfig in secret.
+      id: setcontext
+
+    - name: Building and Pushing api image
+      id: build-image
+      env:
+        DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}
+        IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
+        ENVIRONMENT: staging
+      run: |
+        cd peers
+        PUSH_IMAGE=1 bash build.sh
+    - name: Deploy to kubernetes
+      run: |
+        cd scripts/helmcharts/
+        sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml
+        sed -i "s#minio_access_key.*#minio_access_key: \"${{ secrets.OSS_MINIO_ACCESS_KEY }}\" #g" vars.yaml
+        sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.OSS_MINIO_SECRET_KEY }}\" #g" vars.yaml
+        sed -i "s#domain_name.*#domain_name: \"foss.openreplay.com\" #g" vars.yaml
+        sed -i "s#kubeconfig.*#kubeconfig_path: ${KUBECONFIG}#g" vars.yaml
+        sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml
+        bash kube-install.sh --app peers
+      env:
+        DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}
+        IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
+        ENVIRONMENT: staging
+
+    # - name: Debug Job
+    #   if: ${{ failure() }}
+    #   uses: mxschmitt/action-tmate@v3
+    #   env:
+    #     DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}
+    #     IMAGE_TAG: ${{ github.sha }}
+    #     ENVIRONMENT: staging
+    #

.github/workflows/sourcemaps-reader.yaml

@@ -5,7 +5,9 @@ on:
     branches:
       - dev
     paths:
-      - sourcemap-reader/**
+      - "sourcemap-reader/**"
+      - "!sourcemap-reader/.gitignore"
+      - "!sourcemap-reader/*-dev.sh"
 
 name: Build and Deploy sourcemap-reader
 

.github/workflows/utilities-ee.yaml

@@ -0,0 +1,69 @@
+# This action will push the utilities changes to aws
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - dev
+    paths:
+      - "ee/utilities/**"
+      - "utilities/*/**"
+      - "!utilities/.gitignore"
+      - "!utilities/*-dev.sh"
+
+name: Build and Deploy Utilities EE
+
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        # We need to diff with old commit 
+        # to see which workers got changed.
+        fetch-depth: 2
+
+    - name: Docker login
+      run: |
+        docker login ${{ secrets.EE_REGISTRY_URL }} -u ${{ secrets.EE_DOCKER_USERNAME }} -p "${{ secrets.EE_REGISTRY_TOKEN }}" 
+
+    - uses: azure/k8s-set-context@v1
+      with:
+        method: kubeconfig
+        kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret.
+      id: setcontext
+
+    - name: Building and Pushing api image
+      id: build-image
+      env:
+        DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
+        IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee
+        ENVIRONMENT: staging
+      run: |
+        cd utilities
+        PUSH_IMAGE=1 bash build.sh ee
+    - name: Deploy to kubernetes
+      run: |
+        cd scripts/helmcharts/
+        sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.EE_REGISTRY_URL }}\"#g" vars.yaml
+        sed -i "s#minio_access_key.*#minio_access_key: \"${{ secrets.EE_MINIO_ACCESS_KEY }}\" #g" vars.yaml
+        sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.EE_MINIO_SECRET_KEY }}\" #g" vars.yaml
+        sed -i "s#domain_name.*#domain_name: \"ee.openreplay.com\" #g" vars.yaml
+        sed -i "s#kubeconfig.*#kubeconfig_path: ${EE_KUBECONFIG}#g" vars.yaml
+        sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml
+        bash kube-install.sh --app utilities
+      env:
+        DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
+        IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
+        ENVIRONMENT: staging
+
+    # - name: Debug Job
+    #   if: ${{ failure() }}
+    #   uses: mxschmitt/action-tmate@v3
+    #   env:
+    #     DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
+    #     IMAGE_TAG: ${{ github.sha }}
+    #     ENVIRONMENT: staging
+    #

.github/workflows/utilities.yaml

@@ -5,7 +5,9 @@ on:
     branches:
       - dev
     paths:
-      - utilities/**
+      - "utilities/**"
+      - "!utilities/.gitignore"
+      - "!utilities/*-dev.sh"
 
 name: Build and Deploy Utilities
 

api/chalicelib/core/sessions.py

@@ -301,7 +301,7 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d
                 extra_col = ""
                 extra_where = ""
                 pre_query = ""
-                distinct_on="s.session_id"
+                distinct_on = "s.session_id"
                 if metric_of == schemas.MetricOfTable.user_country:
                     main_col = "user_country"
                 elif metric_of == schemas.MetricOfTable.user_device:
@@ -321,7 +321,7 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d
                 elif metric_of == schemas.MetricOfTable.visited_url:
                     main_col = "path"
                     extra_col = ", path"
-                    distinct_on+=",path"
+                    distinct_on += ",path"
                 main_query = cur.mogrify(f"""{pre_query}
                                              SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS values
                                                         FROM (SELECT {main_col} AS name,
@@ -1194,8 +1194,9 @@ def delete_sessions_by_user_ids(project_id, user_ids):
 
 def count_all():
     with pg_client.PostgresClient(unlimited_query=True) as cur:
-        row = cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions")
-    return row.get("count", 0)
+        cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions")
+        row = cur.fetchone()
+    return row.get("count", 0) if row else 0
 
 
 def session_exists(project_id, session_id):
@@ -1203,7 +1204,8 @@ def session_exists(project_id, session_id):
         query = cur.mogrify("""SELECT 1 
                              FROM public.sessions 
                              WHERE session_id=%(session_id)s 
-                                AND project_id=%(project_id)s""",
+                                AND project_id=%(project_id)s
+                             LIMIT 1;""",
                             {"project_id": project_id, "session_id": session_id})
         cur.execute(query)
         row = cur.fetchone()

api/chalicelib/utils/s3.py

@@ -55,7 +55,7 @@ def get_presigned_url_for_sharing(bucket, expires_in, key, check_exists=False):
     )
 
 
-def get_presigned_url_for_upload_deprecated(bucket, expires_in, key, **args):
+def get_presigned_url_for_upload(bucket, expires_in, key, **args):
     return client.generate_presigned_url(
         'put_object',
         Params={
@@ -66,10 +66,7 @@ def get_presigned_url_for_upload_deprecated(bucket, expires_in, key, **args):
     )
 
 
-
-
-
-def get_presigned_url_for_upload(bucket, expires_in, key, conditions=None, public=False, content_type=None):
+def get_presigned_url_for_upload_secure(bucket, expires_in, key, conditions=None, public=False, content_type=None):
     acl = 'private'
     if public:
         acl = 'public-read'

ee/api/.gitignore

@@ -263,5 +263,6 @@ Pipfile.lock
 /chalicelib/core/saved_search.py
 /app_alerts.py
 /build_alerts.sh
+/build_crons.sh
 /routers/subs/v1_api.py
 #exp /chalicelib/core/dashboards.py

ee/api/chalicelib/core/sessions.py

@@ -304,7 +304,7 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d
                 extra_col = ""
                 extra_where = ""
                 pre_query = ""
-                distinct_on="s.session_id"
+                distinct_on = "s.session_id"
                 if metric_of == schemas.MetricOfTable.user_country:
                     main_col = "user_country"
                 elif metric_of == schemas.MetricOfTable.user_device:
@@ -324,7 +324,7 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d
                 elif metric_of == schemas.MetricOfTable.visited_url:
                     main_col = "path"
                     extra_col = ", path"
-                    distinct_on+=",path"
+                    distinct_on += ",path"
                 main_query = cur.mogrify(f"""{pre_query}
                                              SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS values
                                                         FROM (SELECT {main_col} AS name,
@@ -1197,8 +1197,9 @@ def delete_sessions_by_user_ids(project_id, user_ids):
 
 def count_all():
     with pg_client.PostgresClient(unlimited_query=True) as cur:
-        row = cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions")
-    return row.get("count", 0)
+        cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions")
+        row = cur.fetchone()
+    return row.get("count", 0) if row else 0
 
 
 def session_exists(project_id, session_id):
@@ -1206,7 +1207,8 @@ def session_exists(project_id, session_id):
         query = cur.mogrify("""SELECT 1 
                              FROM public.sessions 
                              WHERE session_id=%(session_id)s 
-                                AND project_id=%(project_id)s""",
+                                AND project_id=%(project_id)s
+                             LIMIT 1;""",
                             {"project_id": project_id, "session_id": session_id})
         cur.execute(query)
         row = cur.fetchone()

ee/api/chalicelib/core/sessions_exp.py

@@ -202,7 +202,7 @@ def _isUndefined_operator(op: schemas.SearchEventOperator):
 
 # This function executes the query and return result
 def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False,
-                    error_status=schemas.ErrorStatus.all, count_only=False, issue=None):
+                    error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False):
     full_args, query_part = search_query_parts_ch(data=data, error_status=error_status, errors_only=errors_only,
                                                   favorite_only=data.bookmarked, issue=issue, project_id=project_id,
                                                   user_id=user_id)
@@ -264,6 +264,12 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_
                                                 GROUP BY user_id
                                             ) AS users_sessions;""",
                                      full_args)
+        elif ids_only:
+            main_query = cur.format(f"""SELECT DISTINCT ON(s.session_id) s.session_id
+                                             {query_part}
+                                             ORDER BY s.session_id desc
+                                             LIMIT %(sessions_limit)s OFFSET %(sessions_limit_s)s;""",
+                                    full_args)
         else:
             if data.order is None:
                 data.order = schemas.SortOrderType.desc.value
@@ -302,8 +308,8 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_
             print(data.json())
             print("--------------------")
             raise err
-        if errors_only:
-            return helper.list_to_camel_case(cur.fetchall())
+        if errors_only or ids_only:
+            return helper.list_to_camel_case(sessions)
 
         if len(sessions) > 0:
             sessions = sessions[0]
@@ -1170,6 +1176,9 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu
             ) {"" if or_events else (f"AS event_{event_index} " + ("ON(TRUE)" if event_index > 0 else ""))}\
             """)
             event_index += 1
+            # limit THEN-events to 7 in CH because sequenceMatch cannot take more arguments
+            if event_index == 7 and data.events_order == schemas.SearchEventOrder._then:
+                break
 
         if event_index < 2:
             data.events_order = schemas.SearchEventOrder._or
@@ -1520,17 +1529,18 @@ def delete_sessions_by_user_ids(project_id, user_ids):
 
 
 def count_all():
-    with pg_client.PostgresClient(unlimited_query=True) as cur:
-        row = cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions")
+    with ch_client.ClickHouseClient() as cur:
+        row = cur.execute(query=f"SELECT COUNT(session_id) AS count FROM {exp_ch_helper.get_main_sessions_table()}")
     return row.get("count", 0)
 
 
 def session_exists(project_id, session_id):
     with ch_client.ClickHouseClient() as cur:
-        query = cur.format("""SELECT 1 
-                         FROM public.sessions 
-                         WHERE session_id=%(session_id)s 
-                            AND project_id=%(project_id)s""",
+        query = cur.format(f"""SELECT 1 
+                               FROM {exp_ch_helper.get_main_sessions_table()} 
+                               WHERE session_id=%(session_id)s 
+                                    AND project_id=%(project_id)s
+                               LIMIT 1""",
                            {"project_id": project_id, "session_id": session_id})
         row = cur.execute(query)
     return row is not None

ee/utilities/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "utilities-server",
+  "name": "assist-server",
   "version": "1.0.0",
   "description": "assist server to get live sessions & sourcemaps reader to get stack trace",
   "main": "peerjs-server.js",

utilities/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "utilities-server",
+  "name": "assist-server",
   "version": "1.0.0",
   "description": "assist server to get live sessions & sourcemaps reader to get stack trace",
   "main": "peerjs-server.js",