From 71c9a82e93816915b42cf145208dbec4ace45956 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 16 Feb 2023 11:31:19 +0100 Subject: [PATCH 01/16] chore(actions): changing paths --- .github/workflows/utilities-ee.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/utilities-ee.yaml b/.github/workflows/utilities-ee.yaml index f9a1ac677..92270dc3d 100644 --- a/.github/workflows/utilities-ee.yaml +++ b/.github/workflows/utilities-ee.yaml @@ -6,6 +6,8 @@ on: - dev paths: - "ee/utilities/**" + - "!ee/utilities/.gitignore" + - "!ee/utilities/*-dev.sh" - "utilities/*/**" - "!utilities/.gitignore" - "!utilities/*-dev.sh" From 553955ccec7d8591375ec207deb800800bbf359a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 16 Feb 2023 11:31:48 +0100 Subject: [PATCH 02/16] chore(actions): changing branches --- .github/workflows/utilities-ee.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/utilities-ee.yaml b/.github/workflows/utilities-ee.yaml index 92270dc3d..d28f72968 100644 --- a/.github/workflows/utilities-ee.yaml +++ b/.github/workflows/utilities-ee.yaml @@ -3,7 +3,7 @@ on: workflow_dispatch: push: branches: - - dev + - api-v1.10.0 paths: - "ee/utilities/**" - "!ee/utilities/.gitignore" From 804a3cfc6addff3ff2b6dc6a3da8da3ad4bf6af9 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 16 Feb 2023 14:46:03 +0100 Subject: [PATCH 03/16] ci(actions): Adding assist GH actions Signed-off-by: rjshrjndrn --- .github/workflows/assist-ee.yaml | 120 ++++++++++++++++++++++++++++ .github/workflows/assist.yaml | 120 ++++++++++++++++++++++++++++ .github/workflows/utilities-ee.yaml | 71 ---------------- .github/workflows/utilities.yaml | 68 ---------------- 4 files changed, 240 insertions(+), 139 deletions(-) create mode 100644 .github/workflows/assist-ee.yaml create mode 100644 .github/workflows/assist.yaml delete mode 100644 .github/workflows/utilities-ee.yaml delete mode 100644 .github/workflows/utilities.yaml diff --git a/.github/workflows/assist-ee.yaml b/.github/workflows/assist-ee.yaml new file mode 100644 index 000000000..78a783dd1 --- /dev/null +++ b/.github/workflows/assist-ee.yaml @@ -0,0 +1,120 @@ +# This action will push the assist changes to aws +on: + workflow_dispatch: + push: + branches: + - dev + paths: + - "ee/utilities/**" + - "utilities/*/**" + - "!utilities/.gitignore" + - "!utilities/*-dev.sh" + +name: Build and Deploy Assist EE + +jobs: + deploy: + name: Deploy + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + # We need to diff with old commit + # to see which workers got changed. + fetch-depth: 2 + + - name: Docker login + run: | + docker login ${{ secrets.EE_REGISTRY_URL }} -u ${{ secrets.EE_DOCKER_USERNAME }} -p "${{ secrets.EE_REGISTRY_TOKEN }}" + + - uses: azure/k8s-set-context@v1 + with: + method: kubeconfig + kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret. + id: setcontext + + - name: Building and Pushing Assist image + id: build-image + env: + DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee + ENVIRONMENT: staging + run: | + skip_security_checks=${{ github.event.inputs.skip_security_checks }} + cd utilities + PUSH_IMAGE=0 bash -x ./build.sh ee + [[ "x$skip_security_checks" == "xtrue" ]] || { + curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ + images=("assist") + for image in ${images[*]};do + ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL" --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG + done + err_code=$? + [[ $err_code -ne 0 ]] && { + exit $err_code + } + } && { + echo "Skipping Security Checks" + } + images=("assist") + for image in ${images[*]};do + docker push $DOCKER_REPO/$image:$IMAGE_TAG + done + - name: Creating old image input + run: | + # + # Create yaml with existing image tags + # + kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\ + tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt + + echo > /tmp/image_override.yaml + + for line in `cat /tmp/image_tag.txt`; + do + image_array=($(echo "$line" | tr ':' '\n')) + cat <> /tmp/image_override.yaml + ${image_array[0]}: + image: + # We've to strip off the -ee, as helm will append it. + tag: `echo ${image_array[1]} | cut -d '-' -f 1` + EOF + done + - name: Deploy to kubernetes + run: | + cd scripts/helmcharts/ + + ## Update secerts + sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml + sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"${{ secrets.EE_PG_PASSWORD }}\"/g" vars.yaml + sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"${{ secrets.EE_MINIO_ACCESS_KEY }}\"/g" vars.yaml + sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"${{ secrets.EE_MINIO_SECRET_KEY }}\"/g" vars.yaml + sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"${{ secrets.EE_JWT_SECRET }}\"/g" vars.yaml + sed -i "s/domainName: \"\"/domainName: \"${{ secrets.EE_DOMAIN_NAME }}\"/g" vars.yaml + sed -i "s/enterpriseEditionLicense: \"\"/enterpriseEditionLicense: \"${{ secrets.EE_LICENSE_KEY }}\"/g" vars.yaml + + # Update changed image tag + sed -i "/assist/{n;n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + + cat /tmp/image_override.yaml + # Deploy command + mv openreplay/charts/{ingress-nginx,chalice,quickwit} /tmp + rm -rf openreplay/charts/* + mv /tmp/{ingress-nginx,chalice,quickwit} openreplay/charts/ + helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks --kube-version=$k_version | kubectl apply -f - + env: + DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} + # We're not passing -ee flag, because helm will add that. + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} + ENVIRONMENT: staging + + # - name: Debug Job + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 + # env: + # DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} + # IMAGE_TAG: ${{ github.sha }} + # ENVIRONMENT: staging + # diff --git a/.github/workflows/assist.yaml b/.github/workflows/assist.yaml new file mode 100644 index 000000000..cf4d184cf --- /dev/null +++ b/.github/workflows/assist.yaml @@ -0,0 +1,120 @@ +# This action will push the assist changes to aws +on: + workflow_dispatch: + push: + branches: + - dev + paths: + - "ee/utilities/**" + - "utilities/*/**" + - "!utilities/.gitignore" + - "!utilities/*-dev.sh" + +name: Build and Deploy Assist EE + +jobs: + deploy: + name: Deploy + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + # We need to diff with old commit + # to see which workers got changed. + fetch-depth: 2 + + - name: Docker login + run: | + docker login ${{ secrets.OSS_REGISTRY_URL }} -u ${{ secrets.OSS_DOCKER_USERNAME }} -p "${{ secrets.OSS_REGISTRY_TOKEN }}" + + - uses: azure/k8s-set-context@v1 + with: + method: kubeconfig + kubeconfig: ${{ secrets.OSS_KUBECONFIG }} # Use content of kubeconfig in secret. + id: setcontext + + - name: Building and Pushing Assist image + id: build-image + env: + DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee + ENVIRONMENT: staging + run: | + skip_security_checks=${{ github.event.inputs.skip_security_checks }} + cd utilities + PUSH_IMAGE=0 bash -x ./build.sh ee + [[ "x$skip_security_checks" == "xtrue" ]] || { + curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ + images=("assist") + for image in ${images[*]};do + ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL" --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG + done + err_code=$? + [[ $err_code -ne 0 ]] && { + exit $err_code + } + } && { + echo "Skipping Security Checks" + } + images=("assist") + for image in ${images[*]};do + docker push $DOCKER_REPO/$image:$IMAGE_TAG + done + - name: Creating old image input + run: | + # + # Create yaml with existing image tags + # + kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\ + tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt + + echo > /tmp/image_override.yaml + + for line in `cat /tmp/image_tag.txt`; + do + image_array=($(echo "$line" | tr ':' '\n')) + cat <> /tmp/image_override.yaml + ${image_array[0]}: + image: + # We've to strip off the -ee, as helm will append it. + tag: `echo ${image_array[1]} | cut -d '-' -f 1` + EOF + done + - name: Deploy to kubernetes + run: | + cd scripts/helmcharts/ + + ## Update secerts + sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml + sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"${{ secrets.OSS_PG_PASSWORD }}\"/g" vars.yaml + sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"${{ secrets.OSS_MINIO_ACCESS_KEY }}\"/g" vars.yaml + sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"${{ secrets.OSS_MINIO_SECRET_KEY }}\"/g" vars.yaml + sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"${{ secrets.OSS_JWT_SECRET }}\"/g" vars.yaml + sed -i "s/domainName: \"\"/domainName: \"${{ secrets.OSS_DOMAIN_NAME }}\"/g" vars.yaml + sed -i "s/enterpriseEditionLicense: \"\"/enterpriseEditionLicense: \"${{ secrets.OSS_LICENSE_KEY }}\"/g" vars.yaml + + # Update changed image tag + sed -i "/assist/{n;n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + + cat /tmp/image_override.yaml + # Deploy command + mv openreplay/charts/{ingress-nginx,chalice,quickwit} /tmp + rm -rf openreplay/charts/* + mv /tmp/{ingress-nginx,chalice,quickwit} openreplay/charts/ + helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks --kube-version=$k_version | kubectl apply -f - + env: + DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} + # We're not passing -ee flag, because helm will add that. + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} + ENVIRONMENT: staging + + # - name: Debug Job + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 + # env: + # DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} + # IMAGE_TAG: ${{ github.sha }} + # ENVIRONMENT: staging + # diff --git a/.github/workflows/utilities-ee.yaml b/.github/workflows/utilities-ee.yaml deleted file mode 100644 index d28f72968..000000000 --- a/.github/workflows/utilities-ee.yaml +++ /dev/null @@ -1,71 +0,0 @@ -# This action will push the assist changes to aws -on: - workflow_dispatch: - push: - branches: - - api-v1.10.0 - paths: - - "ee/utilities/**" - - "!ee/utilities/.gitignore" - - "!ee/utilities/*-dev.sh" - - "utilities/*/**" - - "!utilities/.gitignore" - - "!utilities/*-dev.sh" - -name: Build and Deploy Assist EE - -jobs: - deploy: - name: Deploy - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - # We need to diff with old commit - # to see which workers got changed. - fetch-depth: 2 - - - name: Docker login - run: | - docker login ${{ secrets.EE_REGISTRY_URL }} -u ${{ secrets.EE_DOCKER_USERNAME }} -p "${{ secrets.EE_REGISTRY_TOKEN }}" - - - uses: azure/k8s-set-context@v1 - with: - method: kubeconfig - kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret. - id: setcontext - - - name: Building and Pushing api image - id: build-image - env: - DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} - IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee - ENVIRONMENT: staging - run: | - cd utilities - PUSH_IMAGE=1 bash build.sh ee - - name: Deploy to kubernetes - run: | - cd scripts/helmcharts/ - sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.EE_REGISTRY_URL }}\"#g" vars.yaml - sed -i "s#minio_access_key.*#minio_access_key: \"${{ secrets.EE_MINIO_ACCESS_KEY }}\" #g" vars.yaml - sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.EE_MINIO_SECRET_KEY }}\" #g" vars.yaml - sed -i "s#domain_name.*#domain_name: \"ee.openreplay.com\" #g" vars.yaml - sed -i "s#kubeconfig.*#kubeconfig_path: ${EE_KUBECONFIG}#g" vars.yaml - sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml - bash kube-install.sh --app utilities - env: - DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} - IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} - ENVIRONMENT: staging - - # - name: Debug Job - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v3 - # env: - # DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} - # IMAGE_TAG: ${{ github.sha }} - # ENVIRONMENT: staging - # diff --git a/.github/workflows/utilities.yaml b/.github/workflows/utilities.yaml deleted file mode 100644 index 7d2792d9b..000000000 --- a/.github/workflows/utilities.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# This action will push the assist changes to aws -on: - workflow_dispatch: - push: - branches: - - api-v1.10.0 - paths: - - "utilities/**" - - "!utilities/.gitignore" - - "!utilities/*-dev.sh" - -name: Build and Deploy Assist - -jobs: - deploy: - name: Deploy - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - # We need to diff with old commit - # to see which workers got changed. - fetch-depth: 2 - - - name: Docker login - run: | - docker login ${{ secrets.OSS_REGISTRY_URL }} -u ${{ secrets.OSS_DOCKER_USERNAME }} -p "${{ secrets.OSS_REGISTRY_TOKEN }}" - - - uses: azure/k8s-set-context@v1 - with: - method: kubeconfig - kubeconfig: ${{ secrets.OSS_KUBECONFIG }} # Use content of kubeconfig in secret. - id: setcontext - - - name: Building and Pushing api image - id: build-image - env: - DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} - IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} - ENVIRONMENT: staging - run: | - cd utilities - PUSH_IMAGE=1 bash build.sh - - name: Deploy to kubernetes - run: | - cd scripts/helmcharts/ - sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml - sed -i "s#minio_access_key.*#minio_access_key: \"${{ secrets.OSS_MINIO_ACCESS_KEY }}\" #g" vars.yaml - sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.OSS_MINIO_SECRET_KEY }}\" #g" vars.yaml - sed -i "s#domain_name.*#domain_name: \"foss.openreplay.com\" #g" vars.yaml - sed -i "s#kubeconfig.*#kubeconfig_path: ${KUBECONFIG}#g" vars.yaml - sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml - bash kube-install.sh --app utilities - env: - DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} - IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} - ENVIRONMENT: staging - - # - name: Debug Job - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v3 - # env: - # DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} - # IMAGE_TAG: ${{ github.sha }} - # ENVIRONMENT: staging - # From 3d31bab0600d3fbf5b2dd6e8b723c94412f59508 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 16 Feb 2023 17:11:25 +0100 Subject: [PATCH 04/16] feat(assist): support missing protocol --- ee/utilities/servers/websocket-cluster.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/utilities/servers/websocket-cluster.js b/ee/utilities/servers/websocket-cluster.js index 6aa2bade5..77cfd5067 100644 --- a/ee/utilities/servers/websocket-cluster.js +++ b/ee/utilities/servers/websocket-cluster.js @@ -24,7 +24,7 @@ const { const {createAdapter} = require("@socket.io/redis-adapter"); const {createClient} = require("redis"); const wsRouter = express.Router(); -const REDIS_URL = process.env.REDIS_URL || "redis://localhost:6379"; +const REDIS_URL = (process.env.REDIS_URL || "localhost:6379").replace(/((^\w+:|^)\/\/|^)/, 'redis://'); const pubClient = createClient({url: REDIS_URL}); const subClient = pubClient.duplicate(); console.log(`Using Redis: ${REDIS_URL}`); From 161a84b86d3ab8b6053aa2ff36fd7c84dce0a929 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 16 Feb 2023 18:25:58 +0100 Subject: [PATCH 05/16] feat(assist): upgrade changes --- ee/utilities/.gitignore | 1 - ee/utilities/run-dev.sh | 6 ++++++ ee/utilities/servers/websocket-cluster.js | 5 +++-- ee/utilities/servers/websocket.js | 2 +- utilities/servers/websocket.js | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) create mode 100755 ee/utilities/run-dev.sh diff --git a/ee/utilities/.gitignore b/ee/utilities/.gitignore index 8c9dca279..cd68b1ffb 100644 --- a/ee/utilities/.gitignore +++ b/ee/utilities/.gitignore @@ -15,5 +15,4 @@ servers/sourcemaps-server.js /utils/helper.js /utils/assistHelper.js .local -run-dev.sh *.mmdb diff --git a/ee/utilities/run-dev.sh b/ee/utilities/run-dev.sh new file mode 100755 index 000000000..00e8d5a4b --- /dev/null +++ b/ee/utilities/run-dev.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -a +source .env +set +a + +npm start \ No newline at end of file diff --git a/ee/utilities/servers/websocket-cluster.js b/ee/utilities/servers/websocket-cluster.js index 77cfd5067..fef572a52 100644 --- a/ee/utilities/servers/websocket-cluster.js +++ b/ee/utilities/servers/websocket-cluster.js @@ -283,7 +283,7 @@ module.exports = { wsRouter, start: (server, prefix) => { createSocketIOServer(server, prefix); - io.use(async (socket, next) => await authorizer.check(socket, next)); + // io.use(async (socket, next) => await authorizer.check(socket, next)); io.on('connection', async (socket) => { socket.on(EVENTS_DEFINITION.listen.ERROR, err => errorHandler(EVENTS_DEFINITION.listen.ERROR, err)); debug && console.log(`WS started:${socket.id}, Query:${JSON.stringify(socket.handshake.query)}`); @@ -309,7 +309,8 @@ module.exports = { debug && console.log(`notifying new agent about no SESSIONS`); io.to(socket.id).emit(EVENTS_DEFINITION.emit.NO_SESSIONS); } - await io.of('/').adapter.remoteJoin(socket.id, socket.peerId); + // await io.of('/').adapter.join(socket.id, socket.peerId); + await socket.join(socket.peerId); let rooms = await io.of('/').adapter.allRooms(); if (rooms.has(socket.peerId)) { let connectedSockets = await io.in(socket.peerId).fetchSockets(); diff --git a/ee/utilities/servers/websocket.js b/ee/utilities/servers/websocket.js index bf65789f2..c906b5987 100644 --- a/ee/utilities/servers/websocket.js +++ b/ee/utilities/servers/websocket.js @@ -287,7 +287,7 @@ module.exports = { debug && console.log(`notifying new agent about no SESSIONS`); io.to(socket.id).emit(EVENTS_DEFINITION.emit.NO_SESSIONS); } - socket.join(socket.peerId); + await socket.join(socket.peerId); if (io.sockets.adapter.rooms.get(socket.peerId)) { debug && console.log(`${socket.id} joined room:${socket.peerId}, as:${socket.identity}, members:${io.sockets.adapter.rooms.get(socket.peerId).size}`); } diff --git a/utilities/servers/websocket.js b/utilities/servers/websocket.js index 615390996..f5d029bc2 100644 --- a/utilities/servers/websocket.js +++ b/utilities/servers/websocket.js @@ -268,7 +268,7 @@ module.exports = { debug && console.log(`notifying new agent about no SESSIONS`); io.to(socket.id).emit(EVENTS_DEFINITION.emit.NO_SESSIONS); } - socket.join(socket.peerId); + await socket.join(socket.peerId); if (io.sockets.adapter.rooms.get(socket.peerId)) { debug && console.log(`${socket.id} joined room:${socket.peerId}, as:${socket.identity}, members:${io.sockets.adapter.rooms.get(socket.peerId).size}`); } From f158596e5e5e0f50c52f5bba69c5fcbb5a6f7ba2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 16 Feb 2023 11:31:19 +0100 Subject: [PATCH 06/16] chore(actions): changing paths --- .github/workflows/utilities-ee.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/utilities-ee.yaml b/.github/workflows/utilities-ee.yaml index f9a1ac677..92270dc3d 100644 --- a/.github/workflows/utilities-ee.yaml +++ b/.github/workflows/utilities-ee.yaml @@ -6,6 +6,8 @@ on: - dev paths: - "ee/utilities/**" + - "!ee/utilities/.gitignore" + - "!ee/utilities/*-dev.sh" - "utilities/*/**" - "!utilities/.gitignore" - "!utilities/*-dev.sh" From 1721683eb89b8e8ab47e9499a8d2b37bd1fbc487 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 16 Feb 2023 11:31:48 +0100 Subject: [PATCH 07/16] chore(actions): changing branches --- .github/workflows/utilities-ee.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/utilities-ee.yaml b/.github/workflows/utilities-ee.yaml index 92270dc3d..d28f72968 100644 --- a/.github/workflows/utilities-ee.yaml +++ b/.github/workflows/utilities-ee.yaml @@ -3,7 +3,7 @@ on: workflow_dispatch: push: branches: - - dev + - api-v1.10.0 paths: - "ee/utilities/**" - "!ee/utilities/.gitignore" From 15f4f0e45beac0a3159c84b81c6595237164bf24 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 16 Feb 2023 14:46:03 +0100 Subject: [PATCH 08/16] ci(actions): Adding assist GH actions Signed-off-by: rjshrjndrn --- .github/workflows/assist-ee.yaml | 120 ++++++++++++++++++++++++++++ .github/workflows/assist.yaml | 120 ++++++++++++++++++++++++++++ .github/workflows/utilities-ee.yaml | 71 ---------------- .github/workflows/utilities.yaml | 68 ---------------- 4 files changed, 240 insertions(+), 139 deletions(-) create mode 100644 .github/workflows/assist-ee.yaml create mode 100644 .github/workflows/assist.yaml delete mode 100644 .github/workflows/utilities-ee.yaml delete mode 100644 .github/workflows/utilities.yaml diff --git a/.github/workflows/assist-ee.yaml b/.github/workflows/assist-ee.yaml new file mode 100644 index 000000000..78a783dd1 --- /dev/null +++ b/.github/workflows/assist-ee.yaml @@ -0,0 +1,120 @@ +# This action will push the assist changes to aws +on: + workflow_dispatch: + push: + branches: + - dev + paths: + - "ee/utilities/**" + - "utilities/*/**" + - "!utilities/.gitignore" + - "!utilities/*-dev.sh" + +name: Build and Deploy Assist EE + +jobs: + deploy: + name: Deploy + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + # We need to diff with old commit + # to see which workers got changed. + fetch-depth: 2 + + - name: Docker login + run: | + docker login ${{ secrets.EE_REGISTRY_URL }} -u ${{ secrets.EE_DOCKER_USERNAME }} -p "${{ secrets.EE_REGISTRY_TOKEN }}" + + - uses: azure/k8s-set-context@v1 + with: + method: kubeconfig + kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret. + id: setcontext + + - name: Building and Pushing Assist image + id: build-image + env: + DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee + ENVIRONMENT: staging + run: | + skip_security_checks=${{ github.event.inputs.skip_security_checks }} + cd utilities + PUSH_IMAGE=0 bash -x ./build.sh ee + [[ "x$skip_security_checks" == "xtrue" ]] || { + curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ + images=("assist") + for image in ${images[*]};do + ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL" --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG + done + err_code=$? + [[ $err_code -ne 0 ]] && { + exit $err_code + } + } && { + echo "Skipping Security Checks" + } + images=("assist") + for image in ${images[*]};do + docker push $DOCKER_REPO/$image:$IMAGE_TAG + done + - name: Creating old image input + run: | + # + # Create yaml with existing image tags + # + kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\ + tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt + + echo > /tmp/image_override.yaml + + for line in `cat /tmp/image_tag.txt`; + do + image_array=($(echo "$line" | tr ':' '\n')) + cat <> /tmp/image_override.yaml + ${image_array[0]}: + image: + # We've to strip off the -ee, as helm will append it. + tag: `echo ${image_array[1]} | cut -d '-' -f 1` + EOF + done + - name: Deploy to kubernetes + run: | + cd scripts/helmcharts/ + + ## Update secerts + sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml + sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"${{ secrets.EE_PG_PASSWORD }}\"/g" vars.yaml + sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"${{ secrets.EE_MINIO_ACCESS_KEY }}\"/g" vars.yaml + sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"${{ secrets.EE_MINIO_SECRET_KEY }}\"/g" vars.yaml + sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"${{ secrets.EE_JWT_SECRET }}\"/g" vars.yaml + sed -i "s/domainName: \"\"/domainName: \"${{ secrets.EE_DOMAIN_NAME }}\"/g" vars.yaml + sed -i "s/enterpriseEditionLicense: \"\"/enterpriseEditionLicense: \"${{ secrets.EE_LICENSE_KEY }}\"/g" vars.yaml + + # Update changed image tag + sed -i "/assist/{n;n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + + cat /tmp/image_override.yaml + # Deploy command + mv openreplay/charts/{ingress-nginx,chalice,quickwit} /tmp + rm -rf openreplay/charts/* + mv /tmp/{ingress-nginx,chalice,quickwit} openreplay/charts/ + helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks --kube-version=$k_version | kubectl apply -f - + env: + DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} + # We're not passing -ee flag, because helm will add that. + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} + ENVIRONMENT: staging + + # - name: Debug Job + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 + # env: + # DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} + # IMAGE_TAG: ${{ github.sha }} + # ENVIRONMENT: staging + # diff --git a/.github/workflows/assist.yaml b/.github/workflows/assist.yaml new file mode 100644 index 000000000..cf4d184cf --- /dev/null +++ b/.github/workflows/assist.yaml @@ -0,0 +1,120 @@ +# This action will push the assist changes to aws +on: + workflow_dispatch: + push: + branches: + - dev + paths: + - "ee/utilities/**" + - "utilities/*/**" + - "!utilities/.gitignore" + - "!utilities/*-dev.sh" + +name: Build and Deploy Assist EE + +jobs: + deploy: + name: Deploy + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + # We need to diff with old commit + # to see which workers got changed. + fetch-depth: 2 + + - name: Docker login + run: | + docker login ${{ secrets.OSS_REGISTRY_URL }} -u ${{ secrets.OSS_DOCKER_USERNAME }} -p "${{ secrets.OSS_REGISTRY_TOKEN }}" + + - uses: azure/k8s-set-context@v1 + with: + method: kubeconfig + kubeconfig: ${{ secrets.OSS_KUBECONFIG }} # Use content of kubeconfig in secret. + id: setcontext + + - name: Building and Pushing Assist image + id: build-image + env: + DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee + ENVIRONMENT: staging + run: | + skip_security_checks=${{ github.event.inputs.skip_security_checks }} + cd utilities + PUSH_IMAGE=0 bash -x ./build.sh ee + [[ "x$skip_security_checks" == "xtrue" ]] || { + curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ + images=("assist") + for image in ${images[*]};do + ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL" --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG + done + err_code=$? + [[ $err_code -ne 0 ]] && { + exit $err_code + } + } && { + echo "Skipping Security Checks" + } + images=("assist") + for image in ${images[*]};do + docker push $DOCKER_REPO/$image:$IMAGE_TAG + done + - name: Creating old image input + run: | + # + # Create yaml with existing image tags + # + kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\ + tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt + + echo > /tmp/image_override.yaml + + for line in `cat /tmp/image_tag.txt`; + do + image_array=($(echo "$line" | tr ':' '\n')) + cat <> /tmp/image_override.yaml + ${image_array[0]}: + image: + # We've to strip off the -ee, as helm will append it. + tag: `echo ${image_array[1]} | cut -d '-' -f 1` + EOF + done + - name: Deploy to kubernetes + run: | + cd scripts/helmcharts/ + + ## Update secerts + sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml + sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"${{ secrets.OSS_PG_PASSWORD }}\"/g" vars.yaml + sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"${{ secrets.OSS_MINIO_ACCESS_KEY }}\"/g" vars.yaml + sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"${{ secrets.OSS_MINIO_SECRET_KEY }}\"/g" vars.yaml + sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"${{ secrets.OSS_JWT_SECRET }}\"/g" vars.yaml + sed -i "s/domainName: \"\"/domainName: \"${{ secrets.OSS_DOMAIN_NAME }}\"/g" vars.yaml + sed -i "s/enterpriseEditionLicense: \"\"/enterpriseEditionLicense: \"${{ secrets.OSS_LICENSE_KEY }}\"/g" vars.yaml + + # Update changed image tag + sed -i "/assist/{n;n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + + cat /tmp/image_override.yaml + # Deploy command + mv openreplay/charts/{ingress-nginx,chalice,quickwit} /tmp + rm -rf openreplay/charts/* + mv /tmp/{ingress-nginx,chalice,quickwit} openreplay/charts/ + helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks --kube-version=$k_version | kubectl apply -f - + env: + DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} + # We're not passing -ee flag, because helm will add that. + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} + ENVIRONMENT: staging + + # - name: Debug Job + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 + # env: + # DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} + # IMAGE_TAG: ${{ github.sha }} + # ENVIRONMENT: staging + # diff --git a/.github/workflows/utilities-ee.yaml b/.github/workflows/utilities-ee.yaml deleted file mode 100644 index d28f72968..000000000 --- a/.github/workflows/utilities-ee.yaml +++ /dev/null @@ -1,71 +0,0 @@ -# This action will push the assist changes to aws -on: - workflow_dispatch: - push: - branches: - - api-v1.10.0 - paths: - - "ee/utilities/**" - - "!ee/utilities/.gitignore" - - "!ee/utilities/*-dev.sh" - - "utilities/*/**" - - "!utilities/.gitignore" - - "!utilities/*-dev.sh" - -name: Build and Deploy Assist EE - -jobs: - deploy: - name: Deploy - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - # We need to diff with old commit - # to see which workers got changed. - fetch-depth: 2 - - - name: Docker login - run: | - docker login ${{ secrets.EE_REGISTRY_URL }} -u ${{ secrets.EE_DOCKER_USERNAME }} -p "${{ secrets.EE_REGISTRY_TOKEN }}" - - - uses: azure/k8s-set-context@v1 - with: - method: kubeconfig - kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret. - id: setcontext - - - name: Building and Pushing api image - id: build-image - env: - DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} - IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee - ENVIRONMENT: staging - run: | - cd utilities - PUSH_IMAGE=1 bash build.sh ee - - name: Deploy to kubernetes - run: | - cd scripts/helmcharts/ - sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.EE_REGISTRY_URL }}\"#g" vars.yaml - sed -i "s#minio_access_key.*#minio_access_key: \"${{ secrets.EE_MINIO_ACCESS_KEY }}\" #g" vars.yaml - sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.EE_MINIO_SECRET_KEY }}\" #g" vars.yaml - sed -i "s#domain_name.*#domain_name: \"ee.openreplay.com\" #g" vars.yaml - sed -i "s#kubeconfig.*#kubeconfig_path: ${EE_KUBECONFIG}#g" vars.yaml - sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml - bash kube-install.sh --app utilities - env: - DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} - IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} - ENVIRONMENT: staging - - # - name: Debug Job - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v3 - # env: - # DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} - # IMAGE_TAG: ${{ github.sha }} - # ENVIRONMENT: staging - # diff --git a/.github/workflows/utilities.yaml b/.github/workflows/utilities.yaml deleted file mode 100644 index 7d2792d9b..000000000 --- a/.github/workflows/utilities.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# This action will push the assist changes to aws -on: - workflow_dispatch: - push: - branches: - - api-v1.10.0 - paths: - - "utilities/**" - - "!utilities/.gitignore" - - "!utilities/*-dev.sh" - -name: Build and Deploy Assist - -jobs: - deploy: - name: Deploy - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - # We need to diff with old commit - # to see which workers got changed. - fetch-depth: 2 - - - name: Docker login - run: | - docker login ${{ secrets.OSS_REGISTRY_URL }} -u ${{ secrets.OSS_DOCKER_USERNAME }} -p "${{ secrets.OSS_REGISTRY_TOKEN }}" - - - uses: azure/k8s-set-context@v1 - with: - method: kubeconfig - kubeconfig: ${{ secrets.OSS_KUBECONFIG }} # Use content of kubeconfig in secret. - id: setcontext - - - name: Building and Pushing api image - id: build-image - env: - DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} - IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} - ENVIRONMENT: staging - run: | - cd utilities - PUSH_IMAGE=1 bash build.sh - - name: Deploy to kubernetes - run: | - cd scripts/helmcharts/ - sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml - sed -i "s#minio_access_key.*#minio_access_key: \"${{ secrets.OSS_MINIO_ACCESS_KEY }}\" #g" vars.yaml - sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.OSS_MINIO_SECRET_KEY }}\" #g" vars.yaml - sed -i "s#domain_name.*#domain_name: \"foss.openreplay.com\" #g" vars.yaml - sed -i "s#kubeconfig.*#kubeconfig_path: ${KUBECONFIG}#g" vars.yaml - sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml - bash kube-install.sh --app utilities - env: - DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} - IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} - ENVIRONMENT: staging - - # - name: Debug Job - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v3 - # env: - # DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} - # IMAGE_TAG: ${{ github.sha }} - # ENVIRONMENT: staging - # From a52adbd4f509ff8bf27cbf3ef683e0099a773aed Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 16 Feb 2023 17:11:25 +0100 Subject: [PATCH 09/16] feat(assist): support missing protocol --- ee/utilities/servers/websocket-cluster.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/utilities/servers/websocket-cluster.js b/ee/utilities/servers/websocket-cluster.js index 6aa2bade5..77cfd5067 100644 --- a/ee/utilities/servers/websocket-cluster.js +++ b/ee/utilities/servers/websocket-cluster.js @@ -24,7 +24,7 @@ const { const {createAdapter} = require("@socket.io/redis-adapter"); const {createClient} = require("redis"); const wsRouter = express.Router(); -const REDIS_URL = process.env.REDIS_URL || "redis://localhost:6379"; +const REDIS_URL = (process.env.REDIS_URL || "localhost:6379").replace(/((^\w+:|^)\/\/|^)/, 'redis://'); const pubClient = createClient({url: REDIS_URL}); const subClient = pubClient.duplicate(); console.log(`Using Redis: ${REDIS_URL}`); From 80007f45aee7dc343ef39d081f1d489f02f5e5ee Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 16 Feb 2023 18:25:58 +0100 Subject: [PATCH 10/16] feat(assist): upgrade changes --- ee/utilities/.gitignore | 1 - ee/utilities/run-dev.sh | 6 ++++++ ee/utilities/servers/websocket-cluster.js | 5 +++-- ee/utilities/servers/websocket.js | 2 +- utilities/servers/websocket.js | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) create mode 100755 ee/utilities/run-dev.sh diff --git a/ee/utilities/.gitignore b/ee/utilities/.gitignore index 8c9dca279..cd68b1ffb 100644 --- a/ee/utilities/.gitignore +++ b/ee/utilities/.gitignore @@ -15,5 +15,4 @@ servers/sourcemaps-server.js /utils/helper.js /utils/assistHelper.js .local -run-dev.sh *.mmdb diff --git a/ee/utilities/run-dev.sh b/ee/utilities/run-dev.sh new file mode 100755 index 000000000..00e8d5a4b --- /dev/null +++ b/ee/utilities/run-dev.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -a +source .env +set +a + +npm start \ No newline at end of file diff --git a/ee/utilities/servers/websocket-cluster.js b/ee/utilities/servers/websocket-cluster.js index 77cfd5067..fef572a52 100644 --- a/ee/utilities/servers/websocket-cluster.js +++ b/ee/utilities/servers/websocket-cluster.js @@ -283,7 +283,7 @@ module.exports = { wsRouter, start: (server, prefix) => { createSocketIOServer(server, prefix); - io.use(async (socket, next) => await authorizer.check(socket, next)); + // io.use(async (socket, next) => await authorizer.check(socket, next)); io.on('connection', async (socket) => { socket.on(EVENTS_DEFINITION.listen.ERROR, err => errorHandler(EVENTS_DEFINITION.listen.ERROR, err)); debug && console.log(`WS started:${socket.id}, Query:${JSON.stringify(socket.handshake.query)}`); @@ -309,7 +309,8 @@ module.exports = { debug && console.log(`notifying new agent about no SESSIONS`); io.to(socket.id).emit(EVENTS_DEFINITION.emit.NO_SESSIONS); } - await io.of('/').adapter.remoteJoin(socket.id, socket.peerId); + // await io.of('/').adapter.join(socket.id, socket.peerId); + await socket.join(socket.peerId); let rooms = await io.of('/').adapter.allRooms(); if (rooms.has(socket.peerId)) { let connectedSockets = await io.in(socket.peerId).fetchSockets(); diff --git a/ee/utilities/servers/websocket.js b/ee/utilities/servers/websocket.js index bf65789f2..c906b5987 100644 --- a/ee/utilities/servers/websocket.js +++ b/ee/utilities/servers/websocket.js @@ -287,7 +287,7 @@ module.exports = { debug && console.log(`notifying new agent about no SESSIONS`); io.to(socket.id).emit(EVENTS_DEFINITION.emit.NO_SESSIONS); } - socket.join(socket.peerId); + await socket.join(socket.peerId); if (io.sockets.adapter.rooms.get(socket.peerId)) { debug && console.log(`${socket.id} joined room:${socket.peerId}, as:${socket.identity}, members:${io.sockets.adapter.rooms.get(socket.peerId).size}`); } diff --git a/utilities/servers/websocket.js b/utilities/servers/websocket.js index 615390996..f5d029bc2 100644 --- a/utilities/servers/websocket.js +++ b/utilities/servers/websocket.js @@ -268,7 +268,7 @@ module.exports = { debug && console.log(`notifying new agent about no SESSIONS`); io.to(socket.id).emit(EVENTS_DEFINITION.emit.NO_SESSIONS); } - socket.join(socket.peerId); + await socket.join(socket.peerId); if (io.sockets.adapter.rooms.get(socket.peerId)) { debug && console.log(`${socket.id} joined room:${socket.peerId}, as:${socket.identity}, members:${io.sockets.adapter.rooms.get(socket.peerId).size}`); } From ffdb16d8995e43649f410034b644aaf11d4cd48c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 16 Feb 2023 18:58:32 +0100 Subject: [PATCH 11/16] feat(assist): upgrade changes --- ee/utilities/servers/websocket-cluster.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/utilities/servers/websocket-cluster.js b/ee/utilities/servers/websocket-cluster.js index fef572a52..e129bfcb6 100644 --- a/ee/utilities/servers/websocket-cluster.js +++ b/ee/utilities/servers/websocket-cluster.js @@ -283,7 +283,7 @@ module.exports = { wsRouter, start: (server, prefix) => { createSocketIOServer(server, prefix); - // io.use(async (socket, next) => await authorizer.check(socket, next)); + io.use(async (socket, next) => await authorizer.check(socket, next)); io.on('connection', async (socket) => { socket.on(EVENTS_DEFINITION.listen.ERROR, err => errorHandler(EVENTS_DEFINITION.listen.ERROR, err)); debug && console.log(`WS started:${socket.id}, Query:${JSON.stringify(socket.handshake.query)}`); From a5bab0a438217ec253be430777b6bf8e36f4f38c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 17 Feb 2023 13:10:34 +0100 Subject: [PATCH 12/16] feat(alerts): fixed no-events-join builder --- api/app_alerts.py | 7 +++++++ api/chalicelib/core/alerts_processor.py | 19 ++++++++++--------- api/run-alerts-dev.sh | 3 +++ ee/api/chalicelib/core/alerts_processor.py | 16 +++++++++------- .../chalicelib/core/alerts_processor_exp.py | 10 ++++++++-- 5 files changed, 37 insertions(+), 18 deletions(-) create mode 100755 api/run-alerts-dev.sh diff --git a/api/app_alerts.py b/api/app_alerts.py index 7107423de..111bad2a1 100644 --- a/api/app_alerts.py +++ b/api/app_alerts.py @@ -53,3 +53,10 @@ async def stop_server(): await shutdown() import os, signal os.kill(1, signal.SIGTERM) + + +if config("LOCAL_DEV", default=False, cast=bool): + @app.get('/private/trigger', tags=["private"]) + async def trigger_main_cron(): + logging.info("Triggering main cron") + alerts_processor.process() diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index 8049b2f39..4babe64ce 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -49,10 +49,12 @@ LeftToDb = { schemas.AlertColumn.errors__4xx_5xx__count: { "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(session_id)", "condition": "status/100!=2"}, - schemas.AlertColumn.errors__4xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=4"}, - schemas.AlertColumn.errors__5xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=5"}, + schemas.AlertColumn.errors__4xx__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=4"}, + schemas.AlertColumn.errors__5xx__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=5"}, schemas.AlertColumn.errors__javascript__impacted_sessions__count: { "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(DISTINCT session_id)", "condition": "success= FALSE AND type='script'"}, @@ -95,7 +97,7 @@ def can_check(a) -> bool: a["options"].get("lastNotification") is None or a["options"]["lastNotification"] <= 0 or ((now - a["options"]["lastNotification"]) > a["options"]["renotifyInterval"] * 60 * 1000)) \ - and ((now - a["createdAt"]) % (TimeInterval[repetitionBase] * 60 * 1000)) < 60 * 1000 + and ((now - a["createdAt"]) % (TimeInterval[repetitionBase] * 60 * 1000)) < 60 * 1000 def Build(a): @@ -119,7 +121,7 @@ def Build(a): subQ = f"""SELECT {colDef["formula"]} AS value FROM {colDef["table"]} WHERE project_id = %(project_id)s - {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" + {"AND " + colDef["condition"] if colDef.get("condition") else ""}""" j_s = colDef.get("joinSessions", True) main_table = colDef["table"] is_ss = main_table == "public.sessions" @@ -142,8 +144,7 @@ def Build(a): "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, "timestamp_sub2": TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000} else: - sub1 = f"""{subQ} AND timestamp>=%(startDate)s - AND timestamp<=%(now)s + sub1 = f"""{subQ} {"AND timestamp >= %(startDate)s AND timestamp <= %(now)s" if not is_ss else ""} {"AND start_ts >= %(startDate)s AND start_ts <= %(now)s" if j_s else ""}""" params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 sub2 = f"""{subQ} {"AND timestamp < %(startDate)s AND timestamp >= %(timestamp_sub2)s" if not is_ss else ""} @@ -206,7 +207,7 @@ def process(): cur = cur.recreate(rollback=True) if len(notifications) > 0: cur.execute( - cur.mogrify(f"""UPDATE public.Alerts + cur.mogrify(f"""UPDATE public.alerts SET options = options||'{{"lastNotification":{TimeUTC.now()}}}'::jsonb WHERE alert_id IN %(ids)s;""", {"ids": tuple([n["alertId"] for n in notifications])})) if len(notifications) > 0: diff --git a/api/run-alerts-dev.sh b/api/run-alerts-dev.sh new file mode 100755 index 000000000..54db30171 --- /dev/null +++ b/api/run-alerts-dev.sh @@ -0,0 +1,3 @@ +#!/bin/zsh + +uvicorn app_alerts:app --reload \ No newline at end of file diff --git a/ee/api/chalicelib/core/alerts_processor.py b/ee/api/chalicelib/core/alerts_processor.py index 69a0f7f5f..06663336c 100644 --- a/ee/api/chalicelib/core/alerts_processor.py +++ b/ee/api/chalicelib/core/alerts_processor.py @@ -54,10 +54,12 @@ LeftToDb = { schemas.AlertColumn.errors__4xx_5xx__count: { "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(session_id)", "condition": "status/100!=2"}, - schemas.AlertColumn.errors__4xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=4"}, - schemas.AlertColumn.errors__5xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=5"}, + schemas.AlertColumn.errors__4xx__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=4"}, + schemas.AlertColumn.errors__5xx__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=5"}, schemas.AlertColumn.errors__javascript__impacted_sessions__count: { "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(DISTINCT session_id)", "condition": "success= FALSE AND type='script'"}, @@ -100,7 +102,7 @@ def can_check(a) -> bool: a["options"].get("lastNotification") is None or a["options"]["lastNotification"] <= 0 or ((now - a["options"]["lastNotification"]) > a["options"]["renotifyInterval"] * 60 * 1000)) \ - and ((now - a["createdAt"]) % (TimeInterval[repetitionBase] * 60 * 1000)) < 60 * 1000 + and ((now - a["createdAt"]) % (TimeInterval[repetitionBase] * 60 * 1000)) < 60 * 1000 def Build(a): @@ -124,7 +126,7 @@ def Build(a): subQ = f"""SELECT {colDef["formula"]} AS value FROM {colDef["table"]} WHERE project_id = %(project_id)s - {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" + {"AND " + colDef["condition"] if colDef.get("condition") else ""}""" j_s = colDef.get("joinSessions", True) main_table = colDef["table"] is_ss = main_table == "public.sessions" @@ -211,7 +213,7 @@ def process(): cur = cur.recreate(rollback=True) if len(notifications) > 0: cur.execute( - cur.mogrify(f"""UPDATE public.Alerts + cur.mogrify(f"""UPDATE public.alerts SET options = options||'{{"lastNotification":{TimeUTC.now()}}}'::jsonb WHERE alert_id IN %(ids)s;""", {"ids": tuple([n["alertId"] for n in notifications])})) if len(notifications) > 0: diff --git a/ee/api/chalicelib/core/alerts_processor_exp.py b/ee/api/chalicelib/core/alerts_processor_exp.py index 7a300654c..0d8b7753c 100644 --- a/ee/api/chalicelib/core/alerts_processor_exp.py +++ b/ee/api/chalicelib/core/alerts_processor_exp.py @@ -135,7 +135,7 @@ def Build(a): FROM {colDef["table"](now)} WHERE project_id = %(project_id)s {"AND event_type=%(event_type)s" if params["event_type"] else ""} - {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" + {"AND " + colDef["condition"] if colDef.get("condition") else ""}""" q = f"""SELECT coalesce(value,0) AS value, coalesce(value,0) {a["query"]["operator"]} {a["query"]["right"]} AS valid""" @@ -200,7 +200,13 @@ def process(): if alerts_processor.can_check(alert): logging.info(f"Querying alertId:{alert['alertId']} name: {alert['name']}") query, params = Build(alert) - query = ch_cur.format(query, params) + try: + query = ch_cur.format(query, params) + except Exception as e: + logging.error( + f"!!!Error while building alert query for alertId:{alert['alertId']} name: {alert['name']}") + logging.error(e) + continue logging.debug(alert) logging.debug(query) try: From 5d94e72da2836abdcde01bab06aefd03851b6333 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 17 Feb 2023 12:57:29 +0100 Subject: [PATCH 13/16] fix(ui) - alerts list pagination reset --- .../Dashboard/components/Alerts/AlertsList.tsx | 4 ++-- .../Dashboard/components/Alerts/AlertsView.tsx | 16 +++++++++++++++- frontend/app/mstore/alertsStore.ts | 6 ++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/Dashboard/components/Alerts/AlertsList.tsx b/frontend/app/components/Dashboard/components/Alerts/AlertsList.tsx index e4005098e..d1d4c84ef 100644 --- a/frontend/app/components/Dashboard/components/Alerts/AlertsList.tsx +++ b/frontend/app/components/Dashboard/components/Alerts/AlertsList.tsx @@ -17,10 +17,10 @@ function AlertsList({ siteId }: Props) { const { alertsStore, settingsStore } = useStore(); const { fetchWebhooks, webhooks } = settingsStore const { alerts: alertsList, alertsSearch, fetchList, init } = alertsStore + const page = alertsStore.page; React.useEffect(() => { fetchList(); fetchWebhooks() }, []); const alertsArray = alertsList - const [page, setPage] = React.useState(1); const filteredAlerts = filterList(alertsArray, alertsSearch, ['name'], (item, query) => query.test(item.query.left)) const list = alertsSearch !== '' ? filteredAlerts : alertsArray; @@ -59,7 +59,7 @@ function AlertsList({ siteId }: Props) { setPage(page)} + onPageChange={(page) => alertsStore.updateKey('page', page)} limit={pageSize} debounceRequest={100} /> diff --git a/frontend/app/components/Dashboard/components/Alerts/AlertsView.tsx b/frontend/app/components/Dashboard/components/Alerts/AlertsView.tsx index 631df8e43..544c86f8f 100644 --- a/frontend/app/components/Dashboard/components/Alerts/AlertsView.tsx +++ b/frontend/app/components/Dashboard/components/Alerts/AlertsView.tsx @@ -1,16 +1,30 @@ -import React from 'react'; +import React, { useEffect } from 'react'; import { Button, PageTitle, Icon, Link } from 'UI'; import withPageTitle from 'HOCs/withPageTitle'; import { withSiteId, alertCreate } from 'App/routes'; import AlertsList from './AlertsList'; import AlertsSearch from './AlertsSearch'; +import { useHistory } from 'react-router'; +import { useStore } from 'App/mstore'; interface IAlertsView { siteId: string; } function AlertsView({ siteId }: IAlertsView) { + const history = useHistory(); + const { alertsStore } = useStore(); + + + useEffect(() => { + const unmount = history.listen((location) => { + if (!location.pathname.includes('/alert')) { + alertsStore.updateKey('page', 1); + } + }); + return unmount; + }, [history]); return (
diff --git a/frontend/app/mstore/alertsStore.ts b/frontend/app/mstore/alertsStore.ts index d377af81e..e608c1873 100644 --- a/frontend/app/mstore/alertsStore.ts +++ b/frontend/app/mstore/alertsStore.ts @@ -9,6 +9,7 @@ export default class AlertsStore { // @ts-ignore instance: Alert = new Alert({}, false); loading = false + page: number = 1; constructor() { makeAutoObservable(this); @@ -16,6 +17,11 @@ export default class AlertsStore { changeSearch = (value: string) => { this.alertsSearch = value; + this.page = 1; + } + + updateKey(key: string, value: any) { + this[key] = value } fetchList = async () => { From 1028e2f9482024203920372521dc3655c0a03516 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 17 Feb 2023 13:04:59 +0100 Subject: [PATCH 14/16] fix(ui) - seriesName in alert list item --- .../components/Dashboard/components/Alerts/AlertListItem.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/Dashboard/components/Alerts/AlertListItem.tsx b/frontend/app/components/Dashboard/components/Alerts/AlertListItem.tsx index e3412bdce..78f2aa24f 100644 --- a/frontend/app/components/Dashboard/components/Alerts/AlertListItem.tsx +++ b/frontend/app/components/Dashboard/components/Alerts/AlertListItem.tsx @@ -127,7 +127,7 @@ function AlertListItem(props: Props) { {'When the '} {alert.detectionMethod} {' of '} - {alert.seriesName} + {alert.seriesName || alert.query.left} {' is '} {alert.query.operator} From b226f2bbb934cf999ccfb9fc8278900404a560e4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 17 Feb 2023 13:24:41 +0100 Subject: [PATCH 15/16] feat(alerts): changes --- ee/api/chalicelib/core/alerts_processor_exp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ee/api/chalicelib/core/alerts_processor_exp.py b/ee/api/chalicelib/core/alerts_processor_exp.py index 0d8b7753c..37a1b843f 100644 --- a/ee/api/chalicelib/core/alerts_processor_exp.py +++ b/ee/api/chalicelib/core/alerts_processor_exp.py @@ -198,7 +198,6 @@ def process(): if alert["query"]["left"] != "CUSTOM": continue if alerts_processor.can_check(alert): - logging.info(f"Querying alertId:{alert['alertId']} name: {alert['name']}") query, params = Build(alert) try: query = ch_cur.format(query, params) From 5fff5cbad5ee8c59ab703358ea575ed174e9839b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 17 Feb 2023 15:24:13 +0100 Subject: [PATCH 16/16] feat(alerts): changes feat(chalice): changes --- ee/api/chalicelib/core/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ee/api/chalicelib/core/__init__.py b/ee/api/chalicelib/core/__init__.py index 64529b782..62723d0f1 100644 --- a/ee/api/chalicelib/core/__init__.py +++ b/ee/api/chalicelib/core/__init__.py @@ -6,41 +6,41 @@ logging.basicConfig(level=config("LOGLEVEL", default=logging.INFO)) from . import sessions as sessions_legacy if config("EXP_SESSIONS_SEARCH", cast=bool, default=False): - print(">>> Using experimental sessions search") + logging.info(">>> Using experimental sessions search") from . import sessions_exp as sessions else: from . import sessions as sessions if config("EXP_AUTOCOMPLETE", cast=bool, default=False): - print(">>> Using experimental autocomplete") + logging.info(">>> Using experimental autocomplete") from . import autocomplete_exp as autocomplete else: from . import autocomplete as autocomplete if config("EXP_ERRORS_SEARCH", cast=bool, default=False): - print(">>> Using experimental error search") + logging.info(">>> Using experimental error search") from . import errors as errors_legacy from . import errors_exp as errors if config("EXP_ERRORS_GET", cast=bool, default=False): - print(">>> Using experimental error get") + logging.info(">>> Using experimental error get") else: from . import errors as errors if config("EXP_METRICS", cast=bool, default=False): - print(">>> Using experimental metrics") + logging.info(">>> Using experimental metrics") from . import metrics_exp as metrics else: from . import metrics as metrics if config("EXP_ALERTS", cast=bool, default=False): - print(">>> Using experimental alerts") + logging.info(">>> Using experimental alerts") from . import alerts_processor_exp as alerts_processor else: from . import alerts_processor as alerts_processor if config("EXP_FUNNELS", cast=bool, default=False): - print(">>> Using experimental funnels") + logging.info(">>> Using experimental funnels") if not config("EXP_SESSIONS_SEARCH", cast=bool, default=False): from . import sessions as sessions_legacy @@ -49,4 +49,4 @@ else: from . import significance as significance if config("EXP_RESOURCES", cast=bool, default=False): - print(">>> Using experimental resources for session-replay") + logging.info(">>> Using experimental resources for session-replay")