ui: fix double metric selection in list

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

.github/workflows/assist-server-ee.yaml

@@ -0,0 +1,122 @@
+# This action will push the assist changes to aws
+on:
+  workflow_dispatch:
+    inputs:
+      skip_security_checks:
+        description: "Skip Security checks if there is a unfixable vuln or error. Value: true/false"
+        required: false
+        default: "false"
+  push:
+    branches:
+      - dev
+    paths:
+      - "ee/assist-server/**"
+
+name: Build and Deploy Assist-Server EE
+
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          # We need to diff with old commit
+          # to see which workers got changed.
+          fetch-depth: 2
+
+      - uses: ./.github/composite-actions/update-keys
+        with:
+          assist_jwt_secret: ${{ secrets.ASSIST_JWT_SECRET }}
+          assist_key: ${{ secrets.ASSIST_KEY }}
+          domain_name: ${{ secrets.EE_DOMAIN_NAME }}
+          jwt_refresh_secret: ${{ secrets.JWT_REFRESH_SECRET }}
+          jwt_secret: ${{ secrets.EE_JWT_SECRET }}
+          jwt_spot_refresh_secret: ${{ secrets.JWT_SPOT_REFRESH_SECRET }}
+          jwt_spot_secret: ${{ secrets.JWT_SPOT_SECRET }}
+          license_key: ${{ secrets.EE_LICENSE_KEY }}
+          minio_access_key: ${{ secrets.EE_MINIO_ACCESS_KEY }}
+          minio_secret_key: ${{ secrets.EE_MINIO_SECRET_KEY }}
+          pg_password: ${{ secrets.EE_PG_PASSWORD }}
+          registry_url: ${{ secrets.OSS_REGISTRY_URL }}
+        name: Update Keys
+
+      - name: Docker login
+        run: |
+          docker login ${{ secrets.EE_REGISTRY_URL }} -u ${{ secrets.EE_DOCKER_USERNAME }} -p "${{ secrets.EE_REGISTRY_TOKEN }}"
+
+      - uses: azure/k8s-set-context@v1
+        with:
+          method: kubeconfig
+          kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret.
+        id: setcontext
+
+      - name: Building and Pushing Assist-Server image
+        id: build-image
+        env:
+          DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
+          IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee
+          ENVIRONMENT: staging
+        run: |
+          skip_security_checks=${{ github.event.inputs.skip_security_checks }}
+          cd assist-server
+          PUSH_IMAGE=0 bash -x ./build.sh ee
+          [[ "x$skip_security_checks" == "xtrue" ]]  || {
+            curl -L https://github.com/aquasecurity/trivy/releases/download/v0.56.2/trivy_0.56.2_Linux-64bit.tar.gz | tar -xzf - -C ./ 
+            images=("assist-server")
+            for image in ${images[*]};do
+              ./trivy image --db-repository ghcr.io/aquasecurity/trivy-db:2 --db-repository public.ecr.aws/aquasecurity/trivy-db:2 --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL"  --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG 
+            done
+            err_code=$?
+            [[ $err_code -ne 0 ]] && {
+              exit $err_code
+            }
+          } && {
+            echo "Skipping Security Checks"
+          }
+          images=("assist-server")
+          for image in ${images[*]};do
+            docker push $DOCKER_REPO/$image:$IMAGE_TAG 
+          done
+      - name: Creating old image input
+        run: |
+          #
+          # Create yaml with existing image tags
+          #
+          kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\
+          tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt
+
+          echo > /tmp/image_override.yaml
+
+          for line in `cat /tmp/image_tag.txt`;
+          do
+              image_array=($(echo "$line" | tr ':' '\n'))
+              cat <<EOF >> /tmp/image_override.yaml
+          ${image_array[0]}:
+            image:
+              # We've to strip off the -ee, as helm will append it.
+              tag: `echo ${image_array[1]} | cut -d '-' -f 1`
+          EOF
+          done
+      - name: Deploy to kubernetes
+        run: |
+          pwd
+          cd scripts/helmcharts/
+
+          # Update changed image tag
+          sed -i "/assist-server/{n;n;n;s/.*/    tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml
+
+          cat /tmp/image_override.yaml
+          # Deploy command
+          mkdir -p /tmp/charts
+          mv openreplay/charts/{ingress-nginx,assist-server,quickwit,connector} /tmp/charts/
+          rm -rf  openreplay/charts/*
+          mv /tmp/charts/* openreplay/charts/
+          helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks --kube-version=$k_version | kubectl apply -f -
+        env:
+          DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}
+          # We're not passing -ee flag, because helm will add that.
+          IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
+          ENVIRONMENT: staging

.github/workflows/patch-build-old.yaml

@@ -0,0 +1,189 @@
+# Ref: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions
+
+on:
+  workflow_dispatch:
+    inputs:
+      services:
+        description: 'Comma separated names of services to build(in small letters).'
+        required: true
+        default: 'chalice,frontend'
+      tag:
+        description: 'Tag to update.'
+        required: true
+        type: string
+      branch:
+        description: 'Branch to build patches from. Make sure the branch is uptodate with tag. Else itll cause missing commits.'
+        required: true
+        type: string
+
+name: Build patches from tag, rewrite commit HEAD to older timestamp, and Push the tag
+
+jobs:
+  deploy:
+    name: Build Patch from old tag
+    runs-on: ubuntu-latest
+    env:
+      DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
+      DEPOT_PROJECT_ID: ${{ secrets.DEPOT_PROJECT_ID }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 4
+        ref: ${{ github.event.inputs.tag }}
+
+    - name: Set Remote with GITHUB_TOKEN
+      run: |
+        git config --unset http.https://github.com/.extraheader
+        git remote set-url origin https://x-access-token:${{ secrets.ACTIONS_COMMMIT_TOKEN }}@github.com/${{ github.repository }}.git
+
+    - name: Create backup tag with timestamp
+      run: |
+        set -e  # Exit immediately if a command exits with a non-zero status
+        TIMESTAMP=$(date +%Y%m%d%H%M%S)
+        BACKUP_TAG="${{ github.event.inputs.tag }}-backup-${TIMESTAMP}"
+        echo "BACKUP_TAG=${BACKUP_TAG}" >> $GITHUB_ENV
+        echo "INPUT_TAG=${{ github.event.inputs.tag }}" >> $GITHUB_ENV
+        git tag $BACKUP_TAG || { echo "Failed to create backup tag"; exit 1; }
+        git push origin $BACKUP_TAG || { echo "Failed to push backup tag"; exit 1; }
+        echo "Created backup tag: $BACKUP_TAG"
+
+        # Get the oldest commit date from the last 3 commits in raw format
+        OLDEST_COMMIT_TIMESTAMP=$(git log -3 --pretty=format:"%at" | tail -1)
+        echo "Oldest commit timestamp: $OLDEST_COMMIT_TIMESTAMP"
+        # Add 1 second to the timestamp
+        NEW_TIMESTAMP=$((OLDEST_COMMIT_TIMESTAMP + 1))
+        echo "NEW_TIMESTAMP=$NEW_TIMESTAMP" >> $GITHUB_ENV
+
+
+    - name: Setup yq
+      uses: mikefarah/yq@master
+
+    # Configure AWS credentials for the first registry
+    - name: Configure AWS credentials for RELEASE_ARM_REGISTRY
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_DEPOT_ACCESS_KEY }}
+        aws-secret-access-key: ${{ secrets.AWS_DEPOT_SECRET_KEY }}
+        aws-region: ${{ secrets.AWS_DEPOT_DEFAULT_REGION }}
+
+    - name: Login to Amazon ECR for RELEASE_ARM_REGISTRY
+      id: login-ecr-arm
+      run: |
+        aws ecr get-login-password --region ${{ secrets.AWS_DEPOT_DEFAULT_REGION }} | docker login --username AWS --password-stdin ${{ secrets.RELEASE_ARM_REGISTRY }}
+        aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${{ secrets.RELEASE_OSS_REGISTRY }}
+
+    - uses: depot/setup-action@v1
+    - name: Get HEAD Commit ID
+      run: echo "HEAD_COMMIT_ID=$(git rev-parse HEAD)" >> $GITHUB_ENV
+    - name: Define Branch Name
+      run: echo "BRANCH_NAME=${{inputs.branch}}" >> $GITHUB_ENV
+
+    - name: Build
+      id: build-image
+      env:
+        DOCKER_REPO_ARM: ${{ secrets.RELEASE_ARM_REGISTRY }}
+        DOCKER_REPO_OSS: ${{ secrets.RELEASE_OSS_REGISTRY }}
+        MSAAS_REPO_CLONE_TOKEN: ${{ secrets.MSAAS_REPO_CLONE_TOKEN }}
+        MSAAS_REPO_URL: ${{ secrets.MSAAS_REPO_URL }}
+        MSAAS_REPO_FOLDER: /tmp/msaas
+      run: |
+        set -exo pipefail
+        git config --local user.email "action@github.com"
+        git config --local user.name "GitHub Action"
+        git checkout -b $BRANCH_NAME
+        working_dir=$(pwd)
+        function image_version(){
+          local service=$1
+          chart_path="$working_dir/scripts/helmcharts/openreplay/charts/$service/Chart.yaml"
+          current_version=$(yq eval '.AppVersion' $chart_path)
+          new_version=$(echo $current_version | awk -F. '{$NF += 1 ; print $1"."$2"."$3}')
+          echo $new_version
+          # yq eval ".AppVersion = \"$new_version\"" -i $chart_path
+        }
+        function clone_msaas() {
+          [ -d $MSAAS_REPO_FOLDER ] || {
+          git clone -b $INPUT_TAG --recursive https://x-access-token:$MSAAS_REPO_CLONE_TOKEN@$MSAAS_REPO_URL $MSAAS_REPO_FOLDER
+          cd $MSAAS_REPO_FOLDER
+          cd openreplay && git fetch origin && git checkout $INPUT_TAG
+          git log -1
+          cd $MSAAS_REPO_FOLDER
+          bash git-init.sh
+          git checkout
+          }
+        }
+        function build_managed() {
+          local service=$1
+          local version=$2
+          echo building managed
+          clone_msaas
+          if [[ $service == 'chalice' ]]; then
+            cd $MSAAS_REPO_FOLDER/openreplay/api
+          else
+            cd $MSAAS_REPO_FOLDER/openreplay/$service
+          fi
+          IMAGE_TAG=$version DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=arm64 DOCKER_REPO=$DOCKER_REPO_ARM PUSH_IMAGE=0 bash build.sh >> /tmp/arm.txt
+        }
+        # Checking for backend images
+        ls backend/cmd >> /tmp/backend.txt
+        echo Services: "${{ github.event.inputs.services }}"
+        IFS=',' read -ra SERVICES <<< "${{ github.event.inputs.services }}" 
+        BUILD_SCRIPT_NAME="build.sh"
+        # Build FOSS
+        for SERVICE in "${SERVICES[@]}"; do
+          # Check if service is backend
+          if grep -q $SERVICE /tmp/backend.txt; then 
+            cd backend 
+            foss_build_args="nil $SERVICE"
+            ee_build_args="ee $SERVICE"
+          else 
+            [[ $SERVICE == 'chalice' || $SERVICE == 'alerts' || $SERVICE == 'crons' ]] && cd $working_dir/api || cd $SERVICE
+            [[ $SERVICE == 'alerts' || $SERVICE == 'crons' ]] && BUILD_SCRIPT_NAME="build_${SERVICE}.sh"
+            ee_build_args="ee"
+          fi  
+          version=$(image_version $SERVICE)
+          echo IMAGE_TAG=$version DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=amd64 DOCKER_REPO=$DOCKER_REPO_OSS PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $foss_build_args
+          IMAGE_TAG=$version DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=amd64 DOCKER_REPO=$DOCKER_REPO_OSS PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $foss_build_args
+          echo IMAGE_TAG=$version-ee DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=amd64 DOCKER_REPO=$DOCKER_REPO_OSS PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $ee_build_args
+          IMAGE_TAG=$version-ee DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=amd64 DOCKER_REPO=$DOCKER_REPO_OSS PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $ee_build_args
+          if [[ "$SERVICE" != "chalice" && "$SERVICE" != "frontend" ]]; then
+            IMAGE_TAG=$version DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=arm64 DOCKER_REPO=$DOCKER_REPO_ARM PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $foss_build_args
+            echo IMAGE_TAG=$version DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=arm64 DOCKER_REPO=$DOCKER_REPO_ARM PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $foss_build_args
+          else
+            build_managed $SERVICE $version
+          fi
+          cd $working_dir
+          chart_path="$working_dir/scripts/helmcharts/openreplay/charts/$SERVICE/Chart.yaml"
+          yq eval ".AppVersion = \"$version\"" -i $chart_path
+          git add $chart_path
+          git commit -m "Increment $SERVICE chart version"
+        done
+
+    - name: Change commit timestamp
+      run: |
+        # Convert the timestamp to a date format git can understand
+        NEW_DATE=$(perl -le 'print scalar gmtime($ARGV[0])." +0000"' $NEW_TIMESTAMP)
+        echo "Setting commit date to: $NEW_DATE"
+
+        # Amend the commit with the new date
+        GIT_COMMITTER_DATE="$NEW_DATE" git commit --amend --no-edit --date="$NEW_DATE"
+
+        # Verify the change
+        git log -1 --pretty=format:"Commit now dated: %cD"
+
+        # git tag and push
+        git tag $INPUT_TAG -f
+        git push origin $INPUT_TAG -f
+
+
+    # - name: Debug Job
+    #   if: ${{ failure() }}
+    #   uses: mxschmitt/action-tmate@v3
+    #   env:
+    #     DOCKER_REPO_ARM: ${{ secrets.RELEASE_ARM_REGISTRY }}
+    #     DOCKER_REPO_OSS: ${{ secrets.RELEASE_OSS_REGISTRY }}
+    #     MSAAS_REPO_CLONE_TOKEN: ${{ secrets.MSAAS_REPO_CLONE_TOKEN }}
+    #     MSAAS_REPO_URL: ${{ secrets.MSAAS_REPO_URL }}
+    #     MSAAS_REPO_FOLDER: /tmp/msaas
+    #   with:
+    #     limit-access-to-actor: true

.github/workflows/patch-build.yaml

@@ -2,7 +2,6 @@
 
 on:
   workflow_dispatch:
-    description: 'This workflow will build for patches for latest tag, and will Always use commit from main branch.'
     inputs:
       services:
         description: 'Comma separated names of services to build(in small letters).'
@@ -20,12 +19,20 @@ jobs:
       DEPOT_PROJECT_ID: ${{ secrets.DEPOT_PROJECT_ID }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
-        fetch-depth: 1
+        fetch-depth: 0
+        token: ${{ secrets.GITHUB_TOKEN }}
     - name: Rebase with main branch, to make sure the code has latest main changes
+      if: github.ref != 'refs/heads/main'
       run: |
-        git pull --rebase origin main
+        git remote -v
+        git config --global user.email "action@github.com"
+        git config --global user.name "GitHub Action"
+        git config --global rebase.autoStash true
+        git fetch origin main:main
+        git rebase main
+        git log -3
 
     - name: Downloading yq
       run: |
@@ -48,6 +55,8 @@ jobs:
         aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${{ secrets.RELEASE_OSS_REGISTRY }}
 
     - uses: depot/setup-action@v1
+      env:
+        DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
     - name: Get HEAD Commit ID
       run: echo "HEAD_COMMIT_ID=$(git rev-parse HEAD)" >> $GITHUB_ENV
     - name: Define Branch Name
@@ -65,78 +74,168 @@ jobs:
         MSAAS_REPO_CLONE_TOKEN: ${{ secrets.MSAAS_REPO_CLONE_TOKEN }}
         MSAAS_REPO_URL: ${{ secrets.MSAAS_REPO_URL }}
         MSAAS_REPO_FOLDER: /tmp/msaas
+        SERVICES_INPUT: ${{ github.event.inputs.services }}
       run: |
-        set -exo pipefail
-        git config --local user.email "action@github.com"
-        git config --local user.name "GitHub Action"
-        git checkout -b $BRANCH_NAME
-        working_dir=$(pwd)
-        function image_version(){
-          local service=$1
-          chart_path="$working_dir/scripts/helmcharts/openreplay/charts/$service/Chart.yaml"
-          current_version=$(yq eval '.AppVersion' $chart_path)
-          new_version=$(echo $current_version | awk -F. '{$NF += 1 ; print $1"."$2"."$3}')
-          echo $new_version
-          # yq eval ".AppVersion = \"$new_version\"" -i $chart_path
+        #!/bin/bash
+        set -euo pipefail
+
+        # Configuration
+        readonly WORKING_DIR=$(pwd)
+        readonly BUILD_SCRIPT_NAME="build.sh"
+        readonly BACKEND_SERVICES_FILE="/tmp/backend.txt"
+
+        # Initialize git configuration
+        setup_git() {
+            git config --local user.email "action@github.com"
+            git config --local user.name "GitHub Action"
+            git checkout -b "$BRANCH_NAME"
         }
-        function clone_msaas() {
-          [ -d $MSAAS_REPO_FOLDER ] || {
-          git clone -b dev --recursive https://x-access-token:$MSAAS_REPO_CLONE_TOKEN@$MSAAS_REPO_URL $MSAAS_REPO_FOLDER
-          cd $MSAAS_REPO_FOLDER
-          cd openreplay && git fetch origin && git checkout main # This have to be changed to specific tag
-          git log -1
-          cd $MSAAS_REPO_FOLDER
-          bash git-init.sh
-          git checkout
-          }
+
+        # Get and increment image version
+        image_version() {
+            local service=$1
+            local chart_path="$WORKING_DIR/scripts/helmcharts/openreplay/charts/$service/Chart.yaml"
+            local current_version new_version
+
+            current_version=$(yq eval '.AppVersion' "$chart_path")
+            new_version=$(echo "$current_version" | awk -F. '{$NF += 1; print $1"."$2"."$3}')
+            echo "$new_version"
         }
-        function build_managed() {
-          local service=$1
-          local version=$2
-          echo building managed
-          clone_msaas
-          if [[ $service == 'chalice' ]]; then
-            cd $MSAAS_REPO_FOLDER/openreplay/api
-          else
-            cd $MSAAS_REPO_FOLDER/openreplay/$service
-          fi
-          IMAGE_TAG=$version DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=arm64 DOCKER_REPO=$DOCKER_REPO_ARM PUSH_IMAGE=0 bash build.sh >> /tmp/arm.txt
+
+        # Clone MSAAS repository if not exists
+        clone_msaas() {
+            if [[ ! -d "$MSAAS_REPO_FOLDER" ]]; then
+                git clone -b dev --recursive "https://x-access-token:${MSAAS_REPO_CLONE_TOKEN}@${MSAAS_REPO_URL}" "$MSAAS_REPO_FOLDER"
+                cd "$MSAAS_REPO_FOLDER"
+                cd openreplay && git fetch origin && git checkout main
+                git log -1
+                cd "$MSAAS_REPO_FOLDER"
+                bash git-init.sh
+                git checkout
+            fi
         }
-        # Checking for backend images
-        ls backend/cmd >> /tmp/backend.txt
-        echo Services: "${{ github.event.inputs.services }}"
-        IFS=',' read -ra SERVICES <<< "${{ github.event.inputs.services }}" 
-        BUILD_SCRIPT_NAME="build.sh"
-        # Build FOSS
-        for SERVICE in "${SERVICES[@]}"; do
-          # Check if service is backend
-          if grep -q $SERVICE /tmp/backend.txt; then 
-            cd backend 
-            foss_build_args="nil $SERVICE"
-            ee_build_args="ee $SERVICE"
-          else 
-            [[ $SERVICE == 'chalice' || $SERVICE == 'alerts' || $SERVICE == 'crons' ]] && cd $working_dir/api || cd $SERVICE
-            [[ $SERVICE == 'alerts' || $SERVICE == 'crons' ]] && BUILD_SCRIPT_NAME="build_${SERVICE}.sh"
-            ee_build_args="ee"
-          fi  
-          version=$(image_version $SERVICE)
-          echo IMAGE_TAG=$version DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=amd64 DOCKER_REPO=$DOCKER_REPO_OSS PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $foss_build_args
-          IMAGE_TAG=$version DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=amd64 DOCKER_REPO=$DOCKER_REPO_OSS PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $foss_build_args
-          echo IMAGE_TAG=$version-ee DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=amd64 DOCKER_REPO=$DOCKER_REPO_OSS PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $ee_build_args
-          IMAGE_TAG=$version-ee DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=amd64 DOCKER_REPO=$DOCKER_REPO_OSS PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $ee_build_args
-          if [[ "$SERVICE" != "chalice" && "$SERVICE" != "frontend" ]]; then
-            IMAGE_TAG=$version DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=arm64 DOCKER_REPO=$DOCKER_REPO_ARM PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $foss_build_args
-            echo IMAGE_TAG=$version DOCKER_RUNTIME="depot" DOCKER_BUILD_ARGS="--push" ARCH=arm64 DOCKER_REPO=$DOCKER_REPO_ARM PUSH_IMAGE=0 bash ${BUILD_SCRIPT_NAME} $foss_build_args
-          else
-            build_managed $SERVICE $version
-          fi
-          cd $working_dir
-          chart_path="$working_dir/scripts/helmcharts/openreplay/charts/$SERVICE/Chart.yaml"
-          yq eval ".AppVersion = \"$version\"" -i $chart_path
-          git add $chart_path
-          git commit -m "Increment $SERVICE chart version"
-          git push --set-upstream origin $BRANCH_NAME
-        done
+
+        # Build managed services
+        build_managed() {
+            local service=$1
+            local version=$2
+
+            echo "Building managed service: $service"
+            clone_msaas
+
+            if [[ $service == 'chalice' ]]; then
+                cd "$MSAAS_REPO_FOLDER/openreplay/api"
+            else
+                cd "$MSAAS_REPO_FOLDER/openreplay/$service"
+            fi
+
+            local build_cmd="IMAGE_TAG=$version DOCKER_RUNTIME=depot DOCKER_BUILD_ARGS=--push ARCH=arm64 DOCKER_REPO=$DOCKER_REPO_ARM PUSH_IMAGE=0 bash build.sh"
+
+            echo "Executing: $build_cmd"
+            if ! eval "$build_cmd" 2>&1; then
+                echo "Build failed for $service"
+                exit 1
+            fi
+        }
+
+        # Build service with given arguments
+        build_service() {
+            local service=$1
+            local version=$2
+            local build_args=$3
+            local build_script=${4:-$BUILD_SCRIPT_NAME}
+
+            local command="IMAGE_TAG=$version DOCKER_RUNTIME=depot DOCKER_BUILD_ARGS=--push ARCH=amd64 DOCKER_REPO=$DOCKER_REPO_OSS PUSH_IMAGE=0 bash $build_script $build_args"
+            echo "Executing: $command"
+            eval "$command"
+        }
+
+        # Update chart version and commit changes
+        update_chart_version() {
+            local service=$1
+            local version=$2
+            local chart_path="$WORKING_DIR/scripts/helmcharts/openreplay/charts/$service/Chart.yaml"
+
+            # Ensure we're in the original working directory/repository
+            cd "$WORKING_DIR"
+            yq eval ".AppVersion = \"$version\"" -i "$chart_path"
+            git add "$chart_path"
+            git commit -m "Increment $service chart version to $version"
+            git push --set-upstream origin "$BRANCH_NAME"
+            cd -
+        }
+
+        # Main execution
+        main() {
+            setup_git
+
+            # Get backend services list
+            ls backend/cmd >"$BACKEND_SERVICES_FILE"
+
+            # Parse services input (fix for GitHub Actions syntax)
+            echo "Services: ${SERVICES_INPUT:-$1}"
+            IFS=',' read -ra services <<<"${SERVICES_INPUT:-$1}"
+
+            # Process each service
+            for service in "${services[@]}"; do
+                echo "Processing service: $service"
+                cd "$WORKING_DIR"
+
+                local foss_build_args="" ee_build_args="" build_script="$BUILD_SCRIPT_NAME"
+
+                # Determine build configuration based on service type
+                if grep -q "$service" "$BACKEND_SERVICES_FILE"; then
+                    # Backend service
+                    cd backend
+                    foss_build_args="nil $service"
+                    ee_build_args="ee $service"
+                else
+                    # Non-backend service
+                    case "$service" in
+                    chalice | alerts | crons)
+                        cd "$WORKING_DIR/api"
+                        ;;
+                    *)
+                        cd "$service"
+                        ;;
+                    esac
+
+                    # Special build scripts for alerts/crons
+                    if [[ $service == 'alerts' || $service == 'crons' ]]; then
+                        build_script="build_${service}.sh"
+                    fi
+
+                    ee_build_args="ee"
+                fi
+
+                # Get version and build
+                local version
+                version=$(image_version "$service")
+
+                # Build FOSS and EE versions
+                build_service "$service" "$version" "$foss_build_args"
+                build_service "$service" "${version}-ee" "$ee_build_args"
+
+                # Build managed version for specific services
+                if [[ "$service" != "chalice" && "$service" != "frontend" ]]; then
+                    echo "Nothing to build in managed for service $service"
+                else
+                    build_managed "$service" "$version"
+                fi
+
+                # Update chart and commit
+                update_chart_version "$service" "$version"
+            done
+            cd "$WORKING_DIR"
+
+            # Cleanup
+            rm -f "$BACKEND_SERVICES_FILE"
+        }
+
+        echo "Working directory: $WORKING_DIR"
+        # Run main function with all arguments
+        main "$SERVICES_INPUT"
+        
 
     - name: Create Pull Request
       uses: repo-sync/pull-request@v2
@@ -147,8 +246,7 @@ jobs:
         pr_title: "Updated patch build from main ${{ env.HEAD_COMMIT_ID }}"
         pr_body: | 
           This PR updates the Helm chart version after building the patch from $HEAD_COMMIT_ID.
-          Once this PR is merged, To update the latest tag, run the following workflow.
-          https://github.com/openreplay/openreplay/actions/workflows/update-tag.yaml
+          Once this PR is merged, tag update job will run automatically.
 
     # - name: Debug Job
     #   if: ${{ failure() }}

.github/workflows/update-tag.yaml

@@ -1,35 +1,42 @@
 on:
-  workflow_dispatch:
-    description: "This workflow will build for patches for latest tag, and will Always use commit from main branch."
-    inputs:
-      services:
-        description: "This action will update the latest tag with current main branch HEAD. Should I proceed ? true/false"
-        required: true
-        default: "false"
-
-name: Force Push tag with main branch HEAD
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+name: Release tag update --force
 
 jobs:
   deploy:
     name: Build Patch from main
     runs-on: ubuntu-latest
-    env:
-      DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
-      DEPOT_PROJECT_ID: ${{ secrets.DEPOT_PROJECT_ID }}
+    if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.merged == true) || github.event.inputs.services == 'true' }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+
+      - name: Get latest release tag using GitHub API
+        id: get-latest-tag
+        run: |
+          LATEST_TAG=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/${{ github.repository }}/releases/latest" \
+            | jq -r .tag_name)
+          
+          # Fallback to git command if API doesn't return a tag
+          if [ "$LATEST_TAG" == "null" ] || [ -z "$LATEST_TAG" ]; then
+            echo "Not found latest tag"
+            exit 100
+          fi
+          
+          echo "LATEST_TAG=$LATEST_TAG" >> $GITHUB_ENV
+          echo "Latest tag: $LATEST_TAG"
+
       - name: Set Remote with GITHUB_TOKEN
         run: |
           git config --unset http.https://github.com/.extraheader
-          git remote set-url origin https://x-access-token:${{ secrets.ACTIONS_COMMMIT_TOKEN }}@github.com/${{ github.repository }}.git
+          git remote set-url origin https://x-access-token:${{ secrets.ACTIONS_COMMMIT_TOKEN }}@github.com/${{ github.repository }}
+
       - name: Push main branch to tag
         run: |
-          git fetch --tags
           git checkout main
-          git push origin HEAD:refs/tags/$(git tag --list 'v[0-9]*' --sort=-v:refname | head -n 1) --force
-    # - name: Debug Job
-    #   if: ${{ failure() }}
-    #   uses: mxschmitt/action-tmate@v3
-    #   with:
-    #     limit-access-to-actor: true
+          echo "Updating tag ${{ env.LATEST_TAG }} to point to latest commit on main"
+          git push origin HEAD:refs/tags/${{ env.LATEST_TAG }} --force

api/Dockerfile

@@ -1,10 +1,17 @@
-FROM python:3.12-alpine
-LABEL Maintainer="Rajesh Rajendran<rjshrjndrn@gmail.com>"
-LABEL Maintainer="KRAIEM Taha Yassine<tahayk2@gmail.com>"
-ARG GIT_SHA
-LABEL GIT_SHA=$GIT_SHA
+FROM python:3.12-alpine AS builder
+LABEL maintainer="Rajesh Rajendran<rjshrjndrn@gmail.com>"
+LABEL maintainer="KRAIEM Taha Yassine<tahayk2@gmail.com>"
 
-RUN apk add --no-cache build-base tini
+RUN apk add --no-cache build-base
+WORKDIR /work
+COPY requirements.txt ./requirements.txt
+RUN pip install --no-cache-dir --upgrade uv && \
+  export UV_SYSTEM_PYTHON=true && \
+  uv pip install --no-cache-dir --upgrade pip setuptools wheel && \
+  uv pip install --no-cache-dir --upgrade -r requirements.txt
+
+FROM python:3.12-alpine
+ARG GIT_SHA
 ARG envarg
 # Add Tini
 # Startup daemon
@@ -14,19 +21,11 @@ ENV SOURCE_MAP_VERSION=0.7.4 \
   PRIVATE_ENDPOINTS=false \
   ENTERPRISE_BUILD=${envarg} \
   GIT_SHA=$GIT_SHA
-
+COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
+COPY --from=builder /usr/local/bin /usr/local/bin
 WORKDIR /work
-COPY requirements.txt ./requirements.txt
-RUN pip install --no-cache-dir --upgrade uv
-RUN uv pip install --no-cache-dir --upgrade pip setuptools wheel --system
-RUN uv pip install --no-cache-dir --upgrade -r requirements.txt --system
-
 COPY . .
-RUN mv env.default .env
-
-RUN adduser -u 1001 openreplay -D
-USER 1001
+RUN apk add --no-cache tini && mv env.default .env
 
 ENTRYPOINT ["/sbin/tini", "--"]
-CMD ./entrypoint.sh
-
+CMD ["./entrypoint.sh"]

api/app.py

@@ -1,133 +1,134 @@
 import logging
+import time
 from contextlib import asynccontextmanager
 
-from fastapi import FastAPI
+import psycopg_pool
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from decouple import config
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.gzip import GZipMiddleware
+from psycopg import AsyncConnection
+from psycopg.rows import dict_row
+from starlette.responses import StreamingResponse
 
-from boot.config import settings
-from boot.middleware import timing_middleware, setup_cors
-from boot.scheduler.manager import SchedulerManager
-from boot.health.router import health_router
+from chalicelib.utils import helper
 from chalicelib.utils import pg_client, ch_client
-
-# Import routers and cron jobs
 from crons import core_crons, core_dynamic_crons
 from routers import core, core_dynamic
 from routers.subs import insights, metrics, v1_api, health, usability_tests, spot, product_anaytics
 
-# Configure logging
-settings.configure_logging()
-logger = logging.getLogger(__name__)
+loglevel = config("LOGLEVEL", default=logging.WARNING)
+print(f">Loglevel set to: {loglevel}")
+logging.basicConfig(level=loglevel)
+
+
+class ORPYAsyncConnection(AsyncConnection):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, row_factory=dict_row, **kwargs)
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    """
-    Application lifespan context manager.
-
-    Handles startup and shutdown tasks:
-    - Database connections initialization
-    - Scheduler setup
-    - Cleanup on shutdown
-    """
     # Startup
-    logger.info(">>>>> starting up <<<<<")
+    logging.info(">>>>> starting up <<<<<")
+    ap_logger = logging.getLogger('apscheduler')
+    ap_logger.setLevel(loglevel)
 
-    # Initialize database connections
+    app.schedule = AsyncIOScheduler()
     await pg_client.init()
     await ch_client.init()
+    app.schedule.start()
 
-    # Initialize scheduler with jobs
-    all_jobs = core_crons.cron_jobs + core_dynamic_crons.cron_jobs
-    SchedulerManager.initialize(all_jobs)
+    for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs:
+        app.schedule.add_job(id=job["func"].__name__, **job)
 
-    # Store PostgreSQL pool in app state for backwards compatibility
-    # app.state.postgresql = pg_client.
+    ap_logger.info(">Scheduled jobs:")
+    for job in app.schedule.get_jobs():
+        ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)})
 
-    # App is ready to handle requests
+    database = {
+        "host": config("pg_host", default="localhost"),
+        "dbname": config("pg_dbname", default="orpy"),
+        "user": config("pg_user", default="orpy"),
+        "password": config("pg_password", default="orpy"),
+        "port": config("pg_port", cast=int, default=5432),
+        "application_name": "AIO" + config("APP_NAME", default="PY"),
+    }
+
+    database = psycopg_pool.AsyncConnectionPool(kwargs=database, connection_class=ORPYAsyncConnection,
+                                                min_size=config("PG_AIO_MINCONN", cast=int, default=1),
+                                                max_size=config("PG_AIO_MAXCONN", cast=int, default=5), )
+    app.state.postgresql = database
+
+    # App listening
     yield
 
     # Shutdown
-    logger.info(">>>>> shutting down <<<<<")
-
-    # Shutdown scheduler
-    SchedulerManager.shutdown()
-
-    # Close database connections
+    await database.close()
+    logging.info(">>>>> shutting down <<<<<")
+    app.schedule.shutdown(wait=False)
     await pg_client.terminate()
-    await ch_client.terminate()
 
 
-def create_app() -> FastAPI:
-    """Create and configure the FastAPI application."""
-    # Create FastAPI app
-    app = FastAPI(
-        root_path=settings.ROOT_PATH,
-        docs_url=settings.DOCS_URL,
-        redoc_url=settings.REDOC_URL,
-        lifespan=lifespan,
-        title=f"{settings.APP_NAME} API",
-        description=f"API for {settings.APP_NAME}",
-        version="1.0.0"
-    )
-
-    # Add middlewares
-    app.add_middleware(GZipMiddleware, minimum_size=1000)
-    app.middleware('http')(timing_middleware)
-    setup_cors(app)
-
-    # Register health check routes first for monitoring systems
-    app.include_router(health_router)
-
-    # Register existing application routers
-    register_routers(app)
-
-    return app
+app = FastAPI(root_path=config("root_path", default="/api"), docs_url=config("docs_url", default=""),
+              redoc_url=config("redoc_url", default=""), lifespan=lifespan)
+app.add_middleware(GZipMiddleware, minimum_size=1000)
 
 
-def register_routers(app: FastAPI) -> None:
-    """Register all application routers."""
-    # Core routers
-    app.include_router(core.public_app)
-    app.include_router(core.app)
-    app.include_router(core.app_apikey)
-
-    # Core dynamic routers
-    app.include_router(core_dynamic.public_app)
-    app.include_router(core_dynamic.app)
-    app.include_router(core_dynamic.app_apikey)
-
-    # Feature routers
-    app.include_router(metrics.app)
-    app.include_router(insights.app)
-    app.include_router(v1_api.app_apikey)
-
-    # Health routers (existing ones from the codebase)
-    app.include_router(health.public_app)
-    app.include_router(health.app)
-    app.include_router(health.app_apikey)
-
-    # Usability tests routers
-    app.include_router(usability_tests.public_app)
-    app.include_router(usability_tests.app)
-    app.include_router(usability_tests.app_apikey)
-
-    # Spot routers
-    app.include_router(spot.public_app)
-    app.include_router(spot.app)
-    app.include_router(spot.app_apikey)
-
-    # Product analytics routers
-    app.include_router(product_anaytics.public_app)
-    app.include_router(product_anaytics.app)
-    app.include_router(product_anaytics.app_apikey)
+@app.middleware('http')
+async def or_middleware(request: Request, call_next):
+    if helper.TRACK_TIME:
+        now = time.time()
+    try:
+        response: StreamingResponse = await call_next(request)
+    except:
+        logging.error(f"{request.method}: {request.url.path} FAILED!")
+        raise
+    if response.status_code // 100 != 2:
+        logging.warning(f"{request.method}:{request.url.path} {response.status_code}!")
+    if helper.TRACK_TIME:
+        now = time.time() - now
+        if now > 2:
+            now = round(now, 2)
+            logging.warning(f"Execution time: {now} s for {request.method}: {request.url.path}")
+    response.headers["x-robots-tag"] = 'noindex, nofollow'
+    return response
 
 
-# Create application instance
-app = create_app()
+origins = [
+    "*",
+]
 
-# For running with a proper ASGI server like uvicorn
-if __name__ == "__main__":
-    import uvicorn
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+app.include_router(core.public_app)
+app.include_router(core.app)
+app.include_router(core.app_apikey)
+app.include_router(core_dynamic.public_app)
+app.include_router(core_dynamic.app)
+app.include_router(core_dynamic.app_apikey)
+app.include_router(metrics.app)
+app.include_router(insights.app)
+app.include_router(v1_api.app_apikey)
+app.include_router(health.public_app)
+app.include_router(health.app)
+app.include_router(health.app_apikey)
 
-    uvicorn.run("app:app", host="0.0.0.0", port=8000, reload=True)
+app.include_router(usability_tests.public_app)
+app.include_router(usability_tests.app)
+app.include_router(usability_tests.app_apikey)
+
+app.include_router(spot.public_app)
+app.include_router(spot.app)
+app.include_router(spot.app_apikey)
+
+app.include_router(product_anaytics.public_app)
+app.include_router(product_anaytics.app)
+app.include_router(product_anaytics.app_apikey)

api/boot/config.py

@@ -1,78 +0,0 @@
-import logging
-from typing import Dict, Any, List, Optional
-from decouple import config as decouple_config
-
-
-class Settings:
-    """Centralized application settings."""
-
-    # Application settings
-    APP_NAME: str = decouple_config("APP_NAME", default="OpenReplay")
-    ROOT_PATH: str = decouple_config("root_path", default="/api")
-    DOCS_URL: str = decouple_config("docs_url", default="")
-    REDOC_URL: str = decouple_config("redoc_url", default="")
-    TRACK_TIME: bool = decouple_config("TRACK_TIME", default=False, cast=bool)
-
-    # Logging - get raw string value without casting
-    _LOG_LEVEL_STR: str = decouple_config("LOGLEVEL", default="WARNING")
-    LOG_FORMAT: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-
-    # PostgreSQL settings
-    PG_HOST: str = decouple_config("pg_host", default="localhost")
-    PG_DBNAME: str = decouple_config("pg_dbname", default="orpy")
-    PG_USER: str = decouple_config("pg_user", default="orpy")
-    PG_PASSWORD: str = decouple_config("pg_password", default="orpy")
-    PG_PORT: int = decouple_config("pg_port", cast=int, default=5432)
-    PG_AIO_MINCONN: int = decouple_config("PG_AIO_MINCONN", cast=int, default=1)
-    PG_AIO_MAXCONN: int = decouple_config("PG_AIO_MAXCONN", cast=int, default=5)
-
-    # CORS settings
-    CORS_ORIGINS: List[str] = ["*"]
-    CORS_ALLOW_CREDENTIALS: bool = True
-    CORS_ALLOW_METHODS: List[str] = ["*"]
-    CORS_ALLOW_HEADERS: List[str] = ["*"]
-
-    @property
-    def LOG_LEVEL(self) -> int:
-        """Convert string log level to an integer constant."""
-        level_map = {
-            "CRITICAL": logging.CRITICAL,
-            "FATAL": logging.FATAL,
-            "ERROR": logging.ERROR,
-            "WARNING": logging.WARNING,
-            "WARN": logging.WARN,
-            "INFO": logging.INFO,
-            "DEBUG": logging.DEBUG,
-            "NOTSET": logging.NOTSET,
-        }
-
-        # If it's a digit string, convert to int
-        if self._LOG_LEVEL_STR.isdigit():
-            return int(self._LOG_LEVEL_STR)
-
-        # If it's a known level name, return the corresponding value
-        return level_map.get(self._LOG_LEVEL_STR.upper(), logging.WARNING)
-
-    @property
-    def pg_dsn(self) -> Dict[str, Any]:
-        """Return PostgreSQL connection parameters as a dictionary."""
-        return {
-            "host": self.PG_HOST,
-            "dbname": self.PG_DBNAME,
-            "user": self.PG_USER,
-            "password": self.PG_PASSWORD,
-            "port": self.PG_PORT,
-            "application_name": f"AIO{self.APP_NAME}",
-        }
-
-    def configure_logging(self) -> None:
-        """Configure application logging."""
-        logging.basicConfig(
-            level=self.LOG_LEVEL,
-            format=self.LOG_FORMAT
-        )
-        logging.info(f">Loglevel set to: {self._LOG_LEVEL_STR} ({self.LOG_LEVEL})")
-
-
-# Create a singleton instance
-settings = Settings()

api/boot/health/router.py

@@ -1,82 +0,0 @@
-import logging
-import time
-import psutil
-from typing import Dict, Any
-
-from fastapi import APIRouter, status
-from pydantic import BaseModel
-
-from boot.scheduler.manager import SchedulerManager
-from chalicelib.utils import pg_client, ch_client
-
-logger = logging.getLogger(__name__)
-
-# Create router for health endpoints
-health_router = APIRouter(prefix="/health", tags=["Health"])
-
-
-class HealthResponse(BaseModel):
-    """Health check response model."""
-    status: str
-    details: Dict[str, Any]
-
-
-@health_router.get("/liveness", response_model=HealthResponse, status_code=status.HTTP_200_OK)
-async def liveness():
-    """
-    Liveness probe endpoint.
-
-    This endpoint indicates if the application is running.
-    It's designed to be quick and does not check external dependencies.
-    """
-    return HealthResponse(
-        status="alive",
-        details={
-            "uptime_seconds": time.time() - psutil.boot_time(),
-            "process": {
-                "cpu_percent": psutil.cpu_percent(),
-                "memory_percent": psutil.virtual_memory().percent
-            }
-        }
-    )
-
-
-@health_router.get("/readiness", response_model=HealthResponse)
-async def readiness():
-    """
-    Readiness probe endpoint.
-
-    This endpoint indicates if the application is ready to handle requests.
-    It checks connections to dependent services.
-    """
-    # Check PostgreSQL connection
-    pg_healthy = await pg_client.PostgresClient.health_check()
-
-    # Check ClickHouse connection
-    ch_healthy = await ch_client.ClickHouseClient.health_check()
-
-    # Check scheduler
-    scheduler_healthy = SchedulerManager.health_check()
-
-    # Overall status
-    all_healthy = pg_healthy and ch_healthy and scheduler_healthy
-    status_code = status.HTTP_200_OK if all_healthy else status.HTTP_503_SERVICE_UNAVAILABLE
-
-    response = HealthResponse(
-        status="ready" if all_healthy else "not ready",
-        details={
-            "services": {
-                "postgres": "healthy" if pg_healthy else "unhealthy",
-                "clickhouse": "healthy" if ch_healthy else "unhealthy",
-                "scheduler": "healthy" if scheduler_healthy else "unhealthy"
-            },
-            "system": {
-                "disk_usage_percent": psutil.disk_usage('/').percent,
-                "memory_available_mb": psutil.virtual_memory().available / (1024 * 1024),
-                "cpu_count": psutil.cpu_count(),
-                "cpu_percent": psutil.cpu_percent()
-            }
-        }
-    )
-
-    return response

api/boot/middleware/__init__.py

@@ -1,8 +0,0 @@
-"""
-Middleware components for the FastAPI application.
-"""
-
-from .timing import timing_middleware
-from .cors import setup_cors
-
-__all__ = ["timing_middleware", "setup_cors"]

api/boot/middleware/cors.py

@@ -1,15 +0,0 @@
-from fastapi import FastAPI
-from fastapi.middleware.cors import CORSMiddleware
-
-from boot.config import settings
-
-
-def setup_cors(app: FastAPI) -> None:
-    """Configure CORS middleware for the application."""
-    app.add_middleware(
-        CORSMiddleware,
-        allow_origins=settings.CORS_ORIGINS,
-        allow_credentials=settings.CORS_ALLOW_CREDENTIALS,
-        allow_methods=settings.CORS_ALLOW_METHODS,
-        allow_headers=settings.CORS_ALLOW_HEADERS,
-    )

api/boot/middleware/timing.py

@@ -1,42 +0,0 @@
-import logging
-import time
-from fastapi import Request
-from starlette.responses import StreamingResponse
-
-from boot.config import settings
-from chalicelib.utils import helper
-
-logger = logging.getLogger(__name__)
-
-
-async def timing_middleware(request: Request, call_next):
-    """
-    Middleware to track request timing and log slow requests.
-    Also adds security headers to responses.
-    """
-    if helper.TRACK_TIME:
-        start_time = time.time()
-
-    try:
-        response: StreamingResponse = await call_next(request)
-    except Exception as e:
-        path = request.url.path
-        method = request.method
-        logger.error(f"{method}: {path} FAILED! Error: {str(e)}")
-        raise
-
-    # Log non-successful responses
-    if response.status_code // 100 != 2:
-        logger.warning(f"{request.method}:{request.url.path} {response.status_code}!")
-
-    # Track execution time
-    if helper.TRACK_TIME:
-        elapsed = time.time() - start_time
-        if elapsed > 2:
-            elapsed = round(elapsed, 2)
-            logger.warning(f"Execution time: {elapsed} s for {request.method}: {request.url.path}")
-
-    # Set security headers
-    response.headers["x-robots-tag"] = 'noindex, nofollow'
-
-    return response

api/boot/scheduler/manager.py

@@ -1,71 +0,0 @@
-import logging
-from typing import List, Dict, Any
-
-from apscheduler.schedulers.asyncio import AsyncIOScheduler
-
-logger = logging.getLogger(__name__)
-
-
-class SchedulerManager:
-    """Scheduler manager for cron jobs."""
-
-    _scheduler: AsyncIOScheduler = None
-
-    @classmethod
-    def initialize(cls, jobs: List[Dict[str, Any]]) -> None:
-        """Initialize the scheduler with jobs."""
-        if cls._scheduler is not None:
-            logger.warning("Scheduler already initialized")
-            return
-
-        # Configure APScheduler logger
-        ap_logger = logging.getLogger('apscheduler')
-        ap_logger.setLevel(logging.getLogger().level)
-
-        # Create scheduler
-        cls._scheduler = AsyncIOScheduler()
-
-        # Add all jobs to the scheduler
-        for job in jobs:
-            cls._scheduler.add_job(id=job["func"].__name__, **job)
-
-        # Log scheduled jobs
-        ap_logger.info(">Scheduled jobs:")
-        for job in cls._scheduler.get_jobs():
-            # Get the job information safely without depending on next_run_time
-            job_info = {
-                "Name": str(job.id),
-                "Run Frequency": str(job.trigger),
-            }
-
-            # Try to get next run time if available, otherwise skip it
-            try:
-                if hasattr(job, "next_run_time"):
-                    job_info["Next Run"] = str(job.next_run_time)
-                elif hasattr(job, "_get_run_times"):
-                    # Try to get the next run time using _get_run_times
-                    run_times = job._get_run_times(1)
-                    if run_times:
-                        job_info["Next Run"] = str(run_times[0])
-            except Exception as e:
-                job_info["Next Run"] = "Unknown"
-                logger.debug(f"Could not determine next run time for job {job.id}: {e}")
-
-            ap_logger.info(job_info)
-
-        # Start the scheduler
-        cls._scheduler.start()
-        logger.info("Scheduler started")
-
-    @classmethod
-    def shutdown(cls) -> None:
-        """Shutdown the scheduler."""
-        if cls._scheduler is not None:
-            cls._scheduler.shutdown(wait=False)
-            cls._scheduler = None
-            logger.info("Scheduler shutdown")
-
-    @classmethod
-    def health_check(cls) -> bool:
-        """Check scheduler health."""
-        return cls._scheduler is not None and cls._scheduler.running

api/chalicelib/core/alerts/alerts_processor.py

@@ -4,7 +4,8 @@ from pydantic_core._pydantic_core import ValidationError
 
 import schemas
 from chalicelib.core.alerts import alerts, alerts_listener
-from chalicelib.core.alerts.modules import sessions, alert_helpers
+from chalicelib.core.alerts.modules import alert_helpers
+from chalicelib.core.sessions import sessions_pg as sessions
 from chalicelib.utils import pg_client
 from chalicelib.utils.TimeUTC import TimeUTC
 
@@ -131,6 +132,7 @@ def Build(a):
 
 
 def process():
+    logger.info("> processing alerts on PG")
     notifications = []
     all_alerts = alerts_listener.get_all_alerts()
     with pg_client.PostgresClient() as cur:

api/chalicelib/core/alerts/alerts_processor_ch.py

@@ -5,8 +5,9 @@ from pydantic_core._pydantic_core import ValidationError
 import schemas
 from chalicelib.utils import pg_client, ch_client, exp_ch_helper
 from chalicelib.utils.TimeUTC import TimeUTC
-from . import alerts, alerts_listener
-from .modules import sessions, alert_helpers
+from chalicelib.core.alerts import alerts, alerts_listener
+from chalicelib.core.alerts.modules import alert_helpers
+from chalicelib.core.sessions import sessions_ch as sessions
 
 logger = logging.getLogger(__name__)
 
@@ -155,6 +156,7 @@ def Build(a):
 
 
 def process():
+    logger.info("> processing alerts on CH")
     notifications = []
     all_alerts = alerts_listener.get_all_alerts()
     with pg_client.PostgresClient() as cur, ch_client.ClickHouseClient() as ch_cur:

api/chalicelib/core/alerts/modules/__init__.py

@@ -1,9 +1,3 @@
-from decouple import config
-
 TENANT_ID = "-1"
-if config("EXP_ALERTS", cast=bool, default=False):
-    from chalicelib.core.sessions import sessions_ch as sessions
-else:
-    from chalicelib.core.sessions import sessions
 
 from . import helpers as alert_helpers

api/chalicelib/core/authorizers.py

@@ -28,9 +28,6 @@ def jwt_authorizer(scheme: str, token: str, leeway=0) -> dict | None:
     if scheme.lower() != "bearer":
         return None
     try:
-        logger.warning("Checking JWT token: %s", token)
-        logger.warning("Against: %s", config("JWT_SECRET") if not is_spot_token(token) else config("JWT_SPOT_SECRET"))
-        logger.warning(get_supported_audience())
         payload = jwt.decode(jwt=token,
                              key=config("JWT_SECRET") if not is_spot_token(token) else config("JWT_SPOT_SECRET"),
                              algorithms=config("JWT_ALGORITHM"),
@@ -40,8 +37,7 @@ def jwt_authorizer(scheme: str, token: str, leeway=0) -> dict | None:
         logger.debug("! JWT Expired signature")
         return None
     except BaseException as e:
-        logger.warning("! JWT Base Exception")
-        logger.debug(e)
+        logger.warning("! JWT Base Exception", exc_info=e)
         return None
     return payload
 
@@ -50,10 +46,6 @@ def jwt_refresh_authorizer(scheme: str, token: str):
     if scheme.lower() != "bearer":
         return None
     try:
-        logger.warning("Checking JWT REF token: %s", token)
-        logger.warning("Against REF: %s",
-                       config("JWT_REFRESH_SECRET") if not is_spot_token(token) else config("JWT_SPOT_REFRESH_SECRET"))
-        logger.warning(get_supported_audience())
         payload = jwt.decode(jwt=token,
                              key=config("JWT_REFRESH_SECRET") if not is_spot_token(token) \
                                  else config("JWT_SPOT_REFRESH_SECRET"),
@@ -63,8 +55,7 @@ def jwt_refresh_authorizer(scheme: str, token: str):
         logger.debug("! JWT-refresh Expired signature")
         return None
     except BaseException as e:
-        logger.warning("! JWT-refresh Base Exception")
-        logger.debug(e)
+        logger.error("! JWT-refresh Base Exception", exc_info=e)
         return None
     return payload
 
@@ -83,10 +74,6 @@ def generate_jwt(user_id, tenant_id, iat, aud, for_spot=False):
         key=config("JWT_SECRET") if not for_spot else config("JWT_SPOT_SECRET"),
         algorithm=config("JWT_ALGORITHM")
     )
-    logger.warning("Generated JWT token: %s", token)
-    logger.warning("For spot: %s", for_spot)
-    logger.warning("Using: %s", config("JWT_SECRET") if not for_spot else config("JWT_SPOT_SECRET"))
-    logger.warning(aud)
     return token
 
 
@@ -105,10 +92,6 @@ def generate_jwt_refresh(user_id, tenant_id, iat, aud, jwt_jti, for_spot=False):
         key=config("JWT_REFRESH_SECRET") if not for_spot else config("JWT_SPOT_REFRESH_SECRET"),
         algorithm=config("JWT_ALGORITHM")
     )
-    logger.warning("Generated JWT REF token: %s", token)
-    logger.warning("For spot REF: %s", for_spot)
-    logger.warning("Using: %s", config("JWT_REFRESH_SECRET") if not for_spot else config("JWT_SPOT_REFRESH_SECRET"))
-    logger.warning(aud)
     return token
 
 

api/chalicelib/core/autocomplete/autocomplete.py

@@ -85,7 +85,8 @@ def __generic_query(typename, value_length=None):
                       ORDER BY value"""
 
     if value_length is None or value_length > 2:
-        return f"""(SELECT DISTINCT value, type
+        return f"""SELECT DISTINCT ON(value,type) value, type
+                   ((SELECT DISTINCT value, type
                     FROM {TABLE}
                     WHERE
                       project_id = %(project_id)s
@@ -101,7 +102,7 @@ def __generic_query(typename, value_length=None):
                       AND type='{typename.upper()}'
                       AND value ILIKE %(value)s
                       ORDER BY value
-                    LIMIT 5);"""
+                    LIMIT 5)) AS raw;"""
     return f"""SELECT DISTINCT value, type
                 FROM {TABLE}
                 WHERE
@@ -124,7 +125,7 @@ def __generic_autocomplete(event: Event):
     return f
 
 
-def __generic_autocomplete_metas(typename):
+def generic_autocomplete_metas(typename):
     def f(project_id, text):
         with pg_client.PostgresClient() as cur:
             params = {"project_id": project_id, "value": helper.string_to_sql_like(text),
@@ -326,7 +327,7 @@ def __search_metadata(project_id, value, key=None, source=None):
                                 AND {colname} ILIKE %(svalue)s LIMIT 5)""")
     with pg_client.PostgresClient() as cur:
         cur.execute(cur.mogrify(f"""\
-                    SELECT key, value, 'METADATA' AS TYPE
+                    SELECT DISTINCT ON(key, value) key, value, 'METADATA' AS TYPE
                     FROM({" UNION ALL ".join(sub_from)}) AS all_metas
                     LIMIT 5;""", {"project_id": project_id, "value": helper.string_to_sql_like(value),
                                   "svalue": helper.string_to_sql_like("^" + value)}))

api/chalicelib/core/boarding.py

@@ -13,15 +13,18 @@ def get_state(tenant_id):
 
         if len(pids) > 0:
             cur.execute(
-                cur.mogrify("""SELECT EXISTS((  SELECT 1
+                cur.mogrify(
+                    """SELECT EXISTS((  SELECT 1
                                                 FROM public.sessions AS s
                                                 WHERE s.project_id IN %(ids)s)) AS exists;""",
-                            {"ids": tuple(pids)})
+                    {"ids": tuple(pids)},
+                )
             )
             recorded = cur.fetchone()["exists"]
             meta = False
             if recorded:
-                query = cur.mogrify(f"""SELECT EXISTS((SELECT 1
+                query = cur.mogrify(
+                    f"""SELECT EXISTS((SELECT 1
                                FROM public.projects AS p
                                         LEFT JOIN LATERAL ( SELECT 1
                                                             FROM public.sessions
@@ -36,26 +39,35 @@ def get_state(tenant_id):
                                        OR p.metadata_8 IS NOT NULL OR p.metadata_9 IS NOT NULL
                                        OR p.metadata_10 IS NOT NULL )
                                    )) AS exists;""",
-                                    {"tenant_id": tenant_id})
+                    {"tenant_id": tenant_id},
+                )
                 cur.execute(query)
 
                 meta = cur.fetchone()["exists"]
 
     return [
-        {"task": "Install OpenReplay",
-         "done": recorded,
-         "URL": "https://docs.openreplay.com/getting-started/quick-start"},
-        {"task": "Identify Users",
-         "done": meta,
-         "URL": "https://docs.openreplay.com/data-privacy-security/metadata"},
-        {"task": "Invite Team Members",
-         "done": len(users.get_members(tenant_id=tenant_id)) > 1,
-         "URL": "https://app.openreplay.com/client/manage-users"},
-        {"task": "Integrations",
-         "done": len(datadog.get_all(tenant_id=tenant_id)) > 0 \
-                 or len(sentry.get_all(tenant_id=tenant_id)) > 0 \
-                 or len(stackdriver.get_all(tenant_id=tenant_id)) > 0,
-         "URL": "https://docs.openreplay.com/integrations"}
+        {
+            "task": "Install OpenReplay",
+            "done": recorded,
+            "URL": "https://docs.openreplay.com/getting-started/quick-start",
+        },
+        {
+            "task": "Identify Users",
+            "done": meta,
+            "URL": "https://docs.openreplay.com/data-privacy-security/metadata",
+        },
+        {
+            "task": "Invite Team Members",
+            "done": len(users.get_members(tenant_id=tenant_id)) > 1,
+            "URL": "https://app.openreplay.com/client/manage-users",
+        },
+        {
+            "task": "Integrations",
+            "done": len(datadog.get_all(tenant_id=tenant_id)) > 0
+            or len(sentry.get_all(tenant_id=tenant_id)) > 0
+            or len(stackdriver.get_all(tenant_id=tenant_id)) > 0,
+            "URL": "https://docs.openreplay.com/integrations",
+        },
     ]
 
 
@@ -66,21 +78,26 @@ def get_state_installing(tenant_id):
 
         if len(pids) > 0:
             cur.execute(
-                cur.mogrify("""SELECT EXISTS((  SELECT 1
+                cur.mogrify(
+                    """SELECT EXISTS((  SELECT 1
                                                 FROM public.sessions AS s
                                                 WHERE s.project_id IN %(ids)s)) AS exists;""",
-                            {"ids": tuple(pids)})
+                    {"ids": tuple(pids)},
+                )
             )
             recorded = cur.fetchone()["exists"]
 
-    return {"task": "Install OpenReplay",
-            "done": recorded,
-            "URL": "https://docs.openreplay.com/getting-started/quick-start"}
+    return {
+        "task": "Install OpenReplay",
+        "done": recorded,
+        "URL": "https://docs.openreplay.com/getting-started/quick-start",
+    }
 
 
 def get_state_identify_users(tenant_id):
     with pg_client.PostgresClient() as cur:
-        query = cur.mogrify(f"""SELECT EXISTS((SELECT 1
+        query = cur.mogrify(
+            f"""SELECT EXISTS((SELECT 1
                                        FROM public.projects AS p
                                                 LEFT JOIN LATERAL ( SELECT 1
                                                                     FROM public.sessions
@@ -95,25 +112,32 @@ def get_state_identify_users(tenant_id):
                                                OR p.metadata_8 IS NOT NULL OR p.metadata_9 IS NOT NULL
                                                OR p.metadata_10 IS NOT NULL )
                                            )) AS exists;""",
-                            {"tenant_id": tenant_id})
+            {"tenant_id": tenant_id},
+        )
         cur.execute(query)
 
         meta = cur.fetchone()["exists"]
 
-    return {"task": "Identify Users",
-            "done": meta,
-            "URL": "https://docs.openreplay.com/data-privacy-security/metadata"}
+    return {
+        "task": "Identify Users",
+        "done": meta,
+        "URL": "https://docs.openreplay.com/data-privacy-security/metadata",
+    }
 
 
 def get_state_manage_users(tenant_id):
-    return {"task": "Invite Team Members",
-            "done": len(users.get_members(tenant_id=tenant_id)) > 1,
-            "URL": "https://app.openreplay.com/client/manage-users"}
+    return {
+        "task": "Invite Team Members",
+        "done": len(users.get_members(tenant_id=tenant_id)) > 1,
+        "URL": "https://app.openreplay.com/client/manage-users",
+    }
 
 
 def get_state_integrations(tenant_id):
-    return {"task": "Integrations",
-            "done": len(datadog.get_all(tenant_id=tenant_id)) > 0 \
-                    or len(sentry.get_all(tenant_id=tenant_id)) > 0 \
-                    or len(stackdriver.get_all(tenant_id=tenant_id)) > 0,
-            "URL": "https://docs.openreplay.com/integrations"}
+    return {
+        "task": "Integrations",
+        "done": len(datadog.get_all(tenant_id=tenant_id)) > 0
+        or len(sentry.get_all(tenant_id=tenant_id)) > 0
+        or len(stackdriver.get_all(tenant_id=tenant_id)) > 0,
+        "URL": "https://docs.openreplay.com/integrations",
+    }

api/chalicelib/core/errors/__init__.py

@@ -4,10 +4,10 @@ from decouple import config
 
 logger = logging.getLogger(__name__)
 
-from . import errors as errors_legacy
+from . import errors_pg as errors_legacy
 
 if config("EXP_ERRORS_SEARCH", cast=bool, default=False):
     logger.info(">>> Using experimental error search")
     from . import errors_ch as errors
 else:
-    from . import errors
+    from . import errors_pg as errors

api/chalicelib/core/errors/errors_ch.py

@@ -1,10 +1,11 @@
 import schemas
 from chalicelib.core import metadata
+from chalicelib.core.errors import errors_legacy
+from chalicelib.core.errors.modules import errors_helper
 from chalicelib.core.errors.modules import sessions
 from chalicelib.utils import ch_client, exp_ch_helper
 from chalicelib.utils import helper, metrics_helper
 from chalicelib.utils.TimeUTC import TimeUTC
-from . import errors as errors_legacy
 
 
 def _multiple_values(values, value_key="value"):
@@ -61,25 +62,6 @@ def get_batch(error_ids):
     return errors_legacy.get_batch(error_ids=error_ids)
 
 
-def __get_basic_constraints(platform=None, time_constraint=True, startTime_arg_name="startDate",
-                            endTime_arg_name="endDate", type_condition=True, project_key="project_id", table_name=None):
-    ch_sub_query = [f"{project_key} =toUInt16(%(project_id)s)"]
-    if table_name is not None:
-        table_name = table_name + "."
-    else:
-        table_name = ""
-    if type_condition:
-        ch_sub_query.append(f"{table_name}`$event_name`='ERROR'")
-    if time_constraint:
-        ch_sub_query += [f"{table_name}datetime >= toDateTime(%({startTime_arg_name})s/1000)",
-                         f"{table_name}datetime < toDateTime(%({endTime_arg_name})s/1000)"]
-    if platform == schemas.PlatformType.MOBILE:
-        ch_sub_query.append("user_device_type = 'mobile'")
-    elif platform == schemas.PlatformType.DESKTOP:
-        ch_sub_query.append("user_device_type = 'desktop'")
-    return ch_sub_query
-
-
 def __get_basic_constraints_events(platform=None, time_constraint=True, startTime_arg_name="startDate",
                                    endTime_arg_name="endDate", type_condition=True, project_key="project_id",
                                    table_name=None):
@@ -116,7 +98,7 @@ def search(data: schemas.SearchErrorsSchema, project: schemas.ProjectContext, us
     for f in data.filters:
         if f.type == schemas.FilterType.PLATFORM and len(f.value) > 0:
             platform = f.value[0]
-    ch_sessions_sub_query = __get_basic_constraints(platform, type_condition=False)
+    ch_sessions_sub_query = errors_helper.__get_basic_constraints_ch(platform, type_condition=False)
     # ignore platform for errors table
     ch_sub_query = __get_basic_constraints_events(None, type_condition=True)
     ch_sub_query.append("JSONExtractString(toString(`$properties`), 'source') = 'js_exception'")
@@ -148,7 +130,8 @@ def search(data: schemas.SearchErrorsSchema, project: schemas.ProjectContext, us
         if len(data.events) > errors_condition_count:
             subquery_part_args, subquery_part = sessions.search_query_parts_ch(data=data, error_status=data.status,
                                                                                errors_only=True,
-                                                                               project_id=project.project_id, user_id=user_id,
+                                                                               project_id=project.project_id,
+                                                                               user_id=user_id,
                                                                                issue=None,
                                                                                favorite_only=False)
             subquery_part = f"INNER JOIN {subquery_part} USING(session_id)"
@@ -355,14 +338,14 @@ def search(data: schemas.SearchErrorsSchema, project: schemas.ProjectContext, us
                 SELECT details.error_id as error_id,
                         name, message, users, total, 
                         sessions, last_occurrence, first_occurrence, chart
-                FROM (SELECT JSONExtractString(toString(`$properties`), 'error_id') AS error_id,
+                FROM (SELECT error_id,
                              JSONExtractString(toString(`$properties`), 'name') AS name,
                              JSONExtractString(toString(`$properties`), 'message') AS message,
                              COUNT(DISTINCT user_id)  AS users,
                              COUNT(DISTINCT events.session_id) AS sessions,
                              MAX(created_at)              AS max_datetime,
                              MIN(created_at)              AS min_datetime,
-                             COUNT(DISTINCT JSONExtractString(toString(`$properties`), 'error_id')) 
+                             COUNT(DISTINCT error_id) 
                                 OVER() AS total
                       FROM {MAIN_EVENTS_TABLE} AS events
                             INNER JOIN (SELECT session_id, coalesce(user_id,toString(user_uuid)) AS user_id 
@@ -374,7 +357,7 @@ def search(data: schemas.SearchErrorsSchema, project: schemas.ProjectContext, us
                       GROUP BY error_id, name, message
                       ORDER BY {sort} {order}
                       LIMIT %(errors_limit)s OFFSET %(errors_offset)s) AS details 
-                        INNER JOIN (SELECT JSONExtractString(toString(`$properties`), 'error_id') AS error_id, 
+                        INNER JOIN (SELECT error_id, 
                                             toUnixTimestamp(MAX(created_at))*1000 AS last_occurrence, 
                                             toUnixTimestamp(MIN(created_at))*1000 AS first_occurrence
                                      FROM {MAIN_EVENTS_TABLE}
@@ -383,7 +366,7 @@ def search(data: schemas.SearchErrorsSchema, project: schemas.ProjectContext, us
                                      GROUP BY error_id) AS time_details
                 ON details.error_id=time_details.error_id
                     INNER JOIN (SELECT error_id, groupArray([timestamp, count]) AS chart
-                    FROM (SELECT JSONExtractString(toString(`$properties`), 'error_id') AS error_id, 
+                    FROM (SELECT error_id, 
                                  gs.generate_series AS timestamp,
                                  COUNT(DISTINCT session_id) AS count
                             FROM generate_series(%(startDate)s, %(endDate)s, %(step_size)s) AS gs

api/chalicelib/core/errors/errors_details.py

@@ -1,5 +1,5 @@
-from chalicelib.core.errors import errors_legacy as errors
-from chalicelib.utils import errors_helper
+from chalicelib.core.errors.modules import errors_helper
+
 from chalicelib.utils import pg_client, helper
 from chalicelib.utils.TimeUTC import TimeUTC
 from chalicelib.utils.metrics_helper import get_step_size
@@ -40,26 +40,29 @@ def __process_tags(row):
 
 
 def get_details(project_id, error_id, user_id, **data):
-    pg_sub_query24 = errors.__get_basic_constraints(time_constraint=False, chart=True, step_size_name="step_size24")
+    pg_sub_query24 = errors_helper.__get_basic_constraints(time_constraint=False, chart=True,
+                                                           step_size_name="step_size24")
     pg_sub_query24.append("error_id = %(error_id)s")
-    pg_sub_query30_session = errors.__get_basic_constraints(time_constraint=True, chart=False,
-                                                            startTime_arg_name="startDate30",
-                                                            endTime_arg_name="endDate30",
-                                                            project_key="sessions.project_id")
+    pg_sub_query30_session = errors_helper.__get_basic_constraints(time_constraint=True, chart=False,
+                                                                   startTime_arg_name="startDate30",
+                                                                   endTime_arg_name="endDate30",
+                                                                   project_key="sessions.project_id")
     pg_sub_query30_session.append("sessions.start_ts >= %(startDate30)s")
     pg_sub_query30_session.append("sessions.start_ts <= %(endDate30)s")
     pg_sub_query30_session.append("error_id = %(error_id)s")
-    pg_sub_query30_err = errors.__get_basic_constraints(time_constraint=True, chart=False,
-                                                        startTime_arg_name="startDate30",
-                                                        endTime_arg_name="endDate30", project_key="errors.project_id")
+    pg_sub_query30_err = errors_helper.__get_basic_constraints(time_constraint=True, chart=False,
+                                                               startTime_arg_name="startDate30",
+                                                               endTime_arg_name="endDate30",
+                                                               project_key="errors.project_id")
     pg_sub_query30_err.append("sessions.project_id = %(project_id)s")
     pg_sub_query30_err.append("sessions.start_ts >= %(startDate30)s")
     pg_sub_query30_err.append("sessions.start_ts <= %(endDate30)s")
     pg_sub_query30_err.append("error_id = %(error_id)s")
     pg_sub_query30_err.append("source ='js_exception'")
-    pg_sub_query30 = errors.__get_basic_constraints(time_constraint=False, chart=True, step_size_name="step_size30")
+    pg_sub_query30 = errors_helper.__get_basic_constraints(time_constraint=False, chart=True,
+                                                           step_size_name="step_size30")
     pg_sub_query30.append("error_id = %(error_id)s")
-    pg_basic_query = errors.__get_basic_constraints(time_constraint=False)
+    pg_basic_query = errors_helper.__get_basic_constraints(time_constraint=False)
     pg_basic_query.append("error_id = %(error_id)s")
     with pg_client.PostgresClient() as cur:
         data["startDate24"] = TimeUTC.now(-1)
@@ -95,8 +98,7 @@ def get_details(project_id, error_id, user_id, **data):
                device_partition,
                country_partition,
                chart24,
-               chart30,
-               custom_tags
+               chart30
         FROM (SELECT error_id,
                      name,
                      message,
@@ -111,15 +113,8 @@ def get_details(project_id, error_id, user_id, **data):
                                     MIN(timestamp) AS first_occurrence
                              FROM events.errors
                              WHERE error_id = %(error_id)s) AS time_details ON (TRUE)
-                 INNER JOIN (SELECT session_id AS last_session_id,
-                                    coalesce(custom_tags, '[]')::jsonb AS custom_tags
+                 INNER JOIN (SELECT session_id AS last_session_id
                              FROM events.errors
-                             LEFT JOIN LATERAL (
-                                    SELECT jsonb_agg(jsonb_build_object(errors_tags.key, errors_tags.value)) AS custom_tags
-                                    FROM errors_tags
-                                    WHERE errors_tags.error_id = %(error_id)s
-                                      AND errors_tags.session_id = errors.session_id
-                                      AND errors_tags.message_id = errors.message_id) AS errors_tags ON (TRUE)
                              WHERE error_id = %(error_id)s
                              ORDER BY errors.timestamp DESC
                              LIMIT 1) AS last_session_details ON (TRUE)

api/chalicelib/core/errors/errors_pg.py

@@ -1,7 +1,8 @@
 import json
-from typing import Optional, List
+from typing import List
 
 import schemas
+from chalicelib.core.errors.modules import errors_helper
 from chalicelib.core.sessions import sessions_search
 from chalicelib.core.sourcemaps import sourcemaps
 from chalicelib.utils import pg_client, helper
@@ -51,27 +52,6 @@ def get_batch(error_ids):
         return helper.list_to_camel_case(errors)
 
 
-def __get_basic_constraints(platform: Optional[schemas.PlatformType] = None, time_constraint: bool = True,
-                            startTime_arg_name: str = "startDate", endTime_arg_name: str = "endDate",
-                            chart: bool = False, step_size_name: str = "step_size",
-                            project_key: Optional[str] = "project_id"):
-    if project_key is None:
-        ch_sub_query = []
-    else:
-        ch_sub_query = [f"{project_key} =%(project_id)s"]
-    if time_constraint:
-        ch_sub_query += [f"timestamp >= %({startTime_arg_name})s",
-                         f"timestamp < %({endTime_arg_name})s"]
-    if chart:
-        ch_sub_query += [f"timestamp >=  generated_timestamp",
-                         f"timestamp <  generated_timestamp + %({step_size_name})s"]
-    if platform == schemas.PlatformType.MOBILE:
-        ch_sub_query.append("user_device_type = 'mobile'")
-    elif platform == schemas.PlatformType.DESKTOP:
-        ch_sub_query.append("user_device_type = 'desktop'")
-    return ch_sub_query
-
-
 def __get_sort_key(key):
     return {
         schemas.ErrorSort.OCCURRENCE: "max_datetime",
@@ -90,12 +70,13 @@ def search(data: schemas.SearchErrorsSchema, project: schemas.ProjectContext, us
     for f in data.filters:
         if f.type == schemas.FilterType.PLATFORM and len(f.value) > 0:
             platform = f.value[0]
-    pg_sub_query = __get_basic_constraints(platform, project_key="sessions.project_id")
+    pg_sub_query = errors_helper.__get_basic_constraints(platform, project_key="sessions.project_id")
     pg_sub_query += ["sessions.start_ts>=%(startDate)s", "sessions.start_ts<%(endDate)s", "source ='js_exception'",
                      "pe.project_id=%(project_id)s"]
     # To ignore Script error
     pg_sub_query.append("pe.message!='Script error.'")
-    pg_sub_query_chart = __get_basic_constraints(platform, time_constraint=False, chart=True, project_key=None)
+    pg_sub_query_chart = errors_helper.__get_basic_constraints(platform, time_constraint=False, chart=True,
+                                                               project_key=None)
     if platform:
         pg_sub_query_chart += ["start_ts>=%(startDate)s", "start_ts<%(endDate)s", "project_id=%(project_id)s"]
     pg_sub_query_chart.append("errors.error_id =details.error_id")

api/chalicelib/core/errors/modules/__init__.py

@@ -3,8 +3,9 @@ import logging
 from decouple import config
 
 logger = logging.getLogger(__name__)
+from . import helper as errors_helper
 
 if config("EXP_ERRORS_SEARCH", cast=bool, default=False):
-    from chalicelib.core.sessions import sessions_ch as sessions
+    import chalicelib.core.sessions.sessions_ch as sessions
 else:
-    from chalicelib.core.sessions import sessions
+    import chalicelib.core.sessions.sessions_pg as sessions

api/chalicelib/core/errors/modules/helper.py

@@ -0,0 +1,58 @@
+from typing import Optional
+
+import schemas
+from chalicelib.core.sourcemaps import sourcemaps
+
+
+def __get_basic_constraints(platform: Optional[schemas.PlatformType] = None, time_constraint: bool = True,
+                            startTime_arg_name: str = "startDate", endTime_arg_name: str = "endDate",
+                            chart: bool = False, step_size_name: str = "step_size",
+                            project_key: Optional[str] = "project_id"):
+    if project_key is None:
+        ch_sub_query = []
+    else:
+        ch_sub_query = [f"{project_key} =%(project_id)s"]
+    if time_constraint:
+        ch_sub_query += [f"timestamp >= %({startTime_arg_name})s",
+                         f"timestamp < %({endTime_arg_name})s"]
+    if chart:
+        ch_sub_query += [f"timestamp >=  generated_timestamp",
+                         f"timestamp <  generated_timestamp + %({step_size_name})s"]
+    if platform == schemas.PlatformType.MOBILE:
+        ch_sub_query.append("user_device_type = 'mobile'")
+    elif platform == schemas.PlatformType.DESKTOP:
+        ch_sub_query.append("user_device_type = 'desktop'")
+    return ch_sub_query
+
+
+def __get_basic_constraints_ch(platform=None, time_constraint=True, startTime_arg_name="startDate",
+                               endTime_arg_name="endDate", type_condition=True, project_key="project_id",
+                               table_name=None):
+    ch_sub_query = [f"{project_key} =toUInt16(%(project_id)s)"]
+    if table_name is not None:
+        table_name = table_name + "."
+    else:
+        table_name = ""
+    if type_condition:
+        ch_sub_query.append(f"{table_name}`$event_name`='ERROR'")
+    if time_constraint:
+        ch_sub_query += [f"{table_name}datetime >= toDateTime(%({startTime_arg_name})s/1000)",
+                         f"{table_name}datetime < toDateTime(%({endTime_arg_name})s/1000)"]
+    if platform == schemas.PlatformType.MOBILE:
+        ch_sub_query.append("user_device_type = 'mobile'")
+    elif platform == schemas.PlatformType.DESKTOP:
+        ch_sub_query.append("user_device_type = 'desktop'")
+    return ch_sub_query
+
+
+def format_first_stack_frame(error):
+    error["stack"] = sourcemaps.format_payload(error.pop("payload"), truncate_to_first=True)
+    for s in error["stack"]:
+        for c in s.get("context", []):
+            for sci, sc in enumerate(c):
+                if isinstance(sc, str) and len(sc) > 1000:
+                    c[sci] = sc[:1000]
+        # convert bytes to string:
+        if isinstance(s["filename"], bytes):
+            s["filename"] = s["filename"].decode("utf-8")
+    return error

api/chalicelib/core/events.py

@@ -1,8 +1,9 @@
+from functools import cache
 from typing import Optional
 
 import schemas
-from chalicelib.core.autocomplete import autocomplete
 from chalicelib.core import issues
+from chalicelib.core.autocomplete import autocomplete
 from chalicelib.core.sessions import sessions_metas
 from chalicelib.utils import pg_client, helper
 from chalicelib.utils.TimeUTC import TimeUTC
@@ -137,52 +138,57 @@ class EventType:
                          column=None)  # column=None because errors are searched by name or message
 
 
-SUPPORTED_TYPES = {
-    EventType.CLICK.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.CLICK),
-                                             query=autocomplete.__generic_query(typename=EventType.CLICK.ui_type)),
-    EventType.INPUT.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.INPUT),
-                                             query=autocomplete.__generic_query(typename=EventType.INPUT.ui_type)),
-    EventType.LOCATION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.LOCATION),
-                                                query=autocomplete.__generic_query(
-                                                    typename=EventType.LOCATION.ui_type)),
-    EventType.CUSTOM.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.CUSTOM),
-                                              query=autocomplete.__generic_query(typename=EventType.CUSTOM.ui_type)),
-    EventType.REQUEST.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.REQUEST),
-                                               query=autocomplete.__generic_query(
-                                                   typename=EventType.REQUEST.ui_type)),
-    EventType.GRAPHQL.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.GRAPHQL),
-                                               query=autocomplete.__generic_query(
-                                                   typename=EventType.GRAPHQL.ui_type)),
-    EventType.STATEACTION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.STATEACTION),
+@cache
+def supported_types():
+    return {
+        EventType.CLICK.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.CLICK),
+                                                 query=autocomplete.__generic_query(typename=EventType.CLICK.ui_type)),
+        EventType.INPUT.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.INPUT),
+                                                 query=autocomplete.__generic_query(typename=EventType.INPUT.ui_type)),
+        EventType.LOCATION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.LOCATION),
+                                                    query=autocomplete.__generic_query(
+                                                        typename=EventType.LOCATION.ui_type)),
+        EventType.CUSTOM.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.CUSTOM),
+                                                  query=autocomplete.__generic_query(
+                                                      typename=EventType.CUSTOM.ui_type)),
+        EventType.REQUEST.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.REQUEST),
                                                    query=autocomplete.__generic_query(
-                                                       typename=EventType.STATEACTION.ui_type)),
-    EventType.TAG.ui_type: SupportedFilter(get=_search_tags, query=None),
-    EventType.ERROR.ui_type: SupportedFilter(get=autocomplete.__search_errors,
-                                             query=None),
-    EventType.METADATA.ui_type: SupportedFilter(get=autocomplete.__search_metadata,
-                                                query=None),
-    #     MOBILE
-    EventType.CLICK_MOBILE.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.CLICK_MOBILE),
-                                                    query=autocomplete.__generic_query(
-                                                        typename=EventType.CLICK_MOBILE.ui_type)),
-    EventType.SWIPE_MOBILE.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.SWIPE_MOBILE),
-                                                    query=autocomplete.__generic_query(
-                                                        typename=EventType.SWIPE_MOBILE.ui_type)),
-    EventType.INPUT_MOBILE.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.INPUT_MOBILE),
-                                                    query=autocomplete.__generic_query(
-                                                        typename=EventType.INPUT_MOBILE.ui_type)),
-    EventType.VIEW_MOBILE.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.VIEW_MOBILE),
+                                                       typename=EventType.REQUEST.ui_type)),
+        EventType.GRAPHQL.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.GRAPHQL),
                                                    query=autocomplete.__generic_query(
-                                                       typename=EventType.VIEW_MOBILE.ui_type)),
-    EventType.CUSTOM_MOBILE.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.CUSTOM_MOBILE),
-                                                     query=autocomplete.__generic_query(
-                                                         typename=EventType.CUSTOM_MOBILE.ui_type)),
-    EventType.REQUEST_MOBILE.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.REQUEST_MOBILE),
-                                                      query=autocomplete.__generic_query(
-                                                          typename=EventType.REQUEST_MOBILE.ui_type)),
-    EventType.CRASH_MOBILE.ui_type: SupportedFilter(get=autocomplete.__search_errors_mobile,
+                                                       typename=EventType.GRAPHQL.ui_type)),
+        EventType.STATEACTION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.STATEACTION),
+                                                       query=autocomplete.__generic_query(
+                                                           typename=EventType.STATEACTION.ui_type)),
+        EventType.TAG.ui_type: SupportedFilter(get=_search_tags, query=None),
+        EventType.ERROR.ui_type: SupportedFilter(get=autocomplete.__search_errors,
+                                                 query=None),
+        EventType.METADATA.ui_type: SupportedFilter(get=autocomplete.__search_metadata,
                                                     query=None),
-}
+        #     MOBILE
+        EventType.CLICK_MOBILE.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.CLICK_MOBILE),
+                                                        query=autocomplete.__generic_query(
+                                                            typename=EventType.CLICK_MOBILE.ui_type)),
+        EventType.SWIPE_MOBILE.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.SWIPE_MOBILE),
+                                                        query=autocomplete.__generic_query(
+                                                            typename=EventType.SWIPE_MOBILE.ui_type)),
+        EventType.INPUT_MOBILE.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.INPUT_MOBILE),
+                                                        query=autocomplete.__generic_query(
+                                                            typename=EventType.INPUT_MOBILE.ui_type)),
+        EventType.VIEW_MOBILE.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(EventType.VIEW_MOBILE),
+                                                       query=autocomplete.__generic_query(
+                                                           typename=EventType.VIEW_MOBILE.ui_type)),
+        EventType.CUSTOM_MOBILE.ui_type: SupportedFilter(
+            get=autocomplete.__generic_autocomplete(EventType.CUSTOM_MOBILE),
+            query=autocomplete.__generic_query(
+                typename=EventType.CUSTOM_MOBILE.ui_type)),
+        EventType.REQUEST_MOBILE.ui_type: SupportedFilter(
+            get=autocomplete.__generic_autocomplete(EventType.REQUEST_MOBILE),
+            query=autocomplete.__generic_query(
+                typename=EventType.REQUEST_MOBILE.ui_type)),
+        EventType.CRASH_MOBILE.ui_type: SupportedFilter(get=autocomplete.__search_errors_mobile,
+                                                        query=None),
+    }
 
 
 def get_errors_by_session_id(session_id, project_id):
@@ -202,17 +208,17 @@ def search(text, event_type, project_id, source, key):
     if not event_type:
         return {"data": autocomplete.__get_autocomplete_table(text, project_id)}
 
-    if event_type in SUPPORTED_TYPES.keys():
-        rows = SUPPORTED_TYPES[event_type].get(project_id=project_id, value=text, key=key, source=source)
-    elif event_type + "_MOBILE" in SUPPORTED_TYPES.keys():
-        rows = SUPPORTED_TYPES[event_type + "_MOBILE"].get(project_id=project_id, value=text, key=key, source=source)
-    elif event_type in sessions_metas.SUPPORTED_TYPES.keys():
+    if event_type in supported_types().keys():
+        rows = supported_types()[event_type].get(project_id=project_id, value=text, key=key, source=source)
+    elif event_type + "_MOBILE" in supported_types().keys():
+        rows = supported_types()[event_type + "_MOBILE"].get(project_id=project_id, value=text, key=key, source=source)
+    elif event_type in sessions_metas.supported_types().keys():
         return sessions_metas.search(text, event_type, project_id)
     elif event_type.endswith("_IOS") \
-            and event_type[:-len("_IOS")] in sessions_metas.SUPPORTED_TYPES.keys():
+            and event_type[:-len("_IOS")] in sessions_metas.supported_types().keys():
         return sessions_metas.search(text, event_type, project_id)
     elif event_type.endswith("_MOBILE") \
-            and event_type[:-len("_MOBILE")] in sessions_metas.SUPPORTED_TYPES.keys():
+            and event_type[:-len("_MOBILE")] in sessions_metas.supported_types().keys():
         return sessions_metas.search(text, event_type, project_id)
     else:
         return {"errors": ["unsupported event"]}

api/chalicelib/core/health.py

@@ -27,7 +27,6 @@ HEALTH_ENDPOINTS = {
     "http": app_connection_string("http-openreplay", 8888, "metrics"),
     "ingress-nginx": app_connection_string("ingress-nginx-openreplay", 80, "healthz"),
     "integrations": app_connection_string("integrations-openreplay", 8888, "metrics"),
-    "peers": app_connection_string("peers-openreplay", 8888, "health"),
     "sink": app_connection_string("sink-openreplay", 8888, "metrics"),
     "sourcemapreader": app_connection_string(
         "sourcemapreader-openreplay", 8888, "health"
@@ -39,9 +38,7 @@ HEALTH_ENDPOINTS = {
 def __check_database_pg(*_):
     fail_response = {
         "health": False,
-        "details": {
-            "errors": ["Postgres health-check failed"]
-        }
+        "details": {"errors": ["Postgres health-check failed"]},
     }
     with pg_client.PostgresClient() as cur:
         try:
@@ -63,29 +60,26 @@ def __check_database_pg(*_):
         "details": {
             # "version": server_version["server_version"],
             # "schema": schema_version["version"]
-        }
+        },
     }
 
 
 def __always_healthy(*_):
-    return {
-        "health": True,
-        "details": {}
-    }
+    return {"health": True, "details": {}}
 
 
 def __check_be_service(service_name):
     def fn(*_):
         fail_response = {
             "health": False,
-            "details": {
-                "errors": ["server health-check failed"]
-            }
+            "details": {"errors": ["server health-check failed"]},
         }
         try:
             results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2)
             if results.status_code != 200:
-                logger.error(f"!! issue with the {service_name}-health code:{results.status_code}")
+                logger.error(
+                    f"!! issue with the {service_name}-health code:{results.status_code}"
+                )
                 logger.error(results.text)
                 # fail_response["details"]["errors"].append(results.text)
                 return fail_response
@@ -103,10 +97,7 @@ def __check_be_service(service_name):
                 logger.error("couldn't get response")
                 # fail_response["details"]["errors"].append(str(e))
             return fail_response
-        return {
-            "health": True,
-            "details": {}
-        }
+        return {"health": True, "details": {}}
 
     return fn
 
@@ -114,7 +105,7 @@ def __check_be_service(service_name):
 def __check_redis(*_):
     fail_response = {
         "health": False,
-        "details": {"errors": ["server health-check failed"]}
+        "details": {"errors": ["server health-check failed"]},
     }
     if config("REDIS_STRING", default=None) is None:
         # fail_response["details"]["errors"].append("REDIS_STRING not defined in env-vars")
@@ -133,16 +124,14 @@ def __check_redis(*_):
         "health": True,
         "details": {
             # "version": r.execute_command('INFO')['redis_version']
-        }
+        },
     }
 
 
 def __check_SSL(*_):
     fail_response = {
         "health": False,
-        "details": {
-            "errors": ["SSL Certificate health-check failed"]
-        }
+        "details": {"errors": ["SSL Certificate health-check failed"]},
     }
     try:
         requests.get(config("SITE_URL"), verify=True, allow_redirects=True)
@@ -150,36 +139,28 @@ def __check_SSL(*_):
         logger.error("!! health failed: SSL Certificate")
         logger.exception(e)
         return fail_response
-    return {
-        "health": True,
-        "details": {}
-    }
+    return {"health": True, "details": {}}
 
 
 def __get_sessions_stats(*_):
     with pg_client.PostgresClient() as cur:
         constraints = ["projects.deleted_at IS NULL"]
-        query = cur.mogrify(f"""SELECT COALESCE(SUM(sessions_count),0) AS s_c,
+        query = cur.mogrify(
+            f"""SELECT COALESCE(SUM(sessions_count),0) AS s_c,
                                        COALESCE(SUM(events_count),0) AS e_c
                                 FROM public.projects_stats
                                      INNER JOIN public.projects USING(project_id)
-                                WHERE {" AND ".join(constraints)};""")
+                                WHERE {" AND ".join(constraints)};"""
+        )
         cur.execute(query)
         row = cur.fetchone()
-    return {
-        "numberOfSessionsCaptured": row["s_c"],
-        "numberOfEventCaptured": row["e_c"]
-    }
+    return {"numberOfSessionsCaptured": row["s_c"], "numberOfEventCaptured": row["e_c"]}
 
 
 def get_health(tenant_id=None):
     health_map = {
-        "databases": {
-            "postgres": __check_database_pg
-        },
-        "ingestionPipeline": {
-            "redis": __check_redis
-        },
+        "databases": {"postgres": __check_database_pg},
+        "ingestionPipeline": {"redis": __check_redis},
         "backendServices": {
             "alerts": __check_be_service("alerts"),
             "assets": __check_be_service("assets"),
@@ -192,13 +173,12 @@ def get_health(tenant_id=None):
             "http": __check_be_service("http"),
             "ingress-nginx": __always_healthy,
             "integrations": __check_be_service("integrations"),
-            "peers": __check_be_service("peers"),
             "sink": __check_be_service("sink"),
             "sourcemapreader": __check_be_service("sourcemapreader"),
-            "storage": __check_be_service("storage")
+            "storage": __check_be_service("storage"),
         },
         "details": __get_sessions_stats,
-        "ssl": __check_SSL
+        "ssl": __check_SSL,
     }
     return __process_health(health_map=health_map)
 
@@ -210,10 +190,16 @@ def __process_health(health_map):
             response.pop(parent_key)
         elif isinstance(health_map[parent_key], dict):
             for element_key in health_map[parent_key]:
-                if config(f"SKIP_H_{parent_key.upper()}_{element_key.upper()}", cast=bool, default=False):
+                if config(
+                    f"SKIP_H_{parent_key.upper()}_{element_key.upper()}",
+                    cast=bool,
+                    default=False,
+                ):
                     response[parent_key].pop(element_key)
                 else:
-                    response[parent_key][element_key] = health_map[parent_key][element_key]()
+                    response[parent_key][element_key] = health_map[parent_key][
+                        element_key
+                    ]()
         else:
             response[parent_key] = health_map[parent_key]()
     return response
@@ -221,7 +207,8 @@ def __process_health(health_map):
 
 def cron():
     with pg_client.PostgresClient() as cur:
-        query = cur.mogrify("""SELECT projects.project_id,
+        query = cur.mogrify(
+            """SELECT projects.project_id,
                                       projects.created_at,
                                       projects.sessions_last_check_at,
                                       projects.first_recorded_session_at,
@@ -229,7 +216,8 @@ def cron():
                                 FROM public.projects
                                      LEFT JOIN public.projects_stats USING (project_id)
                                 WHERE projects.deleted_at IS NULL
-                                ORDER BY project_id;""")
+                                ORDER BY project_id;"""
+        )
         cur.execute(query)
         rows = cur.fetchall()
         for r in rows:
@@ -250,20 +238,24 @@ def cron():
                 count_start_from = r["last_update_at"]
 
             count_start_from = TimeUTC.datetime_to_timestamp(count_start_from)
-            params = {"project_id": r["project_id"],
-                      "start_ts": count_start_from,
-                      "end_ts": TimeUTC.now(),
-                      "sessions_count": 0,
-                      "events_count": 0}
+            params = {
+                "project_id": r["project_id"],
+                "start_ts": count_start_from,
+                "end_ts": TimeUTC.now(),
+                "sessions_count": 0,
+                "events_count": 0,
+            }
 
-            query = cur.mogrify("""SELECT COUNT(1) AS sessions_count,
+            query = cur.mogrify(
+                """SELECT COUNT(1) AS sessions_count,
                                           COALESCE(SUM(events_count),0) AS events_count
                                    FROM public.sessions
                                    WHERE project_id=%(project_id)s
                                       AND start_ts>=%(start_ts)s
                                       AND start_ts<=%(end_ts)s
                                       AND duration IS NOT NULL;""",
-                                params)
+                params,
+            )
             cur.execute(query)
             row = cur.fetchone()
             if row is not None:
@@ -271,56 +263,68 @@ def cron():
                 params["events_count"] = row["events_count"]
 
             if insert:
-                query = cur.mogrify("""INSERT INTO public.projects_stats(project_id, sessions_count, events_count, last_update_at)
+                query = cur.mogrify(
+                    """INSERT INTO public.projects_stats(project_id, sessions_count, events_count, last_update_at)
                                        VALUES (%(project_id)s, %(sessions_count)s, %(events_count)s, (now() AT TIME ZONE 'utc'::text));""",
-                                    params)
+                    params,
+                )
             else:
-                query = cur.mogrify("""UPDATE public.projects_stats
+                query = cur.mogrify(
+                    """UPDATE public.projects_stats
                                        SET sessions_count=sessions_count+%(sessions_count)s,
                                            events_count=events_count+%(events_count)s,
                                            last_update_at=(now() AT TIME ZONE 'utc'::text)
                                        WHERE project_id=%(project_id)s;""",
-                                    params)
+                    params,
+                )
             cur.execute(query)
 
 
 # this cron is used to correct the sessions&events count every week
 def weekly_cron():
     with pg_client.PostgresClient(long_query=True) as cur:
-        query = cur.mogrify("""SELECT project_id,
+        query = cur.mogrify(
+            """SELECT project_id,
                                       projects_stats.last_update_at
                                FROM public.projects
                                     LEFT JOIN public.projects_stats USING (project_id)
                                WHERE projects.deleted_at IS NULL
-                               ORDER BY project_id;""")
+                               ORDER BY project_id;"""
+        )
         cur.execute(query)
         rows = cur.fetchall()
         for r in rows:
             if r["last_update_at"] is None:
                 continue
 
-            params = {"project_id": r["project_id"],
-                      "end_ts": TimeUTC.now(),
-                      "sessions_count": 0,
-                      "events_count": 0}
+            params = {
+                "project_id": r["project_id"],
+                "end_ts": TimeUTC.now(),
+                "sessions_count": 0,
+                "events_count": 0,
+            }
 
-            query = cur.mogrify("""SELECT COUNT(1) AS sessions_count,
+            query = cur.mogrify(
+                """SELECT COUNT(1) AS sessions_count,
                                           COALESCE(SUM(events_count),0) AS events_count
                                    FROM public.sessions
                                    WHERE project_id=%(project_id)s
                                       AND start_ts<=%(end_ts)s
                                       AND duration IS NOT NULL;""",
-                                params)
+                params,
+            )
             cur.execute(query)
             row = cur.fetchone()
             if row is not None:
                 params["sessions_count"] = row["sessions_count"]
                 params["events_count"] = row["events_count"]
 
-            query = cur.mogrify("""UPDATE public.projects_stats
+            query = cur.mogrify(
+                """UPDATE public.projects_stats
                                    SET sessions_count=%(sessions_count)s,
                                        events_count=%(events_count)s,
                                        last_update_at=(now() AT TIME ZONE 'utc'::text)
                                    WHERE project_id=%(project_id)s;""",
-                                params)
+                params,
+            )
             cur.execute(query)

api/chalicelib/core/issue_tracking/jira_cloud.py

@@ -50,8 +50,8 @@ class JIRAIntegration(base.BaseIntegration):
             cur.execute(
                 cur.mogrify(
                     """SELECT username, token, url
-                        FROM public.jira_cloud 
-                        WHERE user_id=%(user_id)s;""",
+                       FROM public.jira_cloud
+                       WHERE user_id = %(user_id)s;""",
                     {"user_id": self._user_id})
             )
             data = helper.dict_to_camel_case(cur.fetchone())
@@ -95,10 +95,9 @@ class JIRAIntegration(base.BaseIntegration):
     def add(self, username, token, url, obfuscate=False):
         with pg_client.PostgresClient() as cur:
             cur.execute(
-                cur.mogrify("""\
-                        INSERT INTO public.jira_cloud(username, token, user_id,url)
-                        VALUES (%(username)s, %(token)s, %(user_id)s,%(url)s)
-                        RETURNING  username, token, url;""",
+                cur.mogrify(""" \
+                            INSERT INTO public.jira_cloud(username, token, user_id, url)
+                            VALUES (%(username)s, %(token)s, %(user_id)s, %(url)s) RETURNING  username, token, url;""",
                             {"user_id": self._user_id, "username": username,
                              "token": token, "url": url})
             )
@@ -112,9 +111,10 @@ class JIRAIntegration(base.BaseIntegration):
     def delete(self):
         with pg_client.PostgresClient() as cur:
             cur.execute(
-                cur.mogrify("""\
-                        DELETE FROM public.jira_cloud
-                        WHERE user_id=%(user_id)s;""",
+                cur.mogrify(""" \
+                            DELETE
+                            FROM public.jira_cloud
+                            WHERE user_id = %(user_id)s;""",
                             {"user_id": self._user_id})
             )
             return {"state": "success"}
@@ -125,7 +125,7 @@ class JIRAIntegration(base.BaseIntegration):
                 changes={
                     "username": data.username,
                     "token": data.token if len(data.token) > 0 and data.token.find("***") == -1 \
-                        else self.integration.token,
+                        else self.integration["token"],
                     "url": str(data.url)
                 },
                 obfuscate=True

api/chalicelib/core/metrics/custom_metrics.py

@@ -352,6 +352,100 @@ def update_card(metric_id, user_id, project_id, data: schemas.CardSchema):
     return get_card(metric_id=metric_id, project_id=project_id, user_id=user_id)
 
 
+def search_metrics(project_id, user_id, data: schemas.MetricSearchSchema, include_series=False):
+    constraints = ["metrics.project_id = %(project_id)s", "metrics.deleted_at ISNULL"]
+    params = {
+        "project_id": project_id,
+        "user_id": user_id,
+        "offset": (data.page - 1) * data.limit,
+        "limit": data.limit,
+    }
+    if data.mine_only:
+        constraints.append("user_id = %(user_id)s")
+    else:
+        constraints.append("(user_id = %(user_id)s OR metrics.is_public)")
+    if data.shared_only:
+        constraints.append("is_public")
+
+    if data.filter is not None:
+        if data.filter.type:
+            constraints.append("metrics.metric_type = %(filter_type)s")
+            params["filter_type"] = data.filter.type
+        if data.filter.query and len(data.filter.query) > 0:
+            constraints.append("(metrics.name ILIKE %(filter_query)s OR owner.owner_name ILIKE %(filter_query)s)")
+            params["filter_query"] = helper.values_for_operator(
+                value=data.filter.query, op=schemas.SearchEventOperator.CONTAINS
+            )
+
+    with pg_client.PostgresClient() as cur:
+        sub_join = ""
+        if include_series:
+            sub_join = """LEFT JOIN LATERAL (
+                            SELECT COALESCE(jsonb_agg(metric_series.* ORDER BY index),'[]'::jsonb) AS series
+                            FROM metric_series
+                            WHERE metric_series.metric_id = metrics.metric_id
+                              AND metric_series.deleted_at ISNULL 
+                         ) AS metric_series ON (TRUE)"""
+
+        sort_column = data.sort.field if data.sort.field is not None and len(data.sort.field) > 0 \
+            else "created_at"
+        # change ascend to asc and descend to desc
+        sort_order = data.sort.order.value if hasattr(data.sort.order, "value") else data.sort.order
+        if sort_order == "ascend":
+            sort_order = "asc"
+        elif sort_order == "descend":
+            sort_order = "desc"
+
+        query = cur.mogrify(
+            f"""SELECT count(1) OVER () AS total,metric_id, project_id, user_id, name, is_public, created_at, edited_at,
+                        metric_type, metric_of, metric_format, metric_value, view_type, is_pinned, 
+                        dashboards, owner_email, owner_name, default_config AS config, thumbnail
+                FROM metrics
+                {sub_join}
+                LEFT JOIN LATERAL (
+                    SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public, name),'[]'::jsonb) AS dashboards
+                    FROM (
+                        SELECT DISTINCT dashboard_id, name, is_public
+                        FROM dashboards
+                        INNER JOIN dashboard_widgets USING (dashboard_id)
+                        WHERE deleted_at ISNULL
+                          AND dashboard_widgets.metric_id = metrics.metric_id
+                          AND project_id = %(project_id)s
+                          AND ((dashboards.user_id = %(user_id)s OR is_public))
+                    ) AS connected_dashboards
+                ) AS connected_dashboards ON (TRUE)
+                LEFT JOIN LATERAL (
+                    SELECT email AS owner_email, name AS owner_name
+                    FROM users
+                    WHERE deleted_at ISNULL
+                      AND users.user_id = metrics.user_id
+                ) AS owner ON (TRUE)
+                WHERE {" AND ".join(constraints)}
+                ORDER BY {sort_column} {sort_order}
+                LIMIT %(limit)s OFFSET %(offset)s;""",
+            params
+        )
+        cur.execute(query)
+        rows = cur.fetchall()
+        if len(rows) > 0:
+            total = rows[0]["total"]
+            if include_series:
+                for r in rows:
+                    r.pop("total")
+                    for s in r.get("series", []):
+                        s["filter"] = helper.old_search_payload_to_flat(s["filter"])
+            else:
+                for r in rows:
+                    r.pop("total")
+                    r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"])
+                    r["edited_at"] = TimeUTC.datetime_to_timestamp(r["edited_at"])
+            rows = helper.list_to_camel_case(rows)
+        else:
+            total = 0
+
+    return {"total": total, "list": rows}
+
+
 def search_all(project_id, user_id, data: schemas.SearchCardsSchema, include_series=False):
     constraints = ["metrics.project_id = %(project_id)s",
                    "metrics.deleted_at ISNULL"]

api/chalicelib/core/metrics/modules/__init__.py

@@ -5,8 +5,8 @@ from decouple import config
 logger = logging.getLogger(__name__)
 
 if config("EXP_METRICS", cast=bool, default=False):
-    from chalicelib.core.sessions import sessions_ch as sessions
+    import chalicelib.core.sessions.sessions_ch as sessions
 else:
-    from chalicelib.core.sessions import sessions
+    import chalicelib.core.sessions.sessions_pg as sessions
 
 from chalicelib.core.sessions import sessions_mobs

api/chalicelib/core/metrics/product_analytics/product_analytics_ch.py

@@ -85,6 +85,9 @@ def __complete_missing_steps(start_time, end_time, density, neutral, rows, time_
 # compute avg_time_from_previous at the same level as sessions_count (this was removed in v1.22)
 # if start-point is selected, the selected event is ranked n°1
 def path_analysis(project_id: int, data: schemas.CardPathAnalysis):
+    if not data.hide_excess:
+        data.hide_excess = True
+        data.rows = 50
     sub_events = []
     start_points_conditions = []
     step_0_conditions = []

api/chalicelib/core/sessions/__init__.py

@@ -3,9 +3,11 @@ import logging
 from decouple import config
 
 logger = logging.getLogger(__name__)
-from . import sessions as sessions_legacy
+from . import sessions_pg
+from . import sessions_pg as sessions_legacy
+from . import sessions_ch
 
 if config("EXP_METRICS", cast=bool, default=False):
     from . import sessions_ch as sessions
 else:
-    from . import sessions
+    from . import sessions_pg as sessions

api/chalicelib/core/sessions/sessions_ch.py

@@ -3,7 +3,7 @@ from typing import List, Union
 
 import schemas
 from chalicelib.core import events, metadata
-from . import performance_event, sessions as sessions_legacy
+from . import performance_event, sessions_legacy
 from chalicelib.utils import pg_client, helper, metrics_helper, ch_client, exp_ch_helper
 from chalicelib.utils import sql_helper as sh
 
@@ -153,7 +153,7 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de
                         "isEvent": True,
                         "value": [],
                         "operator": e.operator,
-                        "filters": []
+                        "filters": e.filters
                     })
                 for v in e.value:
                     if v not in extra_conditions[e.operator].value:
@@ -178,7 +178,7 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de
                         "isEvent": True,
                         "value": [],
                         "operator": e.operator,
-                        "filters": []
+                        "filters": e.filters
                     })
                 for v in e.value:
                     if v not in extra_conditions[e.operator].value:
@@ -870,12 +870,12 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu
                 events_conditions[-1]["condition"] = []
                 if not is_any and event.value not in [None, "*", ""]:
                     event_where.append(
-                        sh.multi_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)",
+                        sh.multi_conditions(f"(toString(main1.`$properties`.message) {op} %({e_k})s OR toString(main1.`$properties`.name) {op} %({e_k})s)",
                                             event.value, value_key=e_k))
                     events_conditions[-1]["condition"].append(event_where[-1])
                     events_extra_join += f" AND {event_where[-1]}"
                 if len(event.source) > 0 and event.source[0] not in [None, "*", ""]:
-                    event_where.append(sh.multi_conditions(f"main1.source = %({s_k})s", event.source, value_key=s_k))
+                    event_where.append(sh.multi_conditions(f"toString(main1.`$properties`.source) = %({s_k})s", event.source, value_key=s_k))
                     events_conditions[-1]["condition"].append(event_where[-1])
                     events_extra_join += f" AND {event_where[-1]}"
 
@@ -1108,8 +1108,12 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu
                     is_any = sh.isAny_opreator(f.operator)
                     if is_any or len(f.value) == 0:
                         continue
+                    is_negative_operator = sh.is_negation_operator(f.operator)
                     f.value = helper.values_for_operator(value=f.value, op=f.operator)
                     op = sh.get_sql_operator(f.operator)
+                    r_op = ""
+                    if is_negative_operator:
+                        r_op = sh.reverse_sql_operator(op)
                     e_k_f = e_k + f"_fetch{j}"
                     full_args = {**full_args, **sh.multi_values(f.value, value_key=e_k_f)}
                     if f.type == schemas.FetchFilterType.FETCH_URL:
@@ -1118,6 +1122,12 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu
                         ))
                         events_conditions[-1]["condition"].append(event_where[-1])
                         apply = True
+                        if is_negative_operator:
+                            events_conditions_not.append(
+                                {
+                                    "type": f"sub.`$event_name`='{exp_ch_helper.get_event_type(event_type, platform=platform)}'"})
+                            events_conditions_not[-1]["condition"] = sh.multi_conditions(
+                                f"sub.`$properties`.url_path {r_op} %({e_k_f})s", f.value, value_key=e_k_f)
                     elif f.type == schemas.FetchFilterType.FETCH_STATUS_CODE:
                         event_where.append(json_condition(
                             "main", "$properties", 'status', op, f.value, e_k_f, True, True
@@ -1130,6 +1140,13 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu
                         ))
                         events_conditions[-1]["condition"].append(event_where[-1])
                         apply = True
+                        if is_negative_operator:
+                            events_conditions_not.append(
+                                {
+                                    "type": f"sub.`$event_name`='{exp_ch_helper.get_event_type(event_type, platform=platform)}'"})
+                            events_conditions_not[-1]["condition"] = sh.multi_conditions(
+                                f"sub.`$properties`.method {r_op} %({e_k_f})s", f.value,
+                                value_key=e_k_f)
                     elif f.type == schemas.FetchFilterType.FETCH_DURATION:
                         event_where.append(
                             sh.multi_conditions(f"main.`$duration_s` {f.operator} %({e_k_f})s/1000", f.value,
@@ -1142,12 +1159,26 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu
                         ))
                         events_conditions[-1]["condition"].append(event_where[-1])
                         apply = True
+                        if is_negative_operator:
+                            events_conditions_not.append(
+                                {
+                                    "type": f"sub.`$event_name`='{exp_ch_helper.get_event_type(event_type, platform=platform)}'"})
+                            events_conditions_not[-1]["condition"] = sh.multi_conditions(
+                                f"sub.`$properties`.request_body {r_op} %({e_k_f})s", f.value,
+                                value_key=e_k_f)
                     elif f.type == schemas.FetchFilterType.FETCH_RESPONSE_BODY:
                         event_where.append(json_condition(
                             "main", "$properties", 'response_body', op, f.value, e_k_f
                         ))
                         events_conditions[-1]["condition"].append(event_where[-1])
                         apply = True
+                        if is_negative_operator:
+                            events_conditions_not.append(
+                                {
+                                    "type": f"sub.`$event_name`='{exp_ch_helper.get_event_type(event_type, platform=platform)}'"})
+                            events_conditions_not[-1]["condition"] = sh.multi_conditions(
+                                f"sub.`$properties`.response_body {r_op} %({e_k_f})s", f.value,
+                                value_key=e_k_f)
                     else:
                         logging.warning(f"undefined FETCH filter: {f.type}")
                 if not apply:
@@ -1395,17 +1426,30 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu
         if extra_conditions and len(extra_conditions) > 0:
             _extra_or_condition = []
             for i, c in enumerate(extra_conditions):
-                if sh.isAny_opreator(c.operator):
+                if sh.isAny_opreator(c.operator) and c.type != schemas.EventType.REQUEST_DETAILS.value:
                     continue
                 e_k = f"ec_value{i}"
                 op = sh.get_sql_operator(c.operator)
                 c.value = helper.values_for_operator(value=c.value, op=c.operator)
                 full_args = {**full_args,
                              **sh.multi_values(c.value, value_key=e_k)}
-                if c.type == events.EventType.LOCATION.ui_type:
+                if c.type in (schemas.EventType.LOCATION.value, schemas.EventType.REQUEST.value):
                     _extra_or_condition.append(
                         sh.multi_conditions(f"extra_event.url_path {op} %({e_k})s",
                                             c.value, value_key=e_k))
+                elif c.type == schemas.EventType.REQUEST_DETAILS.value:
+                    for j, c_f in enumerate(c.filters):
+                        if sh.isAny_opreator(c_f.operator) or len(c_f.value) == 0:
+                            continue
+                        e_k += f"_{j}"
+                        op = sh.get_sql_operator(c_f.operator)
+                        c_f.value = helper.values_for_operator(value=c_f.value, op=c_f.operator)
+                        full_args = {**full_args,
+                                     **sh.multi_values(c_f.value, value_key=e_k)}
+                        if c_f.type == schemas.FetchFilterType.FETCH_URL.value:
+                            _extra_or_condition.append(
+                                sh.multi_conditions(f"extra_event.url_path {op} %({e_k})s",
+                                                     c_f.value, value_key=e_k))
                 else:
                     logging.warning(f"unsupported extra_event type:${c.type}")
             if len(_extra_or_condition) > 0:
@@ -1416,9 +1460,10 @@ def search_query_parts_ch(data: schemas.SessionsSearchPayloadSchema, error_statu
         query_part = f"""{f"({events_query_part}) AS f" if len(events_query_part) > 0 else ""}"""
     else:
         if len(events_query_part) > 0:
-            extra_join += f"""INNER JOIN (SELECT * 
+            extra_join += f"""INNER JOIN (SELECT DISTINCT ON (session_id) * 
                                     FROM {MAIN_SESSIONS_TABLE} AS s {extra_event}
-                                    WHERE {" AND ".join(extra_constraints)}) AS s ON(s.session_id=f.session_id)"""
+                                    WHERE {" AND ".join(extra_constraints)}
+                                    ORDER BY _timestamp DESC) AS s ON(s.session_id=f.session_id)"""
         else:
             deduplication_keys = ["session_id"] + extra_deduplication
             extra_join = f"""(SELECT * 

api/chalicelib/core/sessions/sessions_metas.py

@@ -1,76 +1,81 @@
+from functools import cache
+
 import schemas
 from chalicelib.core.autocomplete import autocomplete
 from chalicelib.utils.event_filter_definition import SupportedFilter
 
-SUPPORTED_TYPES = {
-    schemas.FilterType.USER_OS: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_OS),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_OS)),
-    schemas.FilterType.USER_BROWSER: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_BROWSER),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_BROWSER)),
-    schemas.FilterType.USER_DEVICE: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_DEVICE),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_DEVICE)),
-    schemas.FilterType.USER_COUNTRY: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_COUNTRY),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_COUNTRY)),
-    schemas.FilterType.USER_CITY: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_CITY),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_CITY)),
-    schemas.FilterType.USER_STATE: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_STATE),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_STATE)),
-    schemas.FilterType.USER_ID: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_ID),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_ID)),
-    schemas.FilterType.USER_ANONYMOUS_ID: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_ANONYMOUS_ID),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_ANONYMOUS_ID)),
-    schemas.FilterType.REV_ID: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.REV_ID),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.REV_ID)),
-    schemas.FilterType.REFERRER: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.REFERRER),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.REFERRER)),
-    schemas.FilterType.UTM_CAMPAIGN: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.UTM_CAMPAIGN),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.UTM_CAMPAIGN)),
-    schemas.FilterType.UTM_MEDIUM: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.UTM_MEDIUM),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.UTM_MEDIUM)),
-    schemas.FilterType.UTM_SOURCE: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.UTM_SOURCE),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.UTM_SOURCE)),
-    # Mobile
-    schemas.FilterType.USER_OS_MOBILE: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_OS_MOBILE),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_OS_MOBILE)),
-    schemas.FilterType.USER_DEVICE_MOBILE: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(
-            typename=schemas.FilterType.USER_DEVICE_MOBILE),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_DEVICE_MOBILE)),
-    schemas.FilterType.USER_COUNTRY_MOBILE: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_COUNTRY_MOBILE),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_COUNTRY_MOBILE)),
-    schemas.FilterType.USER_ID_MOBILE: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_ID_MOBILE),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_ID_MOBILE)),
-    schemas.FilterType.USER_ANONYMOUS_ID_MOBILE: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_ANONYMOUS_ID_MOBILE),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.USER_ANONYMOUS_ID_MOBILE)),
-    schemas.FilterType.REV_ID_MOBILE: SupportedFilter(
-        get=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.REV_ID_MOBILE),
-        query=autocomplete.__generic_autocomplete_metas(typename=schemas.FilterType.REV_ID_MOBILE)),
 
-}
+@cache
+def supported_types():
+    return {
+        schemas.FilterType.USER_OS: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_OS),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_OS)),
+        schemas.FilterType.USER_BROWSER: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_BROWSER),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_BROWSER)),
+        schemas.FilterType.USER_DEVICE: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_DEVICE),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_DEVICE)),
+        schemas.FilterType.USER_COUNTRY: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_COUNTRY),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_COUNTRY)),
+        schemas.FilterType.USER_CITY: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_CITY),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_CITY)),
+        schemas.FilterType.USER_STATE: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_STATE),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_STATE)),
+        schemas.FilterType.USER_ID: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_ID),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_ID)),
+        schemas.FilterType.USER_ANONYMOUS_ID: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_ANONYMOUS_ID),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_ANONYMOUS_ID)),
+        schemas.FilterType.REV_ID: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.REV_ID),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.REV_ID)),
+        schemas.FilterType.REFERRER: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.REFERRER),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.REFERRER)),
+        schemas.FilterType.UTM_CAMPAIGN: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.UTM_CAMPAIGN),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.UTM_CAMPAIGN)),
+        schemas.FilterType.UTM_MEDIUM: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.UTM_MEDIUM),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.UTM_MEDIUM)),
+        schemas.FilterType.UTM_SOURCE: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.UTM_SOURCE),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.UTM_SOURCE)),
+        # Mobile
+        schemas.FilterType.USER_OS_MOBILE: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_OS_MOBILE),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_OS_MOBILE)),
+        schemas.FilterType.USER_DEVICE_MOBILE: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(
+                typename=schemas.FilterType.USER_DEVICE_MOBILE),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_DEVICE_MOBILE)),
+        schemas.FilterType.USER_COUNTRY_MOBILE: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_COUNTRY_MOBILE),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_COUNTRY_MOBILE)),
+        schemas.FilterType.USER_ID_MOBILE: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_ID_MOBILE),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_ID_MOBILE)),
+        schemas.FilterType.USER_ANONYMOUS_ID_MOBILE: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_ANONYMOUS_ID_MOBILE),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.USER_ANONYMOUS_ID_MOBILE)),
+        schemas.FilterType.REV_ID_MOBILE: SupportedFilter(
+            get=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.REV_ID_MOBILE),
+            query=autocomplete.generic_autocomplete_metas(typename=schemas.FilterType.REV_ID_MOBILE)),
+
+    }
 
 
 def search(text: str, meta_type: schemas.FilterType, project_id: int):
     rows = []
-    if meta_type not in list(SUPPORTED_TYPES.keys()):
+    if meta_type not in list(supported_types().keys()):
         return {"errors": ["unsupported type"]}
-    rows += SUPPORTED_TYPES[meta_type].get(project_id=project_id, text=text)
+    rows += supported_types()[meta_type].get(project_id=project_id, text=text)
     # for IOS events autocomplete
     # if meta_type + "_IOS" in list(SUPPORTED_TYPES.keys()):
     #     rows += SUPPORTED_TYPES[meta_type + "_IOS"].get(project_id=project_id, text=text)

api/chalicelib/core/sessions/sessions_pg.py

@@ -148,7 +148,7 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de
                         "isEvent": True,
                         "value": [],
                         "operator": e.operator,
-                        "filters": []
+                        "filters": e.filters
                     })
                 for v in e.value:
                     if v not in extra_conditions[e.operator].value:
@@ -165,7 +165,7 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de
                         "isEvent": True,
                         "value": [],
                         "operator": e.operator,
-                        "filters": []
+                        "filters": e.filters
                     })
                 for v in e.value:
                     if v not in extra_conditions[e.operator].value:
@@ -989,7 +989,7 @@ def search_query_parts(data: schemas.SessionsSearchPayloadSchema, error_status,
                         sh.multi_conditions(f"ev.{events.EventType.LOCATION.column} {op} %({e_k})s",
                                             c.value, value_key=e_k))
                 else:
-                    logger.warning(f"unsupported extra_event type:${c.type}")
+                    logger.warning(f"unsupported extra_event type: {c.type}")
             if len(_extra_or_condition) > 0:
                 extra_constraints.append("(" + " OR ".join(_extra_or_condition) + ")")
     query_part = f"""\

api/chalicelib/core/sessions/sessions_replay.py

@@ -2,7 +2,7 @@ import schemas
 from chalicelib.core import events, metadata, events_mobile, \
     issues, assist, canvas, user_testing
 from . import sessions_mobs, sessions_devtool
-from chalicelib.utils import errors_helper
+from chalicelib.core.errors.modules import errors_helper
 from chalicelib.utils import pg_client, helper
 from chalicelib.core.modules import MOB_KEY, get_file_key
 

api/chalicelib/core/sessions/sessions_search.py

@@ -2,7 +2,7 @@ import logging
 
 import schemas
 from chalicelib.core import metadata, projects
-from chalicelib.core.sessions import sessions_favorite, sessions_legacy
+from . import sessions_favorite, sessions_legacy
 from chalicelib.utils import pg_client, helper
 
 logger = logging.getLogger(__name__)
@@ -43,7 +43,13 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project: schemas.
                     count_only=False, issue=None, ids_only=False, platform="web"):
     if data.bookmarked:
         data.startTimestamp, data.endTimestamp = sessions_favorite.get_start_end_timestamp(project.project_id, user_id)
-
+    if data.startTimestamp is None:
+        logger.debug(f"No vault sessions found for project:{project.project_id}")
+        return {
+            'total': 0,
+            'sessions': [],
+            'src': 1
+        }
     full_args, query_part = sessions_legacy.search_query_parts(data=data, error_status=error_status,
                                                                errors_only=errors_only,
                                                                favorite_only=data.bookmarked, issue=issue,
@@ -116,7 +122,10 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project: schemas.
             sort = 'session_id'
             if data.sort is not None and data.sort != "session_id":
                 # sort += " " + data.order + "," + helper.key_to_snake_case(data.sort)
-                sort = helper.key_to_snake_case(data.sort)
+                if data.sort == 'datetime':
+                    sort = 'start_ts'
+                else:
+                    sort = helper.key_to_snake_case(data.sort)
 
             meta_keys = metadata.get(project_id=project.project_id)
             main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, 
@@ -166,7 +175,8 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project: schemas.
     #                       reverse=data.order.upper() == "DESC")
     return {
         'total': total,
-        'sessions': helper.list_to_camel_case(sessions)
+        'sessions': helper.list_to_camel_case(sessions),
+        'src': 1
     }
 
 

api/chalicelib/core/sessions/unprocessed_sessions.py

@@ -1,7 +1,7 @@
 import logging
 
 from chalicelib.core import assist
-from chalicelib.core.sessions import sessions
+from . import sessions
 
 logger = logging.getLogger(__name__)
 

api/chalicelib/core/spot.py

@@ -18,7 +18,7 @@ def refresh_spot_jwt_iat_jti(user_id):
                             {"user_id": user_id})
         cur.execute(query)
         row = cur.fetchone()
-        return row.get("spot_jwt_iat"), row.get("spot_jwt_refresh_jti"), row.get("spot_jwt_refresh_iat")
+        return users.RefreshSpotJWTs(**row)
 
 
 def logout(user_id: int):
@@ -26,13 +26,13 @@ def logout(user_id: int):
 
 
 def refresh(user_id: int, tenant_id: int = -1) -> dict:
-    spot_jwt_iat, spot_jwt_r_jti, spot_jwt_r_iat = refresh_spot_jwt_iat_jti(user_id=user_id)
+    j = refresh_spot_jwt_iat_jti(user_id=user_id)
     return {
-        "jwt": authorizers.generate_jwt(user_id=user_id, tenant_id=tenant_id, iat=spot_jwt_iat,
+        "jwt": authorizers.generate_jwt(user_id=user_id, tenant_id=tenant_id, iat=j.spot_jwt_iat,
                                         aud=AUDIENCE, for_spot=True),
-        "refreshToken": authorizers.generate_jwt_refresh(user_id=user_id, tenant_id=tenant_id, iat=spot_jwt_r_iat,
-                                                         aud=AUDIENCE, jwt_jti=spot_jwt_r_jti, for_spot=True),
-        "refreshTokenMaxAge": config("JWT_SPOT_REFRESH_EXPIRATION", cast=int) - (spot_jwt_iat - spot_jwt_r_iat)
+        "refreshToken": authorizers.generate_jwt_refresh(user_id=user_id, tenant_id=tenant_id, iat=j.spot_jwt_refresh_iat,
+                                                         aud=AUDIENCE, jwt_jti=j.spot_jwt_refresh_jti, for_spot=True),
+        "refreshTokenMaxAge": config("JWT_SPOT_REFRESH_EXPIRATION", cast=int) - (j.spot_jwt_iat - j.spot_jwt_refresh_iat)
     }
 
 

api/chalicelib/core/users.py

@@ -1,9 +1,10 @@
 import json
 import secrets
+from typing import Optional
 
 from decouple import config
 from fastapi import BackgroundTasks
-from pydantic import BaseModel
+from pydantic import BaseModel, model_validator
 
 import schemas
 from chalicelib.core import authorizers
@@ -83,7 +84,6 @@ def restore_member(user_id, email, invitation_token, admin, name, owner=False):
                              "name": name, "invitation_token": invitation_token})
         cur.execute(query)
         result = cur.fetchone()
-        cur.execute(query)
         result["created_at"] = TimeUTC.datetime_to_timestamp(result["created_at"])
     return helper.dict_to_camel_case(result)
 
@@ -284,7 +284,7 @@ def edit_member(user_id_to_update, tenant_id, changes: schemas.EditMemberSchema,
     if editor_id != user_id_to_update:
         admin = get_user_role(tenant_id=tenant_id, user_id=editor_id)
         if not admin["superAdmin"] and not admin["admin"]:
-            return {"errors": ["unauthorized"]}
+            return {"errors": ["unauthorized, you must have admin privileges"]}
         if admin["admin"] and user["superAdmin"]:
             return {"errors": ["only the owner can edit his own details"]}
     else:
@@ -552,14 +552,35 @@ def refresh_auth_exists(user_id, jwt_jti=None):
     return r is not None
 
 
-class ChangeJwt(BaseModel):
+class FullLoginJWTs(BaseModel):
     jwt_iat: int
-    jwt_refresh_jti: int
+    jwt_refresh_jti: str
     jwt_refresh_iat: int
     spot_jwt_iat: int
-    spot_jwt_refresh_jti: int
+    spot_jwt_refresh_jti: str
     spot_jwt_refresh_iat: int
 
+    @model_validator(mode="before")
+    @classmethod
+    def _transform_data(cls, values):
+        if values.get("jwt_refresh_jti") is not None:
+            values["jwt_refresh_jti"] = str(values["jwt_refresh_jti"])
+        if values.get("spot_jwt_refresh_jti") is not None:
+            values["spot_jwt_refresh_jti"] = str(values["spot_jwt_refresh_jti"])
+        return values
+
+
+class RefreshLoginJWTs(FullLoginJWTs):
+    spot_jwt_iat: Optional[int] = None
+    spot_jwt_refresh_jti: Optional[str] = None
+    spot_jwt_refresh_iat: Optional[int] = None
+
+
+class RefreshSpotJWTs(FullLoginJWTs):
+    jwt_iat: Optional[int] = None
+    jwt_refresh_jti: Optional[str] = None
+    jwt_refresh_iat: Optional[int] = None
+
 
 def change_jwt_iat_jti(user_id):
     with pg_client.PostgresClient() as cur:
@@ -580,7 +601,7 @@ def change_jwt_iat_jti(user_id):
                             {"user_id": user_id})
         cur.execute(query)
         row = cur.fetchone()
-        return ChangeJwt(**row)
+        return FullLoginJWTs(**row)
 
 
 def refresh_jwt_iat_jti(user_id):
@@ -595,7 +616,7 @@ def refresh_jwt_iat_jti(user_id):
                             {"user_id": user_id})
         cur.execute(query)
         row = cur.fetchone()
-        return row.get("jwt_iat"), row.get("jwt_refresh_jti"), row.get("jwt_refresh_iat")
+        return RefreshLoginJWTs(**row)
 
 
 def authenticate(email, password, for_change_password=False) -> dict | bool | None:
@@ -663,13 +684,13 @@ def logout(user_id: int):
 
 
 def refresh(user_id: int, tenant_id: int = -1) -> dict:
-    jwt_iat, jwt_r_jti, jwt_r_iat = refresh_jwt_iat_jti(user_id=user_id)
+    j = refresh_jwt_iat_jti(user_id=user_id)
     return {
-        "jwt": authorizers.generate_jwt(user_id=user_id, tenant_id=tenant_id, iat=jwt_iat,
+        "jwt": authorizers.generate_jwt(user_id=user_id, tenant_id=tenant_id, iat=j.jwt_iat,
                                         aud=AUDIENCE),
-        "refreshToken": authorizers.generate_jwt_refresh(user_id=user_id, tenant_id=tenant_id, iat=jwt_r_iat,
-                                                         aud=AUDIENCE, jwt_jti=jwt_r_jti),
-        "refreshTokenMaxAge": config("JWT_REFRESH_EXPIRATION", cast=int) - (jwt_iat - jwt_r_iat)
+        "refreshToken": authorizers.generate_jwt_refresh(user_id=user_id, tenant_id=tenant_id, iat=j.jwt_refresh_iat,
+                                                         aud=AUDIENCE, jwt_jti=j.jwt_refresh_jti),
+        "refreshTokenMaxAge": config("JWT_REFRESH_EXPIRATION", cast=int) - (j.jwt_iat - j.jwt_refresh_iat),
     }
 
 

api/chalicelib/utils/ch_client.py

@@ -1,286 +1,73 @@
 import logging
-import threading
-import time
-import asyncio
-from functools import wraps
-from queue import Queue, Empty
-from typing import Dict, Any, Optional
 
-import clickhouse_connect
-from clickhouse_connect.driver.query import QueryContext
+import clickhouse_driver
 from decouple import config
 
 logger = logging.getLogger(__name__)
 
-_CH_CONFIG = {"host": config("ch_host"),
-              "user": config("ch_user", default="default"),
-              "password": config("ch_password", default=""),
-              "port": config("ch_port_http", cast=int),
-              "client_name": config("APP_NAME", default="PY")}
-CH_CONFIG = dict(_CH_CONFIG)
-
 settings = {}
 if config('ch_timeout', cast=int, default=-1) > 0:
-    logging.info(f"CH-max_execution_time set to {config('ch_timeout')}s")
+    logger.info(f"CH-max_execution_time set to {config('ch_timeout')}s")
     settings = {**settings, "max_execution_time": config('ch_timeout', cast=int)}
 
 if config('ch_receive_timeout', cast=int, default=-1) > 0:
-    logging.info(f"CH-receive_timeout set to {config('ch_receive_timeout')}s")
+    logger.info(f"CH-receive_timeout set to {config('ch_receive_timeout')}s")
     settings = {**settings, "receive_timeout": config('ch_receive_timeout', cast=int)}
 
-extra_args = {}
-if config("CH_COMPRESSION", cast=bool, default=True):
-    extra_args["compression"] = "lz4"
-
-
-def transform_result(self, original_function):
-    @wraps(original_function)
-    def wrapper(*args, **kwargs):
-        logger.debug(str.encode(self.format(query=kwargs.get("query", ""), parameters=kwargs.get("parameters"))))
-        result = original_function(*args, **kwargs)
-        if isinstance(result, clickhouse_connect.driver.query.QueryResult):
-            column_names = result.column_names
-            result = result.result_rows
-            result = [dict(zip(column_names, row)) for row in result]
-
-        return result
-
-    return wrapper
-
-
-class ClickHouseConnectionPool:
-    def __init__(self, min_size, max_size):
-        self.min_size = min_size
-        self.max_size = max_size
-        self.pool = Queue()
-        self.lock = threading.Lock()
-        self.total_connections = 0
-
-        # Initialize the pool with min_size connections
-        for _ in range(self.min_size):
-            client = clickhouse_connect.get_client(**CH_CONFIG,
-                                                   database=config("ch_database", default="default"),
-                                                   settings=settings,
-                                                   **extra_args)
-            self.pool.put(client)
-            self.total_connections += 1
-
-    def get_connection(self):
-        try:
-            # Try to get a connection without blocking
-            client = self.pool.get_nowait()
-            return client
-        except Empty:
-            with self.lock:
-                if self.total_connections < self.max_size:
-                    client = clickhouse_connect.get_client(**CH_CONFIG,
-                                                           database=config("ch_database", default="default"),
-                                                           settings=settings,
-                                                           **extra_args)
-                    self.total_connections += 1
-                    return client
-        # If max_size reached, wait until a connection is available
-        client = self.pool.get()
-        return client
-
-    def release_connection(self, client):
-        self.pool.put(client)
-
-    def close_all(self):
-        with self.lock:
-            while not self.pool.empty():
-                client = self.pool.get()
-                client.close()
-            self.total_connections = 0
-
-
-CH_pool: ClickHouseConnectionPool = None
-
-RETRY_MAX = config("CH_RETRY_MAX", cast=int, default=50)
-RETRY_INTERVAL = config("CH_RETRY_INTERVAL", cast=int, default=2)
-RETRY = 0
-
-
-def make_pool():
-    if not config('CH_POOL', cast=bool, default=True):
-        return
-    global CH_pool
-    global RETRY
-    if CH_pool is not None:
-        try:
-            CH_pool.close_all()
-        except Exception as error:
-            logger.error("Error while closing all connexions to CH", exc_info=error)
-    try:
-        CH_pool = ClickHouseConnectionPool(min_size=config("CH_MINCONN", cast=int, default=4),
-                                           max_size=config("CH_MAXCONN", cast=int, default=8))
-        if CH_pool is not None:
-            logger.info("Connection pool created successfully for CH")
-    except ConnectionError as error:
-        logger.error("Error while connecting to CH", exc_info=error)
-        if RETRY < RETRY_MAX:
-            RETRY += 1
-            logger.info(f"waiting for {RETRY_INTERVAL}s before retry n°{RETRY}")
-            time.sleep(RETRY_INTERVAL)
-            make_pool()
-        else:
-            raise error
-
 
 class ClickHouseClient:
+    __client = None
+
     def __init__(self, database=None):
-        self.__client = None
-        if database is not None or not config('CH_POOL', cast=bool, default=True):
-            self.__client = clickhouse_connect.get_client(**CH_CONFIG,
-                                                          database=database if database else config("ch_database",
-                                                                                                    default="default"),
-                                                          settings=settings,
-                                                          **extra_args)
-
-        else:
-            self.__client = CH_pool.get_connection()
-
-        self.__client.execute = transform_result(self, self.__client.query)
-        self.__client.format = self.format
+        extra_args = {}
+        if config("CH_COMPRESSION", cast=bool, default=True):
+            extra_args["compression"] = "lz4"
+        self.__client = clickhouse_driver.Client(host=config("ch_host"),
+                                                 database=database if database else config("ch_database",
+                                                                                           default="default"),
+                                                 user=config("ch_user", default="default"),
+                                                 password=config("ch_password", default=""),
+                                                 port=config("ch_port", cast=int),
+                                                 settings=settings,
+                                                 **extra_args) \
+            if self.__client is None else self.__client
 
     def __enter__(self):
+        return self
+
+    def execute(self, query, parameters=None, **args):
+        try:
+            results = self.__client.execute(query=query, params=parameters, with_column_types=True, **args)
+            keys = tuple(x for x, y in results[1])
+            return [dict(zip(keys, i)) for i in results[0]]
+        except Exception as err:
+            logger.error("--------- CH EXCEPTION -----------", exc_info=err)
+            logger.error("--------- CH QUERY EXCEPTION -----------")
+            logger.error(self.format(query=query, parameters=parameters)
+                         .replace('\n', '\\n')
+                         .replace('    ', ' ')
+                         .replace('        ', ' '))
+            logger.error("--------------------")
+            raise err
+
+    def insert(self, query, params=None, **args):
+        return self.__client.execute(query=query, params=params, **args)
+
+    def client(self):
         return self.__client
 
-    def format(self, query, *, parameters=None):
+    def format(self, query, parameters):
         if parameters is None:
             return query
-        return query % {
-            key: f"'{value}'" if isinstance(value, str) else value
-            for key, value in parameters.items()
-        }
-
-    async def health_check(self) -> Dict[str, Any]:
-        """
-        Check if the connection to ClickHouse is working.
-
-        Returns:
-            Dict with status information:
-            {
-                "status": "ok" | "error",
-                "message": str,
-                "ping_time_ms": float (if available)
-            }
-        """
-        try:
-            start_time = asyncio.get_event_loop().time()
-
-            # Run the query in a thread pool to avoid blocking the event loop
-            loop = asyncio.get_event_loop()
-            await loop.run_in_executor(None, lambda: self.__client.query("SELECT 1"))
-
-            end_time = asyncio.get_event_loop().time()
-            ping_time_ms = (end_time - start_time) * 1000
-
-            return {
-                "status": "ok",
-                "message": "ClickHouse connection is healthy",
-                "ping_time_ms": round(ping_time_ms, 2)
-            }
-        except Exception as e:
-            logger.error(f"ClickHouse health check failed: {str(e)}")
-            return {
-                "status": "error",
-                "message": f"Failed to connect to ClickHouse: {str(e)}",
-            }
-
-    @classmethod
-    async def health_check(cls) -> Dict[str, Any]:
-        """
-        Class method to check if the ClickHouse connection is working.
-        Can be called directly on the class: await ClickHouseClient.health_check()
-
-        Returns:
-            Dict with status information
-        """
-        try:
-            # Create a temporary client just for the health check
-            client = cls()
-
-            start_time = asyncio.get_event_loop().time()
-
-            # Run the query in a thread pool
-            loop = asyncio.get_event_loop()
-            await loop.run_in_executor(None, lambda: client.__client.query("SELECT 1"))
-
-            end_time = asyncio.get_event_loop().time()
-            ping_time_ms = (end_time - start_time) * 1000
-
-            result = {
-                "status": "ok",
-                "message": "ClickHouse connection is healthy",
-                "ping_time_ms": round(ping_time_ms, 2)
-            }
-
-            # Clean up the connection properly
-            if config('CH_POOL', cast=bool, default=True):
-                CH_pool.release_connection(client.__client)
-
-            return result
-
-        except Exception as e:
-            logger.error(f"ClickHouse health check failed: {str(e)}")
-            return {
-                "status": "error",
-                "message": f"Failed to connect to ClickHouse: {str(e)}",
-            }
+        return self.__client.substitute_params(query, parameters, self.__client.connection.context)
 
     def __exit__(self, *args):
-        if config('CH_POOL', cast=bool, default=True):
-            CH_pool.release_connection(self.__client)
-        else:
-            self.__client.close()
-
-
-# Add the get_client function at module level
-def get_client(database=None) -> ClickHouseClient:
-    """
-    Get a ClickHouse client instance.
-
-    Args:
-        database: Optional database name to override the default
-
-    Returns:
-        ClickHouseClient instance
-    """
-    return ClickHouseClient(database=database)
+        pass
 
 
 async def init():
-    logger.info(f">use CH_POOL:{config('CH_POOL', default=True)}")
-    if config('CH_POOL', cast=bool, default=True):
-        make_pool()
-
-    # Do a health check at initialization to verify connection
-    try:
-        health_status = await ClickHouseClient.health_check()
-        if health_status["status"] == "ok":
-            logger.info(f"ClickHouse connection verified. Ping: {health_status.get('ping_time_ms', 'N/A')}ms")
-        else:
-            logger.warning(f"ClickHouse connection check failed: {health_status['message']}")
-    except Exception as e:
-        logger.error(f"Error during initialization health check: {str(e)}")
+    logger.info(f">CH_POOL:not defined")
 
 
 async def terminate():
-    global CH_pool
-    if CH_pool is not None:
-        try:
-            CH_pool.close_all()
-            logger.info("Closed all connexions to CH")
-        except Exception as error:
-            logger.error("Error while closing all connexions to CH", exc_info=error)
-
-
-async def health_check() -> Dict[str, Any]:
-    """
-    Public health check function that can be used by the application.
-
-    Returns:
-        Health status dict
-    """
-    return await ClickHouseClient.health_check()
+    pass

api/chalicelib/utils/ch_client_exp.py

@@ -1,10 +1,8 @@
 import logging
 import threading
 import time
-import asyncio
 from functools import wraps
 from queue import Queue, Empty
-from typing import Dict, Any, Optional
 
 import clickhouse_connect
 from clickhouse_connect.driver.query import QueryContext
@@ -36,7 +34,10 @@ if config("CH_COMPRESSION", cast=bool, default=True):
 def transform_result(self, original_function):
     @wraps(original_function)
     def wrapper(*args, **kwargs):
-        logger.debug(str.encode(self.format(query=kwargs.get("query", ""), parameters=kwargs.get("parameters"))))
+        if kwargs.get("parameters"):
+            logger.debug(str.encode(self.format(query=kwargs.get("query", ""), parameters=kwargs.get("parameters"))))
+        elif len(args) > 0:
+            logger.debug(str.encode(args[0]))
         result = original_function(*args, **kwargs)
         if isinstance(result, clickhouse_connect.driver.query.QueryResult):
             column_names = result.column_names
@@ -148,80 +149,11 @@ class ClickHouseClient:
     def __enter__(self):
         return self.__client
 
-    def format(self, query, *, parameters=None):
-        if parameters is None:
-            return query
-        return query % {
-            key: f"'{value}'" if isinstance(value, str) else value
-            for key, value in parameters.items()
-        }
-
-    async def health_check(self) -> Dict[str, Any]:
-        """
-        Check if the connection to ClickHouse is working.
-
-        Returns:
-            Dict with status information:
-            {
-                "status": "ok" | "error",
-                "message": str,
-                "ping_time_ms": float (if available)
-            }
-        """
-        try:
-            start_time = asyncio.get_event_loop().time()
-
-            # Run the query in a thread pool to avoid blocking the event loop
-            loop = asyncio.get_event_loop()
-            await loop.run_in_executor(None, lambda: self.__client.query("SELECT 1"))
-
-            end_time = asyncio.get_event_loop().time()
-            ping_time_ms = (end_time - start_time) * 1000
-
-            return {
-                "status": "ok",
-                "message": "ClickHouse connection is healthy",
-                "ping_time_ms": round(ping_time_ms, 2)
-            }
-        except Exception as e:
-            logger.error(f"ClickHouse health check failed: {str(e)}")
-            return {
-                "status": "error",
-                "message": f"Failed to connect to ClickHouse: {str(e)}",
-            }
-
-    @classmethod
-    async def health_check(cls) -> Dict[str, Any]:
-        """
-        Class method to check if the ClickHouse connection is working.
-        Can be called directly on the class: await ClickHouseClient.health_check()
-
-        Returns:
-            Dict with status information
-        """
-        client = get_client()
-        try:
-            start_time = asyncio.get_event_loop().time()
-
-            # Run the query in a thread pool to avoid blocking the event loop
-            loop = asyncio.get_event_loop()
-            client_instance = client.__client
-            await loop.run_in_executor(None, lambda: client_instance.query("SELECT 1"))
-
-            end_time = asyncio.get_event_loop().time()
-            ping_time_ms = (end_time - start_time) * 1000
-
-            return {
-                "status": "ok",
-                "message": "ClickHouse connection is healthy",
-                "ping_time_ms": round(ping_time_ms, 2)
-            }
-        except Exception as e:
-            logger.error(f"ClickHouse health check failed: {str(e)}")
-            return {
-                "status": "error",
-                "message": f"Failed to connect to ClickHouse: {str(e)}",
-            }
+    def format(self, query, parameters=None):
+        if parameters:
+            ctx = QueryContext(query=query, parameters=parameters)
+            return ctx.final_query
+        return query
 
     def __exit__(self, *args):
         if config('CH_POOL', cast=bool, default=True):
@@ -230,35 +162,11 @@ class ClickHouseClient:
             self.__client.close()
 
 
-# Add the get_client function at module level
-def get_client(database=None) -> ClickHouseClient:
-    """
-    Get a ClickHouse client instance.
-
-    Args:
-        database: Optional database name to override the default
-
-    Returns:
-        ClickHouseClient instance
-    """
-    return ClickHouseClient(database=database)
-
-
 async def init():
     logger.info(f">use CH_POOL:{config('CH_POOL', default=True)}")
     if config('CH_POOL', cast=bool, default=True):
         make_pool()
 
-    # Do a health check at initialization to verify connection
-    try:
-        health_status = await ClickHouseClient.health_check()
-        if health_status["status"] == "ok":
-            logger.info(f"ClickHouse connection verified. Ping: {health_status.get('ping_time_ms', 'N/A')}ms")
-        else:
-            logger.warning(f"ClickHouse connection check failed: {health_status['message']}")
-    except Exception as e:
-        logger.error(f"Error during initialization health check: {str(e)}")
-
 
 async def terminate():
     global CH_pool
@@ -268,13 +176,3 @@ async def terminate():
             logger.info("Closed all connexions to CH")
         except Exception as error:
             logger.error("Error while closing all connexions to CH", exc_info=error)
-
-
-async def health_check() -> Dict[str, Any]:
-    """
-    Public health check function that can be used by the application.
-
-    Returns:
-        Health status dict
-    """
-    return await ClickHouseClient.health_check()

api/chalicelib/utils/errors_helper.py

@@ -1,14 +0,0 @@
-from chalicelib.core.sourcemaps import sourcemaps
-
-
-def format_first_stack_frame(error):
-    error["stack"] = sourcemaps.format_payload(error.pop("payload"), truncate_to_first=True)
-    for s in error["stack"]:
-        for c in s.get("context", []):
-            for sci, sc in enumerate(c):
-                if isinstance(sc, str) and len(sc) > 1000:
-                    c[sci] = sc[:1000]
-        # convert bytes to string:
-        if isinstance(s["filename"], bytes):
-            s["filename"] = s["filename"].decode("utf-8")
-    return error

api/chalicelib/utils/pg_client.py

@@ -1,8 +1,6 @@
 import logging
 import time
-import asyncio
 from threading import Semaphore
-from typing import Dict, Any, Optional
 
 import psycopg2
 import psycopg2.extras
@@ -21,6 +19,16 @@ PG_CONFIG = dict(_PG_CONFIG)
 if config("PG_TIMEOUT", cast=int, default=0) > 0:
     PG_CONFIG["options"] = f"-c statement_timeout={config('PG_TIMEOUT', cast=int) * 1000}"
 
+if config('PG_POOL', cast=bool, default=True):
+    PG_CONFIG = {
+        **PG_CONFIG,
+        # Keepalive settings
+        "keepalives": 1,  # Enable keepalives
+        "keepalives_idle": 300,  # Seconds before sending keepalive
+        "keepalives_interval": 10,  # Seconds between keepalives
+        "keepalives_count": 3  # Number of keepalives before giving up
+    }
+
 
 class ORThreadedConnectionPool(psycopg2.pool.ThreadedConnectionPool):
     def __init__(self, minconn, maxconn, *args, **kwargs):
@@ -57,6 +65,7 @@ RETRY = 0
 
 def make_pool():
     if not config('PG_POOL', cast=bool, default=True):
+        logger.info("PG_POOL is disabled, not creating a new one")
         return
     global postgreSQL_pool
     global RETRY
@@ -175,114 +184,10 @@ class PostgresClient:
         self.cursor = None
         return self.__enter__()
 
-    async def health_check(self) -> Dict[str, Any]:
-        """
-        Instance method to check DB connection health
-        """
-        try:
-            start_time = asyncio.get_event_loop().time()
-
-            # Run the query in a thread pool to avoid blocking the event loop
-            loop = asyncio.get_event_loop()
-
-            def check_db():
-                cursor = self.connection.cursor()
-                cursor.execute("SELECT 1")
-                cursor.close()
-                return True
-
-            await loop.run_in_executor(None, check_db)
-
-            end_time = asyncio.get_event_loop().time()
-            ping_time_ms = (end_time - start_time) * 1000
-
-            return {
-                "status": "ok",
-                "message": "PostgreSQL connection is healthy",
-                "ping_time_ms": round(ping_time_ms, 2)
-            }
-        except Exception as e:
-            logger.error(f"PostgreSQL health check failed: {e}")
-            return {
-                "status": "error",
-                "message": f"Failed to connect to PostgreSQL: {str(e)}"
-            }
-
-    @classmethod
-    async def health_check(cls) -> Dict[str, Any]:
-        """
-        Class method to check if PostgreSQL connection works.
-        Can be called directly on the class: await PostgresClient.health_check()
-        """
-        try:
-            # Create a temporary client for the health check
-            client = cls()
-
-            start_time = asyncio.get_event_loop().time()
-
-            # Run the query in a thread pool
-            loop = asyncio.get_event_loop()
-
-            def check_db():
-                cursor = client.connection.cursor()
-                cursor.execute("SELECT 1")
-                cursor.close()
-                return True
-
-            await loop.run_in_executor(None, check_db)
-
-            end_time = asyncio.get_event_loop().time()
-            ping_time_ms = (end_time - start_time) * 1000
-
-            # Properly clean up the connection
-            if not client.use_pool or client.long_query or client.unlimited_query:
-                client.connection.close()
-            else:
-                postgreSQL_pool.putconn(client.connection)
-
-            return {
-                "status": "ok",
-                "message": "PostgreSQL connection is healthy",
-                "ping_time_ms": round(ping_time_ms, 2)
-            }
-        except Exception as e:
-            logger.error(f"PostgreSQL health check failed: {e}")
-            return {
-                "status": "error",
-                "message": f"Failed to connect to PostgreSQL: {str(e)}"
-            }
-
-
-# Add get_client function at module level
-def get_client(long_query=False, unlimited_query=False, use_pool=True) -> PostgresClient:
-    """
-    Get a PostgreSQL client instance.
-
-    Args:
-        long_query: Set True for queries with extended timeout
-        unlimited_query: Set True for queries with no timeout
-        use_pool: Whether to use the connection pool
-
-    Returns:
-        PostgresClient instance
-    """
-    return PostgresClient(long_query=long_query, unlimited_query=unlimited_query, use_pool=use_pool)
-
 
 async def init():
     logger.info(f">use PG_POOL:{config('PG_POOL', default=True)}")
-    if config('PG_POOL', cast=bool, default=True):
-        make_pool()
-
-    # Do a health check at initialization
-    try:
-        health_status = await PostgresClient.health_check()
-        if health_status["status"] == "ok":
-            logger.info(f"PostgreSQL connection verified. Ping: {health_status.get('ping_time_ms', 'N/A')}ms")
-        else:
-            logger.warning(f"PostgreSQL connection check failed: {health_status['message']}")
-    except Exception as e:
-        logger.error(f"Error during initialization health check: {str(e)}")
+    make_pool()
 
 
 async def terminate():
@@ -293,13 +198,3 @@ async def terminate():
             logger.info("Closed all connexions to PostgreSQL")
         except (Exception, psycopg2.DatabaseError) as error:
             logger.error("Error while closing all connexions to PostgreSQL", exc_info=error)
-
-
-async def health_check() -> Dict[str, Any]:
-    """
-    Public health check function that can be used by the application.
-
-    Returns:
-        Health status dict
-    """
-    return await PostgresClient.health_check()

api/chalicelib/utils/sql_helper.py

@@ -4,37 +4,41 @@ import schemas
 
 
 def get_sql_operator(op: Union[schemas.SearchEventOperator, schemas.ClickEventExtraOperator, schemas.MathOperator]):
+    if isinstance(op, Enum):
+        op = op.value
     return {
-        schemas.SearchEventOperator.IS: "=",
-        schemas.SearchEventOperator.ON: "=",
-        schemas.SearchEventOperator.ON_ANY: "IN",
-        schemas.SearchEventOperator.IS_NOT: "!=",
-        schemas.SearchEventOperator.NOT_ON: "!=",
-        schemas.SearchEventOperator.CONTAINS: "ILIKE",
-        schemas.SearchEventOperator.NOT_CONTAINS: "NOT ILIKE",
-        schemas.SearchEventOperator.STARTS_WITH: "ILIKE",
-        schemas.SearchEventOperator.ENDS_WITH: "ILIKE",
+        schemas.SearchEventOperator.IS.value: "=",
+        schemas.SearchEventOperator.ON.value: "=",
+        schemas.SearchEventOperator.ON_ANY.value: "IN",
+        schemas.SearchEventOperator.IS_NOT.value: "!=",
+        schemas.SearchEventOperator.NOT_ON.value: "!=",
+        schemas.SearchEventOperator.CONTAINS.value: "ILIKE",
+        schemas.SearchEventOperator.NOT_CONTAINS.value: "NOT ILIKE",
+        schemas.SearchEventOperator.STARTS_WITH.value: "ILIKE",
+        schemas.SearchEventOperator.ENDS_WITH.value: "ILIKE",
         # Selector operators:
-        schemas.ClickEventExtraOperator.IS: "=",
-        schemas.ClickEventExtraOperator.IS_NOT: "!=",
-        schemas.ClickEventExtraOperator.CONTAINS: "ILIKE",
-        schemas.ClickEventExtraOperator.NOT_CONTAINS: "NOT ILIKE",
-        schemas.ClickEventExtraOperator.STARTS_WITH: "ILIKE",
-        schemas.ClickEventExtraOperator.ENDS_WITH: "ILIKE",
+        schemas.ClickEventExtraOperator.IS.value: "=",
+        schemas.ClickEventExtraOperator.IS_NOT.value: "!=",
+        schemas.ClickEventExtraOperator.CONTAINS.value: "ILIKE",
+        schemas.ClickEventExtraOperator.NOT_CONTAINS.value: "NOT ILIKE",
+        schemas.ClickEventExtraOperator.STARTS_WITH.value: "ILIKE",
+        schemas.ClickEventExtraOperator.ENDS_WITH.value: "ILIKE",
 
-        schemas.MathOperator.GREATER: ">",
-        schemas.MathOperator.GREATER_EQ: ">=",
-        schemas.MathOperator.LESS: "<",
-        schemas.MathOperator.LESS_EQ: "<=",
+        schemas.MathOperator.GREATER.value: ">",
+        schemas.MathOperator.GREATER_EQ.value: ">=",
+        schemas.MathOperator.LESS.value: "<",
+        schemas.MathOperator.LESS_EQ.value: "<=",
     }.get(op, "=")
 
 
 def is_negation_operator(op: schemas.SearchEventOperator):
-    return op in [schemas.SearchEventOperator.IS_NOT,
-                  schemas.SearchEventOperator.NOT_ON,
-                  schemas.SearchEventOperator.NOT_CONTAINS,
-                  schemas.ClickEventExtraOperator.IS_NOT,
-                  schemas.ClickEventExtraOperator.NOT_CONTAINS]
+    if isinstance(op, Enum):
+        op = op.value
+    return op in [schemas.SearchEventOperator.IS_NOT.value,
+                  schemas.SearchEventOperator.NOT_ON.value,
+                  schemas.SearchEventOperator.NOT_CONTAINS.value,
+                  schemas.ClickEventExtraOperator.IS_NOT.value,
+                  schemas.ClickEventExtraOperator.NOT_CONTAINS.value]
 
 
 def reverse_sql_operator(op):

api/product_analytics_working_query.sql

@@ -1,591 +0,0 @@
--- -- Original Q3
--- WITH ranked_events AS (SELECT *
---                        FROM ranked_events_1736344377403),
---      n1 AS (SELECT event_number_in_session,
---                    event_type,
---                    e_value,
---                    next_type,
---                    next_value,
---                    COUNT(1) AS sessions_count
---             FROM ranked_events
---             WHERE event_number_in_session = 1
---               AND isNotNull(next_value)
---             GROUP BY event_number_in_session, event_type, e_value, next_type, next_value
---             ORDER BY sessions_count DESC
---             LIMIT 8),
---      n2 AS (SELECT *
---             FROM (SELECT re.event_number_in_session AS event_number_in_session,
---                          re.event_type              AS event_type,
---                          re.e_value                 AS e_value,
---                          re.next_type               AS next_type,
---                          re.next_value              AS next_value,
---                          COUNT(1)                   AS sessions_count
---                   FROM n1
---                            INNER JOIN ranked_events AS re
---                                       ON (n1.next_value = re.e_value AND n1.next_type = re.event_type)
---                   WHERE re.event_number_in_session = 2
---                   GROUP BY re.event_number_in_session, re.event_type, re.e_value, re.next_type,
---                            re.next_value) AS sub_level
---             ORDER BY sessions_count DESC
---             LIMIT 8),
---      n3 AS (SELECT *
---             FROM (SELECT re.event_number_in_session AS event_number_in_session,
---                          re.event_type              AS event_type,
---                          re.e_value                 AS e_value,
---                          re.next_type               AS next_type,
---                          re.next_value              AS next_value,
---                          COUNT(1)                   AS sessions_count
---                   FROM n2
---                            INNER JOIN ranked_events AS re
---                                       ON (n2.next_value = re.e_value AND n2.next_type = re.event_type)
---                   WHERE re.event_number_in_session = 3
---                   GROUP BY re.event_number_in_session, re.event_type, re.e_value, re.next_type,
---                            re.next_value) AS sub_level
---             ORDER BY sessions_count DESC
---             LIMIT 8),
---      n4 AS (SELECT *
---             FROM (SELECT re.event_number_in_session AS event_number_in_session,
---                          re.event_type              AS event_type,
---                          re.e_value                 AS e_value,
---                          re.next_type               AS next_type,
---                          re.next_value              AS next_value,
---                          COUNT(1)                   AS sessions_count
---                   FROM n3
---                            INNER JOIN ranked_events AS re
---                                       ON (n3.next_value = re.e_value AND n3.next_type = re.event_type)
---                   WHERE re.event_number_in_session = 4
---                   GROUP BY re.event_number_in_session, re.event_type, re.e_value, re.next_type,
---                            re.next_value) AS sub_level
---             ORDER BY sessions_count DESC
---             LIMIT 8),
---      n5 AS (SELECT *
---             FROM (SELECT re.event_number_in_session AS event_number_in_session,
---                          re.event_type              AS event_type,
---                          re.e_value                 AS e_value,
---                          re.next_type               AS next_type,
---                          re.next_value              AS next_value,
---                          COUNT(1)                   AS sessions_count
---                   FROM n4
---                            INNER JOIN ranked_events AS re
---                                       ON (n4.next_value = re.e_value AND n4.next_type = re.event_type)
---                   WHERE re.event_number_in_session = 5
---                   GROUP BY re.event_number_in_session, re.event_type, re.e_value, re.next_type,
---                            re.next_value) AS sub_level
---             ORDER BY sessions_count DESC
---             LIMIT 8)
--- SELECT *
--- FROM (SELECT event_number_in_session,
---              event_type,
---              e_value,
---              next_type,
---              next_value,
---              sessions_count
---       FROM n1
---       UNION ALL
---       SELECT event_number_in_session,
---              event_type,
---              e_value,
---              next_type,
---              next_value,
---              sessions_count
---       FROM n2
---       UNION ALL
---       SELECT event_number_in_session,
---              event_type,
---              e_value,
---              next_type,
---              next_value,
---              sessions_count
---       FROM n3
---       UNION ALL
---       SELECT event_number_in_session,
---              event_type,
---              e_value,
---              next_type,
---              next_value,
---              sessions_count
---       FROM n4
---       UNION ALL
---       SELECT event_number_in_session,
---              event_type,
---              e_value,
---              next_type,
---              next_value,
---              sessions_count
---       FROM n5) AS chart_steps
--- ORDER BY event_number_in_session;
-
--- Q1
--- CREATE TEMPORARY TABLE pre_ranked_events_1736344377403 AS
-CREATE TABLE pre_ranked_events_1736344377403 ENGINE = Memory AS
-    (WITH initial_event AS (SELECT events.session_id, MIN(datetime) AS start_event_timestamp
-                            FROM experimental.events AS events
-                            WHERE ((event_type = 'LOCATION' AND (url_path = '/en/deployment/')))
-                              AND events.project_id = toUInt16(65)
-                              AND events.datetime >= toDateTime(1735599600000 / 1000)
-                              AND events.datetime < toDateTime(1736290799999 / 1000)
-                            GROUP BY 1),
-          pre_ranked_events AS (SELECT *
-                                FROM (SELECT session_id,
-                                             event_type,
-                                             datetime,
-                                             url_path             AS e_value,
-                                             row_number() OVER (PARTITION BY session_id
-                                                 ORDER BY datetime ,
-                                                     message_id ) AS event_number_in_session
-                                      FROM experimental.events AS events
-                                               INNER JOIN initial_event ON (events.session_id = initial_event.session_id)
-                                      WHERE events.project_id = toUInt16(65)
-                                        AND events.datetime >= toDateTime(1735599600000 / 1000)
-                                        AND events.datetime < toDateTime(1736290799999 / 1000)
-                                        AND (events.event_type = 'LOCATION')
-                                        AND events.datetime >= initial_event.start_event_timestamp
-                                         ) AS full_ranked_events
-                                WHERE event_number_in_session <= 5)
-     SELECT *
-     FROM pre_ranked_events);
-;
-
-SELECT *
-FROM pre_ranked_events_1736344377403
-WHERE event_number_in_session < 3;
-
-
-
--- ---------Q2-----------
--- CREATE TEMPORARY TABLE ranked_events_1736344377403 AS
-DROP TABLE ranked_events_1736344377403;
-CREATE TABLE ranked_events_1736344377403 ENGINE = Memory AS
-    (WITH pre_ranked_events AS (SELECT *
-                                FROM pre_ranked_events_1736344377403),
-          start_points AS (SELECT DISTINCT session_id
-                           FROM pre_ranked_events
-                           WHERE ((event_type = 'LOCATION' AND (e_value = '/en/deployment/')))
-                             AND pre_ranked_events.event_number_in_session = 1),
-          ranked_events AS (SELECT pre_ranked_events.*,
-                                   leadInFrame(e_value)
-                                               OVER (PARTITION BY session_id ORDER BY datetime
-                                                   ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS next_value,
-                                   leadInFrame(toNullable(event_type))
-                                               OVER (PARTITION BY session_id ORDER BY datetime
-                                                   ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS next_type
-                            FROM start_points
-                                     INNER JOIN pre_ranked_events USING (session_id))
-     SELECT *
-     FROM ranked_events);
-
-
--- ranked events
-SELECT event_number_in_session,
-       event_type,
-       e_value,
-       next_type,
-       next_value,
-       COUNT(1) AS sessions_count
-FROM ranked_events_1736344377403
-WHERE event_number_in_session = 2
---   AND e_value='/en/deployment/deploy-docker/'
---     AND next_value NOT IN ('/en/deployment/','/en/plugins/','/en/using-or/')
---     AND e_value NOT IN ('/en/deployment/deploy-docker/','/en/getting-started/','/en/deployment/deploy-ubuntu/')
-  AND isNotNull(next_value)
-GROUP BY event_number_in_session, event_type, e_value, next_type, next_value
-ORDER BY event_number_in_session, sessions_count DESC;
-
-
-
-SELECT event_number_in_session,
-       event_type,
-       e_value,
-       COUNT(1) AS sessions_count
-FROM ranked_events_1736344377403
-WHERE event_number_in_session = 1
-GROUP BY event_number_in_session, event_type, e_value
-ORDER BY event_number_in_session, sessions_count DESC;
-
-SELECT COUNT(1) AS sessions_count
-FROM ranked_events_1736344377403
-WHERE event_number_in_session = 2
-  AND isNull(next_value)
-;
-
--- ---------Q3 MORE -----------
-WITH ranked_events AS (SELECT *
-                       FROM ranked_events_1736344377403),
-     n1 AS (SELECT event_number_in_session,
-                   event_type,
-                   e_value,
-                   next_type,
-                   next_value,
-                   COUNT(1) AS sessions_count
-            FROM ranked_events
-            WHERE event_number_in_session = 1
-            GROUP BY event_number_in_session, event_type, e_value, next_type, next_value
-            ORDER BY sessions_count DESC),
-     n2 AS (SELECT event_number_in_session,
-                   event_type,
-                   e_value,
-                   next_type,
-                   next_value,
-                   COUNT(1) AS sessions_count
-            FROM ranked_events
-            WHERE event_number_in_session = 2
-            GROUP BY event_number_in_session, event_type, e_value, next_type, next_value
-            ORDER BY sessions_count DESC),
-     n3 AS (SELECT event_number_in_session,
-                   event_type,
-                   e_value,
-                   next_type,
-                   next_value,
-                   COUNT(1) AS sessions_count
-            FROM ranked_events
-            WHERE event_number_in_session = 3
-            GROUP BY event_number_in_session, event_type, e_value, next_type, next_value
-            ORDER BY sessions_count DESC),
-     drop_n AS (-- STEP 1
-         SELECT event_number_in_session,
-                event_type,
-                e_value,
-                'DROP' AS next_type,
-                NULL   AS next_value,
-                sessions_count
-         FROM n1
-         WHERE isNull(n1.next_type)
-         UNION ALL
-         -- STEP 2
-         SELECT event_number_in_session,
-                event_type,
-                e_value,
-                'DROP' AS next_type,
-                NULL   AS next_value,
-                sessions_count
-         FROM n2
-         WHERE isNull(n2.next_type)),
---      TODO: make this as top_steps, where every step will go to next as top/others
-     top_n1 AS (-- STEP 1
-         SELECT event_number_in_session,
-                event_type,
-                e_value,
-                next_type,
-                next_value,
-                sessions_count
-         FROM n1
-         WHERE isNotNull(next_type)
-         ORDER BY sessions_count DESC
-         LIMIT 3),
-     top_n2 AS (-- STEP 2
-         SELECT event_number_in_session,
-                event_type,
-                e_value,
-                next_type,
-                next_value,
-                sessions_count
-         FROM n2
-         WHERE (event_type, e_value) IN (SELECT event_type,
-                                                e_value
-                                         FROM n2
-                                         WHERE isNotNull(next_type)
-                                         GROUP BY event_type, e_value
-                                         ORDER BY SUM(sessions_count) DESC
-                                         LIMIT 3)
-         ORDER BY sessions_count DESC),
-     top_n AS (SELECT *
-               FROM top_n1
-               UNION ALL
-               SELECT *
-               FROM top_n2),
-     u_top_n AS (SELECT DISTINCT event_number_in_session,
-                                 event_type,
-                                 e_value
-                 FROM top_n),
-     others_n AS (
-         -- STEP 1
-         SELECT event_number_in_session,
-                event_type,
-                e_value,
-                next_type,
-                next_value,
-                sessions_count
-         FROM n1
-         WHERE isNotNull(next_type)
-         ORDER BY sessions_count DESC
-         LIMIT 1000000 OFFSET 3
-         UNION ALL
-         -- STEP 2
-         SELECT event_number_in_session,
-                event_type,
-                e_value,
-                next_type,
-                next_value,
-                sessions_count
-         FROM n2
-         WHERE isNotNull(next_type)
---          GROUP BY event_number_in_session, event_type, e_value
-         ORDER BY sessions_count DESC
-         LIMIT 1000000 OFFSET 3)
-SELECT *
-FROM (
--- Top
-         SELECT *
-         FROM top_n
-         --          UNION ALL
--- -- Others
---          SELECT event_number_in_session,
---                 event_type,
---                 e_value,
---                 'OTHER' AS next_type,
---                 NULL    AS next_value,
---                 SUM(sessions_count)
---          FROM others_n
---          GROUP BY event_number_in_session, event_type, e_value
---          UNION ALL
--- -- Top go to Drop
---          SELECT drop_n.event_number_in_session,
---                 drop_n.event_type,
---                 drop_n.e_value,
---                 drop_n.next_type,
---                 drop_n.next_value,
---                 drop_n.sessions_count
---          FROM drop_n
---                   INNER JOIN u_top_n ON (drop_n.event_number_in_session = u_top_n.event_number_in_session
---              AND drop_n.event_type = u_top_n.event_type
---              AND drop_n.e_value = u_top_n.e_value)
---          ORDER BY drop_n.event_number_in_session
---          --          --          UNION ALL
--- -- -- Top go to Others
---          SELECT top_n.event_number_in_session,
---                 top_n.event_type,
---                 top_n.e_value,
---                 'OTHER'                   AS next_type,
---                 NULL                      AS next_value,
---                 SUM(top_n.sessions_count) AS sessions_count
---          FROM top_n
---                   LEFT JOIN others_n ON (others_n.event_number_in_session = (top_n.event_number_in_session + 1)
---              AND top_n.next_type = others_n.event_type
---              AND top_n.next_value = others_n.e_value)
---          WHERE others_n.event_number_in_session IS NULL
---            AND top_n.next_type IS NOT NULL
---          GROUP BY event_number_in_session, event_type, e_value
---          UNION ALL
--- -- Others got to Top
---          SELECT others_n.event_number_in_session,
---                 'OTHER'               AS event_type,
---                 NULL                  AS e_value,
---                 others_n.s_next_type  AS next_type,
---                 others_n.s_next_value AS next_value,
---                 SUM(sessions_count)   AS sessions_count
---          FROM others_n
---                   INNER JOIN top_n ON (others_n.event_number_in_session = top_n.event_number_in_session + 1 AND
---                                        others_n.s_next_type = top_n.event_type AND
---                                        others_n.s_next_value = top_n.event_type)
---          GROUP BY others_n.event_number_in_session, next_type, next_value
-         --          UNION ALL
---          -- TODO: find if this works or not
--- -- Others got to Others
---          SELECT others_n.event_number_in_session,
---                 'OTHER'             AS event_type,
---                 NULL                AS e_value,
---                 'OTHERS'            AS next_type,
---                 NULL                AS next_value,
---                 SUM(sessions_count) AS sessions_count
---          FROM others_n
---                   LEFT JOIN u_top_n ON ((others_n.event_number_in_session + 1) = u_top_n.event_number_in_session
---              AND others_n.s_next_type = u_top_n.event_type
---              AND others_n.s_next_value = u_top_n.e_value)
---          WHERE u_top_n.event_number_in_session IS NULL
---          GROUP BY others_n.event_number_in_session
-         )
-ORDER BY event_number_in_session;
-
-
--- ---------Q3 TOP ON VALUE ONLY -----------
-WITH ranked_events AS (SELECT *
-                       FROM ranked_events_1736344377403),
-     n1 AS (SELECT event_number_in_session,
-                   event_type,
-                   e_value,
-                   next_type,
-                   next_value,
-                   COUNT(1) AS sessions_count
-            FROM ranked_events
-            WHERE event_number_in_session = 1
-            GROUP BY event_number_in_session, event_type, e_value, next_type, next_value
-            ORDER BY sessions_count DESC),
-     n2 AS (SELECT event_number_in_session,
-                   event_type,
-                   e_value,
-                   next_type,
-                   next_value,
-                   COUNT(1) AS sessions_count
-            FROM ranked_events
-            WHERE event_number_in_session = 2
-            GROUP BY event_number_in_session, event_type, e_value, next_type, next_value
-            ORDER BY sessions_count DESC),
-     n3 AS (SELECT event_number_in_session,
-                   event_type,
-                   e_value,
-                   next_type,
-                   next_value,
-                   COUNT(1) AS sessions_count
-            FROM ranked_events
-            WHERE event_number_in_session = 3
-            GROUP BY event_number_in_session, event_type, e_value, next_type, next_value
-            ORDER BY sessions_count DESC),
-
-     drop_n AS (-- STEP 1
-         SELECT event_number_in_session,
-                event_type,
-                e_value,
-                'DROP' AS next_type,
-                NULL   AS next_value,
-                sessions_count
-         FROM n1
-         WHERE isNull(n1.next_type)
-         UNION ALL
-         -- STEP 2
-         SELECT event_number_in_session,
-                event_type,
-                e_value,
-                'DROP' AS next_type,
-                NULL   AS next_value,
-                sessions_count
-         FROM n2
-         WHERE isNull(n2.next_type)),
-     top_n AS (SELECT event_number_in_session,
-                      event_type,
-                      e_value,
-                      SUM(sessions_count) AS sessions_count
-               FROM n1
-               GROUP BY event_number_in_session, event_type, e_value
-               LIMIT 1
-               UNION ALL
-               -- STEP 2
-               SELECT event_number_in_session,
-                      event_type,
-                      e_value,
-                      SUM(sessions_count) AS sessions_count
-               FROM n2
-               GROUP BY event_number_in_session, event_type, e_value
-               ORDER BY sessions_count DESC
-               LIMIT 3
-               UNION ALL
-               -- STEP 3
-               SELECT event_number_in_session,
-                      event_type,
-                      e_value,
-                      SUM(sessions_count) AS sessions_count
-               FROM n3
-               GROUP BY event_number_in_session, event_type, e_value
-               ORDER BY sessions_count DESC
-               LIMIT 3),
-     top_n_with_next AS (SELECT n1.*
-                         FROM n1
-                         UNION ALL
-                         SELECT n2.*
-                         FROM n2
-                                  INNER JOIN top_n ON (n2.event_number_in_session = top_n.event_number_in_session
-                             AND n2.event_type = top_n.event_type
-                             AND n2.e_value = top_n.e_value)),
-     others_n AS (
-         -- STEP 2
-         SELECT n2.*
-         FROM n2
-         WHERE (n2.event_number_in_session, n2.event_type, n2.e_value) NOT IN
-               (SELECT event_number_in_session, event_type, e_value
-                FROM top_n
-                WHERE top_n.event_number_in_session = 2)
-         UNION ALL
-         -- STEP 3
-         SELECT n3.*
-         FROM n3
-         WHERE (n3.event_number_in_session, n3.event_type, n3.e_value) NOT IN
-               (SELECT event_number_in_session, event_type, e_value
-                FROM top_n
-                WHERE top_n.event_number_in_session = 3))
-SELECT *
-FROM (
-         --          SELECT sum(top_n_with_next.sessions_count)
---          FROM top_n_with_next
---          WHERE event_number_in_session = 1
--- --            AND isNotNull(next_value)
---            AND (next_type, next_value) IN
---                (SELECT others_n.event_type, others_n.e_value FROM others_n WHERE others_n.event_number_in_session = 2)
---            --                   SELECT * FROM others_n
---            --     SELECT * FROM n2
---              SELECT *
---          FROM top_n
---          );
--- Top to Top: valid
-         SELECT top_n_with_next.*
-         FROM top_n_with_next
-                  INNER JOIN top_n
-                             ON (top_n_with_next.event_number_in_session + 1 = top_n.event_number_in_session
-                                 AND top_n_with_next.next_type = top_n.event_type
-                                 AND top_n_with_next.next_value = top_n.e_value)
-         UNION ALL
--- Top to Others: valid
-         SELECT top_n_with_next.event_number_in_session,
-                top_n_with_next.event_type,
-                top_n_with_next.e_value,
-                'OTHER'                             AS next_type,
-                NULL                                AS next_value,
-                SUM(top_n_with_next.sessions_count) AS sessions_count
-         FROM top_n_with_next
-         WHERE (top_n_with_next.event_number_in_session + 1, top_n_with_next.next_type, top_n_with_next.next_value) IN
-               (SELECT others_n.event_number_in_session, others_n.event_type, others_n.e_value FROM others_n)
-         GROUP BY top_n_with_next.event_number_in_session, top_n_with_next.event_type, top_n_with_next.e_value
-         UNION ALL
--- Top go to Drop: valid
-         SELECT drop_n.event_number_in_session,
-                drop_n.event_type,
-                drop_n.e_value,
-                drop_n.next_type,
-                drop_n.next_value,
-                drop_n.sessions_count
-         FROM drop_n
-                  INNER JOIN top_n ON (drop_n.event_number_in_session = top_n.event_number_in_session
-             AND drop_n.event_type = top_n.event_type
-             AND drop_n.e_value = top_n.e_value)
-         ORDER BY drop_n.event_number_in_session
-         UNION ALL
--- Others got to Drop: valid
-         SELECT others_n.event_number_in_session,
-                'OTHER'                      AS event_type,
-                NULL                         AS e_value,
-                'DROP'                       AS next_type,
-                NULL                         AS next_value,
-                SUM(others_n.sessions_count) AS sessions_count
-         FROM others_n
-         WHERE isNull(others_n.next_type)
-           AND others_n.event_number_in_session < 3
-         GROUP BY others_n.event_number_in_session, next_type, next_value
-         UNION ALL
--- Others got to Top:valid
-         SELECT others_n.event_number_in_session,
-                'OTHER'                      AS event_type,
-                NULL                         AS e_value,
-                others_n.next_type,
-                others_n.next_value,
-                SUM(others_n.sessions_count) AS sessions_count
-         FROM others_n
-         WHERE isNotNull(others_n.next_type)
-           AND (others_n.event_number_in_session + 1, others_n.next_type, others_n.next_value) IN
-               (SELECT top_n.event_number_in_session, top_n.event_type, top_n.e_value FROM top_n)
-         GROUP BY others_n.event_number_in_session, others_n.next_type, others_n.next_value
-         UNION ALL
--- Others got to Others
-         SELECT others_n.event_number_in_session,
-                'OTHER'             AS event_type,
-                NULL                AS e_value,
-                'OTHERS'            AS next_type,
-                NULL                AS next_value,
-                SUM(sessions_count) AS sessions_count
-         FROM others_n
-         WHERE isNotNull(others_n.next_type)
-           AND others_n.event_number_in_session < 3
-           AND (others_n.event_number_in_session + 1, others_n.next_type, others_n.next_value) NOT IN
-               (SELECT event_number_in_session, event_type, e_value FROM top_n)
-         GROUP BY others_n.event_number_in_session)
-ORDER BY event_number_in_session, sessions_count
-        DESC;
-
-

api/routers/core_dynamic.py

@@ -7,27 +7,30 @@ from fastapi import HTTPException, status
 from starlette.responses import RedirectResponse, FileResponse, JSONResponse, Response
 
 import schemas
-from chalicelib.core import scope
 from chalicelib.core import assist, signup, feature_flags
-from chalicelib.core.metrics import heatmaps
-from chalicelib.core.errors import errors, errors_details
-from chalicelib.core.sessions import sessions, sessions_notes, sessions_replay, sessions_favorite, sessions_viewed, \
-    sessions_assignments, unprocessed_sessions, sessions_search
+from chalicelib.core import scope
 from chalicelib.core import tenants, users, projects, license
 from chalicelib.core import webhook
 from chalicelib.core.collaborations.collaboration_slack import Slack
+from chalicelib.core.errors import errors, errors_details
+from chalicelib.core.metrics import heatmaps
+from chalicelib.core.sessions import sessions, sessions_notes, sessions_replay, sessions_favorite, sessions_viewed, \
+    sessions_assignments, unprocessed_sessions, sessions_search
 from chalicelib.utils import captcha, smtp
+from chalicelib.utils import contextual_validators
 from chalicelib.utils import helper
 from chalicelib.utils.TimeUTC import TimeUTC
 from or_dependencies import OR_context, OR_role
 from routers.base import get_routers
 from routers.subs import spot
-from chalicelib.utils import contextual_validators
 
 logger = logging.getLogger(__name__)
 public_app, app, app_apikey = get_routers()
 
-COOKIE_PATH = "/api/refresh"
+if config("LOCAL_DEV", cast=bool, default=False):
+    COOKIE_PATH = "/refresh"
+else:
+    COOKIE_PATH = "/api/refresh"
 
 
 @public_app.get('/signup', tags=['signup'])
@@ -68,18 +71,11 @@ def __process_authentication_response(response: JSONResponse, data: dict) -> dic
                         max_age=refresh_token_max_age, secure=True, httponly=True)
     response.set_cookie(key="spotRefreshToken", value=spot_refresh_token, path=spot.COOKIE_PATH,
                         max_age=spot_refresh_token_max_age, secure=True, httponly=True)
-    logger.warning("cookie refreshToken: %s", refresh_token)
-    logger.warning("cookie spotRefreshToken: %s", spot_refresh_token)
     return data
 
 
 @public_app.post('/login', tags=["authentication"])
 def login_user(response: JSONResponse, data: schemas.UserLoginSchema = Body(...)):
-    # if data.email != 'tahay@asayer.io':
-    #     raise HTTPException(
-    #         status_code=status.HTTP_401_UNAUTHORIZED,
-    #         detail="Enforced testing mode is active."
-    #     )
     if helper.allow_captcha() and not captcha.is_valid(data.g_recaptcha_response):
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,

api/routers/subs/metrics.py

@@ -9,172 +9,244 @@ from routers.base import get_routers
 public_app, app, app_apikey = get_routers()
 
 
-@app.post('/{projectId}/dashboards', tags=["dashboard"])
+@app.post("/{projectId}/dashboards", tags=["dashboard"])
 def create_dashboards(projectId: int, data: schemas.CreateDashboardSchema = Body(...),
                       context: schemas.CurrentContext = Depends(OR_context)):
-    return dashboards.create_dashboard(project_id=projectId, user_id=context.user_id, data=data)
+    return dashboards.create_dashboard(
+        project_id=projectId, user_id=context.user_id, data=data
+    )
 
 
-@app.get('/{projectId}/dashboards', tags=["dashboard"])
+@app.get("/{projectId}/dashboards", tags=["dashboard"])
 def get_dashboards(projectId: int, context: schemas.CurrentContext = Depends(OR_context)):
-    return {"data": dashboards.get_dashboards(project_id=projectId, user_id=context.user_id)}
+    return {
+        "data": dashboards.get_dashboards(project_id=projectId, user_id=context.user_id)
+    }
 
 
-@app.get('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"])
+@app.get("/{projectId}/dashboards/{dashboardId}", tags=["dashboard"])
 def get_dashboard(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)):
-    data = dashboards.get_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId)
+    data = dashboards.get_dashboard(
+        project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId
+    )
     if data is None:
         return {"errors": ["dashboard not found"]}
     return {"data": data}
 
 
-@app.put('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"])
+@app.put("/{projectId}/dashboards/{dashboardId}", tags=["dashboard"])
 def update_dashboard(projectId: int, dashboardId: int, data: schemas.EditDashboardSchema = Body(...),
                      context: schemas.CurrentContext = Depends(OR_context)):
-    return {"data": dashboards.update_dashboard(project_id=projectId, user_id=context.user_id,
-                                                dashboard_id=dashboardId, data=data)}
+    return {
+        "data": dashboards.update_dashboard(
+            project_id=projectId,
+            user_id=context.user_id,
+            dashboard_id=dashboardId,
+            data=data,
+        )
+    }
 
 
-@app.delete('/{projectId}/dashboards/{dashboardId}', tags=["dashboard"])
+@app.delete("/{projectId}/dashboards/{dashboardId}", tags=["dashboard"])
 def delete_dashboard(projectId: int, dashboardId: int, _=Body(None),
                      context: schemas.CurrentContext = Depends(OR_context)):
-    return dashboards.delete_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId)
+    return dashboards.delete_dashboard(
+        project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId
+    )
 
 
-@app.get('/{projectId}/dashboards/{dashboardId}/pin', tags=["dashboard"])
+@app.get("/{projectId}/dashboards/{dashboardId}/pin", tags=["dashboard"])
 def pin_dashboard(projectId: int, dashboardId: int, context: schemas.CurrentContext = Depends(OR_context)):
-    return {"data": dashboards.pin_dashboard(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId)}
+    return {
+        "data": dashboards.pin_dashboard(
+            project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId
+        )
+    }
 
 
-@app.post('/{projectId}/dashboards/{dashboardId}/cards', tags=["cards"])
-def add_card_to_dashboard(projectId: int, dashboardId: int,
-                          data: schemas.AddWidgetToDashboardPayloadSchema = Body(...),
+@app.post("/{projectId}/dashboards/{dashboardId}/cards", tags=["cards"])
+def add_card_to_dashboard(projectId: int, dashboardId: int, data: schemas.AddWidgetToDashboardPayloadSchema = Body(...),
                           context: schemas.CurrentContext = Depends(OR_context)):
-    return {"data": dashboards.add_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId,
-                                          data=data)}
+    return {
+        "data": dashboards.add_widget(
+            project_id=projectId,
+            user_id=context.user_id,
+            dashboard_id=dashboardId,
+            data=data,
+        )
+    }
 
 
-@app.post('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard"])
+@app.post("/{projectId}/dashboards/{dashboardId}/metrics", tags=["dashboard"])
 # @app.put('/{projectId}/dashboards/{dashboardId}/metrics', tags=["dashboard"])
-def create_metric_and_add_to_dashboard(projectId: int, dashboardId: int,
-                                       data: schemas.CardSchema = Body(...),
+def create_metric_and_add_to_dashboard(projectId: int, dashboardId: int, data: schemas.CardSchema = Body(...),
                                        context: schemas.CurrentContext = Depends(OR_context)):
-    return {"data": dashboards.create_metric_add_widget(project=context.project, user_id=context.user_id,
-                                                        dashboard_id=dashboardId, data=data)}
+    return {
+        "data": dashboards.create_metric_add_widget(
+            project=context.project,
+            user_id=context.user_id,
+            dashboard_id=dashboardId,
+            data=data,
+        )
+    }
 
 
-@app.put('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"])
+@app.put("/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}", tags=["dashboard"])
 def update_widget_in_dashboard(projectId: int, dashboardId: int, widgetId: int,
                                data: schemas.UpdateWidgetPayloadSchema = Body(...),
                                context: schemas.CurrentContext = Depends(OR_context)):
-    return dashboards.update_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId,
-                                    widget_id=widgetId, data=data)
+    return dashboards.update_widget(
+        project_id=projectId,
+        user_id=context.user_id,
+        dashboard_id=dashboardId,
+        widget_id=widgetId,
+        data=data,
+    )
 
 
-@app.delete('/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}', tags=["dashboard"])
+@app.delete("/{projectId}/dashboards/{dashboardId}/widgets/{widgetId}", tags=["dashboard"])
 def remove_widget_from_dashboard(projectId: int, dashboardId: int, widgetId: int, _=Body(None),
                                  context: schemas.CurrentContext = Depends(OR_context)):
-    return dashboards.remove_widget(project_id=projectId, user_id=context.user_id, dashboard_id=dashboardId,
-                                    widget_id=widgetId)
+    return dashboards.remove_widget(
+        project_id=projectId,
+        user_id=context.user_id,
+        dashboard_id=dashboardId,
+        widget_id=widgetId,
+    )
 
 
-@app.post('/{projectId}/cards/try', tags=["cards"])
+@app.post("/{projectId}/cards/try", tags=["cards"])
 def try_card(projectId: int, data: schemas.CardSchema = Body(...),
              context: schemas.CurrentContext = Depends(OR_context)):
-    return {"data": custom_metrics.get_chart(project=context.project, data=data, user_id=context.user_id)}
+    return {
+        "data": custom_metrics.get_chart(
+            project=context.project, data=data, user_id=context.user_id
+        )
+    }
 
 
-@app.post('/{projectId}/cards/try/sessions', tags=["cards"])
+@app.post("/{projectId}/cards/try/sessions", tags=["cards"])
 def try_card_sessions(projectId: int, data: schemas.CardSessionsSchema = Body(...),
                       context: schemas.CurrentContext = Depends(OR_context)):
-    data = custom_metrics.get_sessions(project=context.project, user_id=context.user_id, data=data)
+    data = custom_metrics.get_sessions(
+        project=context.project, user_id=context.user_id, data=data
+    )
     return {"data": data}
 
 
-@app.post('/{projectId}/cards/try/issues', tags=["cards"])
+@app.post("/{projectId}/cards/try/issues", tags=["cards"])
 def try_card_issues(projectId: int, data: schemas.CardSchema = Body(...),
                     context: schemas.CurrentContext = Depends(OR_context)):
-    return {"data": custom_metrics.get_issues(project=context.project, user_id=context.user_id, data=data)}
+    return {
+        "data": custom_metrics.get_issues(
+            project=context.project, user_id=context.user_id, data=data
+        )
+    }
 
 
-@app.get('/{projectId}/cards', tags=["cards"])
+@app.get("/{projectId}/cards", tags=["cards"])
 def get_cards(projectId: int, context: schemas.CurrentContext = Depends(OR_context)):
-    return {"data": custom_metrics.get_all(project_id=projectId, user_id=context.user_id)}
+    return {
+        "data": custom_metrics.get_all(project_id=projectId, user_id=context.user_id)
+    }
 
 
-@app.post('/{projectId}/cards', tags=["cards"])
+@app.post("/{projectId}/cards", tags=["cards"])
 def create_card(projectId: int, data: schemas.CardSchema = Body(...),
                 context: schemas.CurrentContext = Depends(OR_context)):
-    return custom_metrics.create_card(project=context.project, user_id=context.user_id, data=data)
+    return custom_metrics.create_card(
+        project=context.project, user_id=context.user_id, data=data
+    )
 
 
-@app.post('/{projectId}/cards/search', tags=["cards"])
-def search_cards(projectId: int, data: schemas.SearchCardsSchema = Body(...),
+@app.post("/{projectId}/cards/search", tags=["cards"])
+def search_cards(projectId: int, data: schemas.MetricSearchSchema = Body(...),
                  context: schemas.CurrentContext = Depends(OR_context)):
-    return {"data": custom_metrics.search_all(project_id=projectId, user_id=context.user_id, data=data)}
+    return {
+        "data": custom_metrics.search_metrics(
+            project_id=projectId, user_id=context.user_id, data=data
+        )
+    }
 
 
-@app.get('/{projectId}/cards/{metric_id}', tags=["cards"])
+@app.get("/{projectId}/cards/{metric_id}", tags=["cards"])
 def get_card(projectId: int, metric_id: Union[int, str], context: schemas.CurrentContext = Depends(OR_context)):
     if metric_id.isnumeric():
         metric_id = int(metric_id)
     else:
         return {"errors": ["invalid card_id"]}
-    data = custom_metrics.get_card(project_id=projectId, user_id=context.user_id, metric_id=metric_id)
+    data = custom_metrics.get_card(
+        project_id=projectId, user_id=context.user_id, metric_id=metric_id
+    )
     if data is None:
         return {"errors": ["card not found"]}
     return {"data": data}
 
 
-@app.post('/{projectId}/cards/{metric_id}/sessions', tags=["cards"])
-def get_card_sessions(projectId: int, metric_id: int,
-                      data: schemas.CardSessionsSchema = Body(...),
+@app.post("/{projectId}/cards/{metric_id}/sessions", tags=["cards"])
+def get_card_sessions(projectId: int, metric_id: int, data: schemas.CardSessionsSchema = Body(...),
                       context: schemas.CurrentContext = Depends(OR_context)):
-    data = custom_metrics.get_sessions_by_card_id(project=context.project, user_id=context.user_id, metric_id=metric_id,
-                                                  data=data)
+    data = custom_metrics.get_sessions_by_card_id(
+        project=context.project, user_id=context.user_id, metric_id=metric_id, data=data
+    )
     if data is None:
         return {"errors": ["custom metric not found"]}
     return {"data": data}
 
 
-@app.post('/{projectId}/cards/{metric_id}/issues/{issueId}/sessions', tags=["dashboard"])
+@app.post("/{projectId}/cards/{metric_id}/issues/{issueId}/sessions", tags=["dashboard"])
 def get_metric_funnel_issue_sessions(projectId: int, metric_id: int, issueId: str,
                                      data: schemas.CardSessionsSchema = Body(...),
                                      context: schemas.CurrentContext = Depends(OR_context)):
-    data = custom_metrics.get_funnel_sessions_by_issue(project_id=projectId, user_id=context.user_id,
-                                                       metric_id=metric_id, issue_id=issueId, data=data)
+    data = custom_metrics.get_funnel_sessions_by_issue(
+        project_id=projectId,
+        user_id=context.user_id,
+        metric_id=metric_id,
+        issue_id=issueId,
+        data=data,
+    )
     if data is None:
         return {"errors": ["custom metric not found"]}
     return {"data": data}
 
 
-@app.post('/{projectId}/cards/{metric_id}/chart', tags=["card"])
+@app.post("/{projectId}/cards/{metric_id}/chart", tags=["card"])
 def get_card_chart(projectId: int, metric_id: int, data: schemas.CardSessionsSchema = Body(...),
                    context: schemas.CurrentContext = Depends(OR_context)):
-    data = custom_metrics.make_chart_from_card(project=context.project, user_id=context.user_id, metric_id=metric_id,
-                                               data=data)
+    data = custom_metrics.make_chart_from_card(
+        project=context.project, user_id=context.user_id, metric_id=metric_id, data=data
+    )
     return {"data": data}
 
 
-@app.post('/{projectId}/cards/{metric_id}', tags=["dashboard"])
+@app.post("/{projectId}/cards/{metric_id}", tags=["dashboard"])
 def update_card(projectId: int, metric_id: int, data: schemas.CardSchema = Body(...),
                 context: schemas.CurrentContext = Depends(OR_context)):
-    data = custom_metrics.update_card(project_id=projectId, user_id=context.user_id, metric_id=metric_id, data=data)
+    data = custom_metrics.update_card(
+        project_id=projectId, user_id=context.user_id, metric_id=metric_id, data=data
+    )
     if data is None:
         return {"errors": ["custom metric not found"]}
     return {"data": data}
 
 
-@app.post('/{projectId}/cards/{metric_id}/status', tags=["dashboard"])
-def update_card_state(projectId: int, metric_id: int,
-                      data: schemas.UpdateCardStatusSchema = Body(...),
+@app.post("/{projectId}/cards/{metric_id}/status", tags=["dashboard"])
+def update_card_state(projectId: int, metric_id: int, data: schemas.UpdateCardStatusSchema = Body(...),
                       context: schemas.CurrentContext = Depends(OR_context)):
     return {
-        "data": custom_metrics.change_state(project_id=projectId, user_id=context.user_id, metric_id=metric_id,
-                                            status=data.active)}
+        "data": custom_metrics.change_state(
+            project_id=projectId,
+            user_id=context.user_id,
+            metric_id=metric_id,
+            status=data.active,
+        )
+    }
 
 
-@app.delete('/{projectId}/cards/{metric_id}', tags=["dashboard"])
-def delete_card(projectId: int, metric_id: int, _=Body(None),
-                context: schemas.CurrentContext = Depends(OR_context)):
-    return {"data": custom_metrics.delete_card(project_id=projectId, user_id=context.user_id, metric_id=metric_id)}
+@app.delete("/{projectId}/cards/{metric_id}", tags=["dashboard"])
+def delete_card(projectId: int, metric_id: int, _=Body(None), context: schemas.CurrentContext = Depends(OR_context)):
+    return {
+        "data": custom_metrics.delete_card(
+            project_id=projectId, user_id=context.user_id, metric_id=metric_id
+        )
+    }

api/routers/subs/spot.py

@@ -1,3 +1,4 @@
+from decouple import config
 from fastapi import Depends
 from starlette.responses import JSONResponse, Response
 
@@ -8,7 +9,10 @@ from routers.base import get_routers
 
 public_app, app, app_apikey = get_routers(prefix="/spot", tags=["spot"])
 
-COOKIE_PATH = "/api/spot/refresh"
+if config("LOCAL_DEV", cast=bool, default=False):
+    COOKIE_PATH = "/spot/refresh"
+else:
+    COOKIE_PATH = "/api/spot/refresh"
 
 
 @app.get('/logout')

api/schemas/schemas.py

@@ -960,36 +960,6 @@ class CardSessionsSchema(_TimedSchema, _PaginatedSchema):
 
         return self
 
-    # We don't need this as the UI is expecting filters to override the full series' filters
-    # @model_validator(mode="after")
-    # def __merge_out_filters_with_series(self):
-    #     for f in self.filters:
-    #         for s in self.series:
-    #             found = False
-    #
-    #             if f.is_event:
-    #                 sub = s.filter.events
-    #             else:
-    #                 sub = s.filter.filters
-    #
-    #             for e in sub:
-    #                 if f.type == e.type and f.operator == e.operator:
-    #                     found = True
-    #                     if f.is_event:
-    #                         # If extra event: append value
-    #                         for v in f.value:
-    #                             if v not in e.value:
-    #                                 e.value.append(v)
-    #                     else:
-    #                         # If extra filter: override value
-    #                         e.value = f.value
-    #             if not found:
-    #                 sub.append(f)
-    #
-    #     self.filters = []
-    #
-    #     return self
-
     # UI is expecting filters to override the full series' filters
     @model_validator(mode="after")
     def __override_series_filters_with_outer_filters(self):
@@ -1060,6 +1030,16 @@ class CardTable(__CardSchema):
             values["metricValue"] = []
         return values
 
+    @model_validator(mode="after")
+    def __enforce_AND_operator(self):
+        self.metric_of = MetricOfTable(self.metric_of)
+        if self.metric_of in (MetricOfTable.VISITED_URL, MetricOfTable.FETCH, \
+                              MetricOfTable.VISITED_URL.value, MetricOfTable.FETCH.value):
+            for s in self.series:
+                if s.filter is not None:
+                    s.filter.events_order = SearchEventOrder.AND
+        return self
+
     @model_validator(mode="after")
     def __transform(self):
         self.metric_of = MetricOfTable(self.metric_of)
@@ -1135,7 +1115,7 @@ class CardPathAnalysis(__CardSchema):
     view_type: MetricOtherViewType = Field(...)
     metric_value: List[ProductAnalyticsSelectedEventType] = Field(default_factory=list)
     density: int = Field(default=4, ge=2, le=10)
-    rows: int = Field(default=3, ge=1, le=10)
+    rows: int = Field(default=5, ge=1, le=10)
 
     start_type: Literal["start", "end"] = Field(default="start")
     start_point: List[PathAnalysisSubFilterSchema] = Field(default_factory=list)
@@ -1368,6 +1348,42 @@ class SearchCardsSchema(_PaginatedSchema):
     query: Optional[str] = Field(default=None)
 
 
+class MetricSortColumnType(str, Enum):
+    NAME = "name"
+    METRIC_TYPE = "metric_type"
+    METRIC_OF = "metric_of"
+    IS_PUBLIC = "is_public"
+    CREATED_AT = "created_at"
+    EDITED_AT = "edited_at"
+
+
+class MetricFilterColumnType(str, Enum):
+    NAME = "name"
+    METRIC_TYPE = "metric_type"
+    METRIC_OF = "metric_of"
+    IS_PUBLIC = "is_public"
+    USER_ID = "user_id"
+    CREATED_AT = "created_at"
+    EDITED_AT = "edited_at"
+
+
+class MetricListSort(BaseModel):
+    field: Optional[str] = Field(default=None)
+    order: Optional[str] = Field(default=SortOrderType.DESC)
+
+
+class MetricFilter(BaseModel):
+    type: Optional[str] = Field(default=None)
+    query: Optional[str] = Field(default=None)
+
+
+class MetricSearchSchema(_PaginatedSchema):
+    filter: Optional[MetricFilter] = Field(default=None)
+    sort: Optional[MetricListSort] = Field(default=MetricListSort())
+    shared_only: bool = Field(default=False)
+    mine_only: bool = Field(default=False)
+
+
 class _HeatMapSearchEventRaw(SessionSearchEventSchema2):
     type: Literal[EventType.LOCATION] = Field(...)
 

assist/utils/assistHelper.js

@@ -19,14 +19,16 @@ const EVENTS_DEFINITION = {
     }
 };
 EVENTS_DEFINITION.emit = {
-    NEW_AGENT: "NEW_AGENT",
-    NO_AGENTS: "NO_AGENT",
-    AGENT_DISCONNECT: "AGENT_DISCONNECTED",
-    AGENTS_CONNECTED: "AGENTS_CONNECTED",
-    NO_SESSIONS: "SESSION_DISCONNECTED",
-    SESSION_ALREADY_CONNECTED: "SESSION_ALREADY_CONNECTED",
-    SESSION_RECONNECTED: "SESSION_RECONNECTED",
-    UPDATE_EVENT: EVENTS_DEFINITION.listen.UPDATE_EVENT
+  NEW_AGENT: "NEW_AGENT",
+  NO_AGENTS: "NO_AGENT",
+  AGENT_DISCONNECT: "AGENT_DISCONNECTED",
+  AGENTS_CONNECTED: "AGENTS_CONNECTED",
+  AGENTS_INFO_CONNECTED: "AGENTS_INFO_CONNECTED",
+  NO_SESSIONS: "SESSION_DISCONNECTED",
+  SESSION_ALREADY_CONNECTED: "SESSION_ALREADY_CONNECTED",
+  SESSION_RECONNECTED: "SESSION_RECONNECTED",
+  UPDATE_EVENT: EVENTS_DEFINITION.listen.UPDATE_EVENT,
+  WEBRTC_CONFIG: "WEBRTC_CONFIG",
 };
 
 const BASE_sessionInfo = {

assist/utils/httpHandlers.js

@@ -27,9 +27,14 @@ const respond = function (req, res, data) {
         res.setHeader('Content-Type', 'application/json');
         res.end(JSON.stringify(result));
     } else {
-        res.cork(() => {
-            res.writeStatus('200 OK').writeHeader('Content-Type', 'application/json').end(JSON.stringify(result));
-        });
+        if (!res.aborted) {
+            res.cork(() => {
+                res.writeStatus('200 OK').writeHeader('Content-Type', 'application/json').end(JSON.stringify(result));
+            });
+        } else {
+            logger.debug("response aborted");
+            return;
+        }
     }
     const duration = performance.now() - req.startTs;
     IncreaseTotalRequests();

assist/utils/socketHandlers.js

@@ -42,7 +42,7 @@ const findSessionSocketId = async (io, roomId, tabId) => {
 };
 
 async function getRoomData(io, roomID) {
-    let tabsCount = 0, agentsCount = 0, tabIDs = [], agentIDs = [];
+    let tabsCount = 0, agentsCount = 0, tabIDs = [], agentIDs = [], config = null, agentInfos = [];
     const connected_sockets = await io.in(roomID).fetchSockets();
     if (connected_sockets.length > 0) {
         for (let socket of connected_sockets) {
@@ -52,13 +52,19 @@ async function getRoomData(io, roomID) {
             } else {
                 agentsCount++;
                 agentIDs.push(socket.id);
+                agentInfos.push({ ...socket.handshake.query.agentInfo, socketId: socket.id });
+                if (socket.handshake.query.config !== undefined) {
+                    config = socket.handshake.query.config;
+                }
             }
         }
     } else {
         tabsCount = -1;
         agentsCount = -1;
+        agentInfos = [];
+        agentIDs = [];
     }
-    return {tabsCount, agentsCount, tabIDs, agentIDs};
+    return {tabsCount, agentsCount, tabIDs, agentIDs, config, agentInfos};
 }
 
 function processNewSocket(socket) {
@@ -78,7 +84,7 @@ async function onConnect(socket) {
     IncreaseOnlineConnections(socket.handshake.query.identity);
 
     const io = getServer();
-    const {tabsCount, agentsCount, tabIDs, agentIDs} = await getRoomData(io, socket.handshake.query.roomId);
+    const {tabsCount, agentsCount, tabIDs, agentInfos, agentIDs, config} = await getRoomData(io, socket.handshake.query.roomId);
 
     if (socket.handshake.query.identity === IDENTITIES.session) {
         // Check if session with the same tabID already connected, if so, refuse new connexion
@@ -100,7 +106,9 @@ async function onConnect(socket) {
         // Inform all connected agents about reconnected session
         if (agentsCount > 0) {
             logger.debug(`notifying new session about agent-existence`);
+            io.to(socket.id).emit(EVENTS_DEFINITION.emit.WEBRTC_CONFIG, config);
             io.to(socket.id).emit(EVENTS_DEFINITION.emit.AGENTS_CONNECTED, agentIDs);
+            io.to(socket.id).emit(EVENTS_DEFINITION.emit.AGENTS_INFO_CONNECTED, agentInfos);
             socket.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.SESSION_RECONNECTED, socket.id);
         }
     } else if (tabsCount <= 0) {
@@ -118,7 +126,8 @@ async function onConnect(socket) {
             // Stats
             startAssist(socket, socket.handshake.query.agentID);
         }
-        socket.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.NEW_AGENT, socket.id, socket.handshake.query.agentInfo);
+        io.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.WEBRTC_CONFIG, socket.handshake.query.config);
+        socket.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.NEW_AGENT, socket.id, { ...socket.handshake.query.agentInfo });
     }
 
     // Set disconnect handler

backend/cmd/analytics/main.go

@@ -8,8 +8,7 @@ import (
 	"openreplay/backend/pkg/db/postgres/pool"
 	"openreplay/backend/pkg/logger"
 	"openreplay/backend/pkg/metrics"
-	analyticsMetrics "openreplay/backend/pkg/metrics/analytics"
-	databaseMetrics "openreplay/backend/pkg/metrics/database"
+	"openreplay/backend/pkg/metrics/database"
 	"openreplay/backend/pkg/metrics/web"
 	"openreplay/backend/pkg/server"
 	"openreplay/backend/pkg/server/api"
@@ -19,16 +18,18 @@ func main() {
 	ctx := context.Background()
 	log := logger.New()
 	cfg := analyticsConfig.New(log)
+	// Observability
 	webMetrics := web.New("analytics")
-	metrics.New(log, append(webMetrics.List(), append(analyticsMetrics.List(), databaseMetrics.List()...)...))
+	dbMetrics := database.New("analytics")
+	metrics.New(log, append(webMetrics.List(), dbMetrics.List()...))
 
-	pgConn, err := pool.New(cfg.Postgres.String())
+	pgConn, err := pool.New(dbMetrics, cfg.Postgres.String())
 	if err != nil {
 		log.Fatal(ctx, "can't init postgres connection: %s", err)
 	}
 	defer pgConn.Close()
 
-	builder, err := analytics.NewServiceBuilder(log, cfg, webMetrics, pgConn)
+	builder, err := analytics.NewServiceBuilder(log, cfg, webMetrics, dbMetrics, pgConn)
 	if err != nil {
 		log.Fatal(ctx, "can't init services: %s", err)
 	}

backend/cmd/assets/main.go

@@ -22,13 +22,15 @@ func main() {
 	ctx := context.Background()
 	log := logger.New()
 	cfg := config.New(log)
-	metrics.New(log, assetsMetrics.List())
+	// Observability
+	assetMetrics := assetsMetrics.New("assets")
+	metrics.New(log, assetMetrics.List())
 
 	objStore, err := store.NewStore(&cfg.ObjectsConfig)
 	if err != nil {
 		log.Fatal(ctx, "can't init object storage: %s", err)
 	}
-	cacher, err := cacher.NewCacher(cfg, objStore)
+	cacher, err := cacher.NewCacher(cfg, objStore, assetMetrics)
 	if err != nil {
 		log.Fatal(ctx, "can't init cacher: %s", err)
 	}
@@ -37,7 +39,7 @@ func main() {
 		switch m := msg.(type) {
 		case *messages.AssetCache:
 			cacher.CacheURL(m.SessionID(), m.URL)
-			assetsMetrics.IncreaseProcessesSessions()
+			assetMetrics.IncreaseProcessesSessions()
 		case *messages.JSException:
 			sourceList, err := assets.ExtractJSExceptionSources(&m.Payload)
 			if err != nil {

backend/cmd/canvases/main.go

@@ -22,6 +22,7 @@ func main() {
 	ctx := context.Background()
 	log := logger.New()
 	cfg := config.New(log)
+	// Observability
 	canvasMetrics := canvasesMetrics.New("canvases")
 	metrics.New(log, canvasMetrics.List())
 

backend/cmd/db/main.go

@@ -14,7 +14,7 @@ import (
 	"openreplay/backend/pkg/memory"
 	"openreplay/backend/pkg/messages"
 	"openreplay/backend/pkg/metrics"
-	databaseMetrics "openreplay/backend/pkg/metrics/database"
+	"openreplay/backend/pkg/metrics/database"
 	"openreplay/backend/pkg/projects"
 	"openreplay/backend/pkg/queue"
 	"openreplay/backend/pkg/sessions"
@@ -26,22 +26,24 @@ func main() {
 	ctx := context.Background()
 	log := logger.New()
 	cfg := config.New(log)
-	metrics.New(log, databaseMetrics.List())
+	// Observability
+	dbMetric := database.New("db")
+	metrics.New(log, dbMetric.List())
 
-	pgConn, err := pool.New(cfg.Postgres.String())
+	pgConn, err := pool.New(dbMetric, cfg.Postgres.String())
 	if err != nil {
 		log.Fatal(ctx, "can't init postgres connection: %s", err)
 	}
 	defer pgConn.Close()
 
-	chConn := clickhouse.NewConnector(cfg.Clickhouse)
+	chConn := clickhouse.NewConnector(cfg.Clickhouse, dbMetric)
 	if err := chConn.Prepare(); err != nil {
 		log.Fatal(ctx, "can't prepare clickhouse: %s", err)
 	}
 	defer chConn.Stop()
 
 	// Init db proxy module (postgres + clickhouse + batches)
-	dbProxy := postgres.NewConn(log, pgConn, chConn)
+	dbProxy := postgres.NewConn(log, pgConn, chConn, dbMetric)
 	defer dbProxy.Close()
 
 	// Init redis connection
@@ -51,8 +53,8 @@ func main() {
 	}
 	defer redisClient.Close()
 
-	projManager := projects.New(log, pgConn, redisClient)
-	sessManager := sessions.New(log, pgConn, projManager, redisClient)
+	projManager := projects.New(log, pgConn, redisClient, dbMetric)
+	sessManager := sessions.New(log, pgConn, projManager, redisClient, dbMetric)
 	tagsManager := tags.New(log, pgConn)
 
 	// Init data saver

backend/cmd/ender/main.go

@@ -19,7 +19,7 @@ import (
 	"openreplay/backend/pkg/memory"
 	"openreplay/backend/pkg/messages"
 	"openreplay/backend/pkg/metrics"
-	databaseMetrics "openreplay/backend/pkg/metrics/database"
+	"openreplay/backend/pkg/metrics/database"
 	enderMetrics "openreplay/backend/pkg/metrics/ender"
 	"openreplay/backend/pkg/projects"
 	"openreplay/backend/pkg/queue"
@@ -31,9 +31,12 @@ func main() {
 	ctx := context.Background()
 	log := logger.New()
 	cfg := ender.New(log)
-	metrics.New(log, append(enderMetrics.List(), databaseMetrics.List()...))
+	// Observability
+	dbMetric := database.New("ender")
+	enderMetric := enderMetrics.New("ender")
+	metrics.New(log, append(enderMetric.List(), dbMetric.List()...))
 
-	pgConn, err := pool.New(cfg.Postgres.String())
+	pgConn, err := pool.New(dbMetric, cfg.Postgres.String())
 	if err != nil {
 		log.Fatal(ctx, "can't init postgres connection: %s", err)
 	}
@@ -45,10 +48,10 @@ func main() {
 	}
 	defer redisClient.Close()
 
-	projManager := projects.New(log, pgConn, redisClient)
-	sessManager := sessions.New(log, pgConn, projManager, redisClient)
+	projManager := projects.New(log, pgConn, redisClient, dbMetric)
+	sessManager := sessions.New(log, pgConn, projManager, redisClient, dbMetric)
 
-	sessionEndGenerator, err := sessionender.New(intervals.EVENTS_SESSION_END_TIMEOUT, cfg.PartitionsNumber)
+	sessionEndGenerator, err := sessionender.New(enderMetric, intervals.EVENTS_SESSION_END_TIMEOUT, cfg.PartitionsNumber)
 	if err != nil {
 		log.Fatal(ctx, "can't init ender service: %s", err)
 	}

backend/cmd/heuristics/main.go

@@ -23,7 +23,9 @@ func main() {
 	ctx := context.Background()
 	log := logger.New()
 	cfg := config.New(log)
-	metrics.New(log, heuristicsMetrics.List())
+	// Observability
+	heuristicsMetric := heuristicsMetrics.New("heuristics")
+	metrics.New(log, heuristicsMetric.List())
 
 	// HandlersFabric returns the list of message handlers we want to be applied to each incoming message.
 	handlersFabric := func() []handlers.MessageProcessor {
@@ -62,7 +64,7 @@ func main() {
 	}
 
 	// Run service and wait for TERM signal
-	service := heuristics.New(log, cfg, producer, consumer, eventBuilder, memoryManager)
+	service := heuristics.New(log, cfg, producer, consumer, eventBuilder, memoryManager, heuristicsMetric)
 	log.Info(ctx, "Heuristics service started")
 	terminator.Wait(log, service)
 }

backend/cmd/http/main.go

@@ -9,7 +9,7 @@ import (
 	"openreplay/backend/pkg/db/redis"
 	"openreplay/backend/pkg/logger"
 	"openreplay/backend/pkg/metrics"
-	databaseMetrics "openreplay/backend/pkg/metrics/database"
+	"openreplay/backend/pkg/metrics/database"
 	"openreplay/backend/pkg/metrics/web"
 	"openreplay/backend/pkg/queue"
 	"openreplay/backend/pkg/server"
@@ -20,13 +20,15 @@ func main() {
 	ctx := context.Background()
 	log := logger.New()
 	cfg := http.New(log)
+	// Observability
 	webMetrics := web.New("http")
-	metrics.New(log, append(webMetrics.List(), databaseMetrics.List()...))
+	dbMetric := database.New("http")
+	metrics.New(log, append(webMetrics.List(), dbMetric.List()...))
 
 	producer := queue.NewProducer(cfg.MessageSizeLimit, true)
 	defer producer.Close(15000)
 
-	pgConn, err := pool.New(cfg.Postgres.String())
+	pgConn, err := pool.New(dbMetric, cfg.Postgres.String())
 	if err != nil {
 		log.Fatal(ctx, "can't init postgres connection: %s", err)
 	}
@@ -38,7 +40,7 @@ func main() {
 	}
 	defer redisClient.Close()
 
-	builder, err := services.New(log, cfg, webMetrics, producer, pgConn, redisClient)
+	builder, err := services.New(log, cfg, webMetrics, dbMetric, producer, pgConn, redisClient)
 	if err != nil {
 		log.Fatal(ctx, "failed while creating services: %s", err)
 	}

backend/cmd/images/main.go

@@ -14,7 +14,7 @@ import (
 	"openreplay/backend/pkg/logger"
 	"openreplay/backend/pkg/messages"
 	"openreplay/backend/pkg/metrics"
-	storageMetrics "openreplay/backend/pkg/metrics/images"
+	imagesMetrics "openreplay/backend/pkg/metrics/images"
 	"openreplay/backend/pkg/objectstorage/store"
 	"openreplay/backend/pkg/queue"
 )
@@ -23,14 +23,16 @@ func main() {
 	ctx := context.Background()
 	log := logger.New()
 	cfg := config.New(log)
-	metrics.New(log, storageMetrics.List())
+	// Observability
+	imageMetrics := imagesMetrics.New("images")
+	metrics.New(log, imageMetrics.List())
 
 	objStore, err := store.NewStore(&cfg.ObjectsConfig)
 	if err != nil {
 		log.Fatal(ctx, "can't init object storage: %s", err)
 	}
 
-	srv, err := images.New(cfg, log, objStore)
+	srv, err := images.New(cfg, log, objStore, imageMetrics)
 	if err != nil {
 		log.Fatal(ctx, "can't init images service: %s", err)
 	}

backend/cmd/integrations/main.go

@@ -18,16 +18,18 @@ func main() {
 	ctx := context.Background()
 	log := logger.New()
 	cfg := config.New(log)
+	// Observability
 	webMetrics := web.New("integrations")
-	metrics.New(log, append(webMetrics.List(), database.List()...))
+	dbMetric := database.New("integrations")
+	metrics.New(log, append(webMetrics.List(), dbMetric.List()...))
 
-	pgConn, err := pool.New(cfg.Postgres.String())
+	pgConn, err := pool.New(dbMetric, cfg.Postgres.String())
 	if err != nil {
 		log.Fatal(ctx, "can't init postgres connection: %s", err)
 	}
 	defer pgConn.Close()
 
-	builder, err := integrations.NewServiceBuilder(log, cfg, webMetrics, pgConn)
+	builder, err := integrations.NewServiceBuilder(log, cfg, webMetrics, dbMetric, pgConn)
 	if err != nil {
 		log.Fatal(ctx, "can't init services: %s", err)
 	}

backend/cmd/sink/main.go

@@ -9,14 +9,14 @@ import (
 	"syscall"
 	"time"
 
-	"openreplay/backend/internal/config/sink"
+	config "openreplay/backend/internal/config/sink"
 	"openreplay/backend/internal/sink/assetscache"
 	"openreplay/backend/internal/sink/sessionwriter"
 	"openreplay/backend/internal/storage"
 	"openreplay/backend/pkg/logger"
 	"openreplay/backend/pkg/messages"
 	"openreplay/backend/pkg/metrics"
-	sinkMetrics "openreplay/backend/pkg/metrics/sink"
+	"openreplay/backend/pkg/metrics/sink"
 	"openreplay/backend/pkg/queue"
 	"openreplay/backend/pkg/url/assets"
 )
@@ -24,7 +24,9 @@ import (
 func main() {
 	ctx := context.Background()
 	log := logger.New()
-	cfg := sink.New(log)
+	cfg := config.New(log)
+	// Observability
+	sinkMetrics := sink.New("sink")
 	metrics.New(log, sinkMetrics.List())
 
 	if _, err := os.Stat(cfg.FsDir); os.IsNotExist(err) {
@@ -39,7 +41,7 @@ func main() {
 	if err != nil {
 		log.Fatal(ctx, "can't init rewriter: %s", err)
 	}
-	assetMessageHandler := assetscache.New(log, cfg, rewriter, producer)
+	assetMessageHandler := assetscache.New(log, cfg, rewriter, producer, sinkMetrics)
 	counter := storage.NewLogCounter()
 
 	var (
@@ -191,7 +193,7 @@ func main() {
 			cfg.TopicRawWeb,
 			cfg.TopicRawMobile,
 		},
-		messages.NewSinkMessageIterator(log, msgHandler, nil, false),
+		messages.NewSinkMessageIterator(log, msgHandler, nil, false, sinkMetrics),
 		false,
 		cfg.MessageSizeLimit,
 	)

backend/cmd/spot/main.go

@@ -19,16 +19,20 @@ func main() {
 	ctx := context.Background()
 	log := logger.New()
 	cfg := spotConfig.New(log)
+	// Observability
 	webMetrics := web.New("spot")
-	metrics.New(log, append(webMetrics.List(), append(spotMetrics.List(), databaseMetrics.List()...)...))
+	spotMetric := spotMetrics.New("spot")
+	dbMetric := databaseMetrics.New("spot")
+	metrics.New(log, append(webMetrics.List(), append(spotMetric.List(), dbMetric.List()...)...))
 
-	pgConn, err := pool.New(cfg.Postgres.String())
+	pgConn, err := pool.New(dbMetric, cfg.Postgres.String())
 	if err != nil {
 		log.Fatal(ctx, "can't init postgres connection: %s", err)
 	}
 	defer pgConn.Close()
 
-	builder, err := spot.NewServiceBuilder(log, cfg, webMetrics, pgConn)
+	prefix := api.NoPrefix
+	builder, err := spot.NewServiceBuilder(log, cfg, webMetrics, spotMetric, dbMetric, pgConn, prefix)
 	if err != nil {
 		log.Fatal(ctx, "can't init services: %s", err)
 	}
@@ -37,7 +41,7 @@ func main() {
 	if err != nil {
 		log.Fatal(ctx, "failed while creating router: %s", err)
 	}
-	router.AddHandlers(api.NoPrefix, builder.SpotsAPI)
+	router.AddHandlers(prefix, builder.SpotsAPI)
 	router.AddMiddlewares(builder.Auth.Middleware, builder.RateLimiter.Middleware, builder.AuditTrail.Middleware)
 
 	server.Run(ctx, log, &cfg.HTTP, router)

backend/cmd/storage/main.go

@@ -23,13 +23,15 @@ func main() {
 	ctx := context.Background()
 	log := logger.New()
 	cfg := config.New(log)
-	metrics.New(log, storageMetrics.List())
+	// Observability
+	storageMetric := storageMetrics.New("storage")
+	metrics.New(log, storageMetric.List())
 
 	objStore, err := store.NewStore(&cfg.ObjectsConfig)
 	if err != nil {
 		log.Fatal(ctx, "can't init object storage: %s", err)
 	}
-	srv, err := storage.New(cfg, log, objStore)
+	srv, err := storage.New(cfg, log, objStore, storageMetric)
 	if err != nil {
 		log.Fatal(ctx, "can't init storage service: %s", err)
 	}

backend/internal/assets/cacher/cacher.go

@@ -27,6 +27,7 @@ type cacher struct {
 	objStorage     objectstorage.ObjectStorage // AWS Docs: "These clients are safe to use concurrently."
 	httpClient     *http.Client                // Docs: "Clients are safe for concurrent use by multiple goroutines."
 	rewriter       *assets.Rewriter            // Read only
+	metrics        metrics.Assets
 	Errors         chan error
 	sizeLimit      int
 	requestHeaders map[string]string
@@ -37,7 +38,7 @@ func (c *cacher) CanCache() bool {
 	return c.workers.CanAddTask()
 }
 
-func NewCacher(cfg *config.Config, store objectstorage.ObjectStorage) (*cacher, error) {
+func NewCacher(cfg *config.Config, store objectstorage.ObjectStorage, metrics metrics.Assets) (*cacher, error) {
 	switch {
 	case cfg == nil:
 		return nil, errors.New("config is nil")
@@ -93,6 +94,7 @@ func NewCacher(cfg *config.Config, store objectstorage.ObjectStorage) (*cacher,
 		Errors:         make(chan error),
 		sizeLimit:      cfg.AssetsSizeLimit,
 		requestHeaders: cfg.AssetsRequestHeaders,
+		metrics:        metrics,
 	}
 	c.workers = NewPool(64, c.CacheFile)
 	return c, nil
@@ -115,7 +117,7 @@ func (c *cacher) cacheURL(t *Task) {
 		c.Errors <- errors.Wrap(err, t.urlContext)
 		return
 	}
-	metrics.RecordDownloadDuration(float64(time.Now().Sub(start).Milliseconds()), res.StatusCode)
+	c.metrics.RecordDownloadDuration(float64(time.Now().Sub(start).Milliseconds()), res.StatusCode)
 	defer res.Body.Close()
 	if res.StatusCode >= 400 {
 		printErr := true
@@ -162,12 +164,12 @@ func (c *cacher) cacheURL(t *Task) {
 	start = time.Now()
 	err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, contentEncoding, objectstorage.NoCompression)
 	if err != nil {
-		metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), true)
+		c.metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), true)
 		c.Errors <- errors.Wrap(err, t.urlContext)
 		return
 	}
-	metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), false)
-	metrics.IncreaseSavedSessions()
+	c.metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), false)
+	c.metrics.IncreaseSavedSessions()
 
 	if isCSS {
 		if t.depth > 0 {

backend/internal/db/datasaver/saver.go

@@ -2,11 +2,12 @@ package datasaver
 
 import (
 	"context"
+	"encoding/json"
+	"openreplay/backend/pkg/db/types"
 
 	"openreplay/backend/internal/config/db"
 	"openreplay/backend/pkg/db/clickhouse"
 	"openreplay/backend/pkg/db/postgres"
-	"openreplay/backend/pkg/db/types"
 	"openreplay/backend/pkg/logger"
 	. "openreplay/backend/pkg/messages"
 	queue "openreplay/backend/pkg/queue/types"
@@ -50,10 +51,6 @@ func New(log logger.Logger, cfg *db.Config, pg *postgres.Conn, ch clickhouse.Con
 }
 
 func (s *saverImpl) Handle(msg Message) {
-	if msg.TypeID() == MsgCustomEvent {
-		defer s.Handle(types.WrapCustomEvent(msg.(*CustomEvent)))
-	}
-
 	var (
 		sessCtx = context.WithValue(context.Background(), "sessionID", msg.SessionID())
 		session *sessions.Session
@@ -69,6 +66,23 @@ func (s *saverImpl) Handle(msg Message) {
 		return
 	}
 
+	if msg.TypeID() == MsgCustomEvent {
+		m := msg.(*CustomEvent)
+		// Try to parse custom event payload to JSON and extract or_payload field
+		type CustomEventPayload struct {
+			CustomTimestamp uint64 `json:"or_timestamp"`
+		}
+		customPayload := &CustomEventPayload{}
+		if err := json.Unmarshal([]byte(m.Payload), customPayload); err == nil {
+			if customPayload.CustomTimestamp >= session.Timestamp {
+				s.log.Info(sessCtx, "custom event timestamp received: %v", m.Timestamp)
+				msg.Meta().Timestamp = customPayload.CustomTimestamp
+				s.log.Info(sessCtx, "custom event timestamp updated: %v", m.Timestamp)
+			}
+		}
+		defer s.Handle(types.WrapCustomEvent(m))
+	}
+
 	if IsMobileType(msg.TypeID()) {
 		if err := s.handleMobileMessage(sessCtx, session, msg); err != nil {
 			if !postgres.IsPkeyViolation(err) {

backend/internal/heuristics/service.go

@@ -11,7 +11,7 @@ import (
 	"openreplay/backend/pkg/logger"
 	"openreplay/backend/pkg/memory"
 	"openreplay/backend/pkg/messages"
-	metrics "openreplay/backend/pkg/metrics/heuristics"
+	heuristicMetrics "openreplay/backend/pkg/metrics/heuristics"
 	"openreplay/backend/pkg/queue/types"
 )
 
@@ -23,11 +23,12 @@ type heuristicsImpl struct {
 	consumer types.Consumer
 	events   builders.EventBuilder
 	mm       memory.Manager
+	metrics  heuristicMetrics.Heuristics
 	done     chan struct{}
 	finished chan struct{}
 }
 
-func New(log logger.Logger, cfg *heuristics.Config, p types.Producer, c types.Consumer, e builders.EventBuilder, mm memory.Manager) service.Interface {
+func New(log logger.Logger, cfg *heuristics.Config, p types.Producer, c types.Consumer, e builders.EventBuilder, mm memory.Manager, metrics heuristicMetrics.Heuristics) service.Interface {
 	s := &heuristicsImpl{
 		log:      log,
 		ctx:      context.Background(),
@@ -36,6 +37,7 @@ func New(log logger.Logger, cfg *heuristics.Config, p types.Producer, c types.Co
 		consumer: c,
 		events:   e,
 		mm:       mm,
+		metrics:  metrics,
 		done:     make(chan struct{}),
 		finished: make(chan struct{}),
 	}
@@ -51,7 +53,7 @@ func (h *heuristicsImpl) run() {
 			if err := h.producer.Produce(h.cfg.TopicAnalytics, evt.SessionID(), evt.Encode()); err != nil {
 				h.log.Error(h.ctx, "can't send new event to queue: %s", err)
 			} else {
-				metrics.IncreaseTotalEvents(messageTypeName(evt))
+				h.metrics.IncreaseTotalEvents(messageTypeName(evt))
 			}
 		case <-tick:
 			h.producer.Flush(h.cfg.ProducerTimeout)

backend/internal/http/services/services.go

@@ -12,6 +12,7 @@ import (
 	featureflagsAPI "openreplay/backend/pkg/featureflags/api"
 	"openreplay/backend/pkg/flakeid"
 	"openreplay/backend/pkg/logger"
+	"openreplay/backend/pkg/metrics/database"
 	"openreplay/backend/pkg/metrics/web"
 	"openreplay/backend/pkg/objectstorage/store"
 	"openreplay/backend/pkg/projects"
@@ -36,8 +37,8 @@ type ServicesBuilder struct {
 	UxTestsAPI      api.Handlers
 }
 
-func New(log logger.Logger, cfg *http.Config, metrics web.Web, producer types.Producer, pgconn pool.Pool, redis *redis.Client) (*ServicesBuilder, error) {
-	projs := projects.New(log, pgconn, redis)
+func New(log logger.Logger, cfg *http.Config, webMetrics web.Web, dbMetrics database.Database, producer types.Producer, pgconn pool.Pool, redis *redis.Client) (*ServicesBuilder, error) {
+	projs := projects.New(log, pgconn, redis, dbMetrics)
 	objStore, err := store.NewStore(&cfg.ObjectsConfig)
 	if err != nil {
 		return nil, err
@@ -53,11 +54,11 @@ func New(log logger.Logger, cfg *http.Config, metrics web.Web, producer types.Pr
 	tokenizer := token.NewTokenizer(cfg.TokenSecret)
 	conditions := conditions.New(pgconn)
 	flaker := flakeid.NewFlaker(cfg.WorkerID)
-	sessions := sessions.New(log, pgconn, projs, redis)
+	sessions := sessions.New(log, pgconn, projs, redis, dbMetrics)
 	featureFlags := featureflags.New(pgconn)
 	tags := tags.New(log, pgconn)
 	uxTesting := uxtesting.New(pgconn)
-	responser := api.NewResponser(metrics)
+	responser := api.NewResponser(webMetrics)
 	builder := &ServicesBuilder{}
 	if builder.WebAPI, err = websessions.NewHandlers(cfg, log, responser, producer, projs, sessions, uaModule, geoModule, tokenizer, conditions, flaker); err != nil {
 		return nil, err

backend/internal/images/service.go

@@ -15,6 +15,7 @@ import (
 
 	config "openreplay/backend/internal/config/images"
 	"openreplay/backend/pkg/logger"
+	"openreplay/backend/pkg/metrics/images"
 	"openreplay/backend/pkg/objectstorage"
 	"openreplay/backend/pkg/pool"
 )
@@ -38,9 +39,10 @@ type ImageStorage struct {
 	objStorage   objectstorage.ObjectStorage
 	saverPool    pool.WorkerPool
 	uploaderPool pool.WorkerPool
+	metrics      images.Images
 }
 
-func New(cfg *config.Config, log logger.Logger, objStorage objectstorage.ObjectStorage) (*ImageStorage, error) {
+func New(cfg *config.Config, log logger.Logger, objStorage objectstorage.ObjectStorage, metrics images.Images) (*ImageStorage, error) {
 	switch {
 	case cfg == nil:
 		return nil, fmt.Errorf("config is empty")
@@ -48,11 +50,14 @@ func New(cfg *config.Config, log logger.Logger, objStorage objectstorage.ObjectS
 		return nil, fmt.Errorf("logger is empty")
 	case objStorage == nil:
 		return nil, fmt.Errorf("objStorage is empty")
+	case metrics == nil:
+		return nil, fmt.Errorf("metrics is empty")
 	}
 	s := &ImageStorage{
 		cfg:        cfg,
 		log:        log,
 		objStorage: objStorage,
+		metrics:    metrics,
 	}
 	s.saverPool = pool.NewPool(4, 8, s.writeToDisk)
 	s.uploaderPool = pool.NewPool(8, 8, s.sendToS3)
@@ -92,8 +97,11 @@ func (v *ImageStorage) Process(ctx context.Context, sessID uint64, data []byte)
 			v.log.Error(ctx, "ExtractTarGz: unknown type: %d in %s", header.Typeflag, header.Name)
 		}
 	}
+	v.metrics.RecordOriginalArchiveExtractionDuration(time.Since(start).Seconds())
+	v.metrics.RecordOriginalArchiveSize(float64(len(images)))
+	v.metrics.IncreaseTotalSavedArchives()
 
-	v.log.Info(ctx, "arch size: %d, extracted archive in: %s", len(data), time.Since(start))
+	v.log.Debug(ctx, "arch size: %d, extracted archive in: %s", len(data), time.Since(start))
 	v.saverPool.Submit(&saveTask{ctx: ctx, sessionID: sessID, images: images})
 	return nil
 }
@@ -115,6 +123,7 @@ func (v *ImageStorage) writeToDisk(payload interface{}) {
 	// Write images to disk
 	saved := 0
 	for name, img := range task.images {
+		start := time.Now()
 		outFile, err := os.Create(path + name) // or open file in rewrite mode
 		if err != nil {
 			v.log.Error(task.ctx, "can't create file: %s", err.Error())
@@ -128,9 +137,11 @@ func (v *ImageStorage) writeToDisk(payload interface{}) {
 		if err := outFile.Close(); err != nil {
 			v.log.Warn(task.ctx, "can't close file: %s", err.Error())
 		}
+		v.metrics.RecordSavingImageDuration(time.Since(start).Seconds())
+		v.metrics.IncreaseTotalSavedImages()
 		saved++
 	}
-	v.log.Info(task.ctx, "saved %d images to disk", saved)
+	v.log.Debug(task.ctx, "saved %d images to disk", saved)
 	return
 }
 
@@ -151,8 +162,10 @@ func (v *ImageStorage) PackScreenshots(ctx context.Context, sessID uint64, files
 	if err != nil {
 		return fmt.Errorf("failed to execute command: %v, stderr: %v", err, stderr.String())
 	}
-	v.log.Info(ctx, "packed replay in %v", time.Since(start))
+	v.metrics.RecordArchivingDuration(time.Since(start).Seconds())
+	v.metrics.IncreaseTotalCreatedArchives()
 
+	v.log.Debug(ctx, "packed replay in %v", time.Since(start))
 	v.uploaderPool.Submit(&uploadTask{ctx: ctx, sessionID: sessionID, path: archPath, name: sessionID + "/replay.tar.zst"})
 	return nil
 }
@@ -167,6 +180,9 @@ func (v *ImageStorage) sendToS3(payload interface{}) {
 	if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.NoContentEncoding, objectstorage.Zstd); err != nil {
 		v.log.Fatal(task.ctx, "failed to upload replay file: %s", err)
 	}
-	v.log.Info(task.ctx, "replay file (size: %d) uploaded successfully in %v", len(video), time.Since(start))
+	v.metrics.RecordUploadingDuration(time.Since(start).Seconds())
+	v.metrics.RecordArchiveSize(float64(len(video)))
+
+	v.log.Debug(task.ctx, "replay file (size: %d) uploaded successfully in %v", len(video), time.Since(start))
 	return
 }

backend/internal/sessionender/ender.go

@@ -21,6 +21,7 @@ type session struct {
 
 // SessionEnder updates timestamp of last message for each session
 type SessionEnder struct {
+	metrics  ender.Ender
 	timeout  int64
 	sessions map[uint64]*session // map[sessionID]session
 	timeCtrl *timeController
@@ -28,8 +29,9 @@ type SessionEnder struct {
 	enabled  bool
 }
 
-func New(timeout int64, parts int) (*SessionEnder, error) {
+func New(metrics ender.Ender, timeout int64, parts int) (*SessionEnder, error) {
 	return &SessionEnder{
+		metrics:  metrics,
 		timeout:  timeout,
 		sessions: make(map[uint64]*session),
 		timeCtrl: NewTimeController(parts),
@@ -56,7 +58,7 @@ func (se *SessionEnder) ActivePartitions(parts []uint64) {
 	for sessID, _ := range se.sessions {
 		if !activeParts[sessID%se.parts] {
 			delete(se.sessions, sessID)
-			ender.DecreaseActiveSessions()
+			se.metrics.DecreaseActiveSessions()
 			removedSessions++
 		} else {
 			activeSessions++
@@ -89,8 +91,8 @@ func (se *SessionEnder) UpdateSession(msg messages.Message) {
 			isEnded:       false,
 			isMobile:      messages.IsMobileType(msg.TypeID()),
 		}
-		ender.IncreaseActiveSessions()
-		ender.IncreaseTotalSessions()
+		se.metrics.IncreaseActiveSessions()
+		se.metrics.IncreaseTotalSessions()
 		return
 	}
 	// Keep the highest user's timestamp for correct session duration value
@@ -139,8 +141,8 @@ func (se *SessionEnder) HandleEndedSessions(handler EndedSessionHandler) {
 			sess.isEnded = true
 			if res, _ := handler(sessID, sess.lastUserTime); res {
 				delete(se.sessions, sessID)
-				ender.DecreaseActiveSessions()
-				ender.IncreaseClosedSessions()
+				se.metrics.DecreaseActiveSessions()
+				se.metrics.IncreaseClosedSessions()
 				removedSessions++
 				if endCase == 2 {
 					brokerTime[1]++

backend/internal/sink/assetscache/assets.go

@@ -12,7 +12,7 @@ import (
 	"openreplay/backend/internal/config/sink"
 	"openreplay/backend/pkg/logger"
 	"openreplay/backend/pkg/messages"
-	metrics "openreplay/backend/pkg/metrics/sink"
+	sinkMetrics "openreplay/backend/pkg/metrics/sink"
 	"openreplay/backend/pkg/queue/types"
 	"openreplay/backend/pkg/url/assets"
 )
@@ -30,9 +30,10 @@ type AssetsCache struct {
 	producer  types.Producer
 	cache     map[string]*CachedAsset
 	blackList []string // use "example.com" to filter all domains or ".example.com" to filter only third-level domain
+	metrics   sinkMetrics.Sink
 }
 
-func New(log logger.Logger, cfg *sink.Config, rewriter *assets.Rewriter, producer types.Producer) *AssetsCache {
+func New(log logger.Logger, cfg *sink.Config, rewriter *assets.Rewriter, producer types.Producer, metrics sinkMetrics.Sink) *AssetsCache {
 	assetsCache := &AssetsCache{
 		log:       log,
 		cfg:       cfg,
@@ -40,6 +41,7 @@ func New(log logger.Logger, cfg *sink.Config, rewriter *assets.Rewriter, produce
 		producer:  producer,
 		cache:     make(map[string]*CachedAsset, 64),
 		blackList: make([]string, 0),
+		metrics:   metrics,
 	}
 	// Parse black list for cache layer
 	if len(cfg.CacheBlackList) > 0 {
@@ -76,7 +78,7 @@ func (e *AssetsCache) clearCache() {
 		if int64(now.Sub(cache.ts).Minutes()) > e.cfg.CacheExpiration {
 			deleted++
 			delete(e.cache, id)
-			metrics.DecreaseCachedAssets()
+			e.metrics.DecreaseCachedAssets()
 		}
 	}
 	e.log.Info(context.Background(), "cache cleaner: deleted %d/%d assets", deleted, cacheSize)
@@ -194,7 +196,7 @@ func parseHost(baseURL string) (string, error) {
 }
 
 func (e *AssetsCache) handleCSS(sessionID uint64, baseURL string, css string) string {
-	metrics.IncreaseTotalAssets()
+	e.metrics.IncreaseTotalAssets()
 	// Try to find asset in cache
 	h := md5.New()
 	// Cut first part of url (scheme + host)
@@ -217,7 +219,7 @@ func (e *AssetsCache) handleCSS(sessionID uint64, baseURL string, css string) st
 	e.mutex.RUnlock()
 	if ok {
 		if int64(time.Now().Sub(cachedAsset.ts).Minutes()) < e.cfg.CacheExpiration {
-			metrics.IncreaseSkippedAssets()
+			e.metrics.IncreaseSkippedAssets()
 			return cachedAsset.msg
 		}
 	}
@@ -229,8 +231,8 @@ func (e *AssetsCache) handleCSS(sessionID uint64, baseURL string, css string) st
 	start := time.Now()
 	res := e.getRewrittenCSS(sessionID, baseURL, css)
 	duration := time.Now().Sub(start).Milliseconds()
-	metrics.RecordAssetSize(float64(len(res)))
-	metrics.RecordProcessAssetDuration(float64(duration))
+	e.metrics.RecordAssetSize(float64(len(res)))
+	e.metrics.RecordProcessAssetDuration(float64(duration))
 	// Save asset to cache if we spent more than threshold
 	if duration > e.cfg.CacheThreshold {
 		e.mutex.Lock()
@@ -239,7 +241,7 @@ func (e *AssetsCache) handleCSS(sessionID uint64, baseURL string, css string) st
 			ts:  time.Now(),
 		}
 		e.mutex.Unlock()
-		metrics.IncreaseCachedAssets()
+		e.metrics.IncreaseCachedAssets()
 	}
 	// Return rewritten asset
 	return res

backend/internal/storage/storage.go

@@ -18,7 +18,7 @@ import (
 	config "openreplay/backend/internal/config/storage"
 	"openreplay/backend/pkg/logger"
 	"openreplay/backend/pkg/messages"
-	metrics "openreplay/backend/pkg/metrics/storage"
+	storageMetrics "openreplay/backend/pkg/metrics/storage"
 	"openreplay/backend/pkg/objectstorage"
 	"openreplay/backend/pkg/pool"
 )
@@ -77,9 +77,10 @@ type Storage struct {
 	splitTime     uint64
 	processorPool pool.WorkerPool
 	uploaderPool  pool.WorkerPool
+	metrics       storageMetrics.Storage
 }
 
-func New(cfg *config.Config, log logger.Logger, objStorage objectstorage.ObjectStorage) (*Storage, error) {
+func New(cfg *config.Config, log logger.Logger, objStorage objectstorage.ObjectStorage, metrics storageMetrics.Storage) (*Storage, error) {
 	switch {
 	case cfg == nil:
 		return nil, fmt.Errorf("config is empty")
@@ -92,6 +93,7 @@ func New(cfg *config.Config, log logger.Logger, objStorage objectstorage.ObjectS
 		objStorage: objStorage,
 		startBytes: make([]byte, cfg.FileSplitSize),
 		splitTime:  parseSplitTime(cfg.FileSplitTime),
+		metrics:    metrics,
 	}
 	s.processorPool = pool.NewPool(1, 1, s.doCompression)
 	s.uploaderPool = pool.NewPool(1, 1, s.uploadSession)
@@ -141,7 +143,7 @@ func (s *Storage) Process(ctx context.Context, msg *messages.SessionEnd) (err er
 	if err != nil {
 		if strings.Contains(err.Error(), "big file") {
 			s.log.Warn(ctx, "can't process session: %s", err)
-			metrics.IncreaseStorageTotalSkippedSessions()
+			s.metrics.IncreaseStorageTotalSkippedSessions()
 			return nil
 		}
 		return err
@@ -159,8 +161,8 @@ func (s *Storage) prepareSession(path string, tp FileType, task *Task) error {
 		return err
 	}
 
-	metrics.RecordSessionReadDuration(float64(time.Now().Sub(startRead).Milliseconds()), tp.String())
-	metrics.RecordSessionSize(float64(len(mob)), tp.String())
+	s.metrics.RecordSessionReadDuration(float64(time.Now().Sub(startRead).Milliseconds()), tp.String())
+	s.metrics.RecordSessionSize(float64(len(mob)), tp.String())
 
 	// Put opened session file into task struct
 	task.SetMob(mob, index, tp)
@@ -174,7 +176,7 @@ func (s *Storage) openSession(ctx context.Context, filePath string, tp FileType)
 	// Check file size before download into memory
 	info, err := os.Stat(filePath)
 	if err == nil && info.Size() > s.cfg.MaxFileSize {
-		metrics.RecordSkippedSessionSize(float64(info.Size()), tp.String())
+		s.metrics.RecordSkippedSessionSize(float64(info.Size()), tp.String())
 		return nil, -1, fmt.Errorf("big file, size: %d", info.Size())
 	}
 	// Read file into memory
@@ -190,7 +192,7 @@ func (s *Storage) openSession(ctx context.Context, filePath string, tp FileType)
 	if err != nil {
 		return nil, -1, fmt.Errorf("can't sort session, err: %s", err)
 	}
-	metrics.RecordSessionSortDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
+	s.metrics.RecordSessionSortDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
 	return mob, index, nil
 }
 
@@ -234,12 +236,12 @@ func (s *Storage) packSession(task *Task, tp FileType) {
 		// Compression
 		start := time.Now()
 		data := s.compress(task.ctx, mob, task.compression)
-		metrics.RecordSessionCompressDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
+		s.metrics.RecordSessionCompressDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
 
 		// Encryption
 		start = time.Now()
 		result := s.encryptSession(task.ctx, data.Bytes(), task.key)
-		metrics.RecordSessionEncryptionDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
+		s.metrics.RecordSessionEncryptionDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
 
 		if tp == DOM {
 			task.doms = bytes.NewBuffer(result)
@@ -296,8 +298,8 @@ func (s *Storage) packSession(task *Task, tp FileType) {
 	wg.Wait()
 
 	// Record metrics
-	metrics.RecordSessionEncryptionDuration(float64(firstEncrypt+secondEncrypt), tp.String())
-	metrics.RecordSessionCompressDuration(float64(firstPart+secondPart), tp.String())
+	s.metrics.RecordSessionEncryptionDuration(float64(firstEncrypt+secondEncrypt), tp.String())
+	s.metrics.RecordSessionCompressDuration(float64(firstPart+secondPart), tp.String())
 }
 
 func (s *Storage) encryptSession(ctx context.Context, data []byte, encryptionKey string) []byte {
@@ -382,7 +384,7 @@ func (s *Storage) uploadSession(payload interface{}) {
 	go func() {
 		if task.doms != nil {
 			// Record compression ratio
-			metrics.RecordSessionCompressionRatio(task.domsRawSize/float64(task.doms.Len()), DOM.String())
+			s.metrics.RecordSessionCompressionRatio(task.domsRawSize/float64(task.doms.Len()), DOM.String())
 			// Upload session to s3
 			start := time.Now()
 			if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil {
@@ -395,7 +397,7 @@ func (s *Storage) uploadSession(payload interface{}) {
 	go func() {
 		if task.dome != nil {
 			// Record compression ratio
-			metrics.RecordSessionCompressionRatio(task.domeRawSize/float64(task.dome.Len()), DOM.String())
+			s.metrics.RecordSessionCompressionRatio(task.domeRawSize/float64(task.dome.Len()), DOM.String())
 			// Upload session to s3
 			start := time.Now()
 			if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil {
@@ -408,7 +410,7 @@ func (s *Storage) uploadSession(payload interface{}) {
 	go func() {
 		if task.dev != nil {
 			// Record compression ratio
-			metrics.RecordSessionCompressionRatio(task.devRawSize/float64(task.dev.Len()), DEV.String())
+			s.metrics.RecordSessionCompressionRatio(task.devRawSize/float64(task.dev.Len()), DEV.String())
 			// Upload session to s3
 			start := time.Now()
 			if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil {
@@ -419,9 +421,9 @@ func (s *Storage) uploadSession(payload interface{}) {
 		wg.Done()
 	}()
 	wg.Wait()
-	metrics.RecordSessionUploadDuration(float64(uploadDoms+uploadDome), DOM.String())
-	metrics.RecordSessionUploadDuration(float64(uploadDev), DEV.String())
-	metrics.IncreaseStorageTotalSessions()
+	s.metrics.RecordSessionUploadDuration(float64(uploadDoms+uploadDome), DOM.String())
+	s.metrics.RecordSessionUploadDuration(float64(uploadDev), DEV.String())
+	s.metrics.IncreaseStorageTotalSessions()
 }
 
 func (s *Storage) doCompression(payload interface{}) {

backend/pkg/analytics/builder.go

@@ -3,6 +3,7 @@ package analytics
 import (
 	"github.com/go-playground/validator/v10"
 	"openreplay/backend/pkg/analytics/charts"
+	"openreplay/backend/pkg/metrics/database"
 	"time"
 
 	"openreplay/backend/internal/config/analytics"
@@ -26,9 +27,9 @@ type ServicesBuilder struct {
 	ChartsAPI     api.Handlers
 }
 
-func NewServiceBuilder(log logger.Logger, cfg *analytics.Config, webMetrics web.Web, pgconn pool.Pool) (*ServicesBuilder, error) {
+func NewServiceBuilder(log logger.Logger, cfg *analytics.Config, webMetrics web.Web, dbMetrics database.Database, pgconn pool.Pool) (*ServicesBuilder, error) {
 	responser := api.NewResponser(webMetrics)
-	audiTrail, err := tracer.NewTracer(log, pgconn)
+	audiTrail, err := tracer.NewTracer(log, pgconn, dbMetrics)
 	if err != nil {
 		return nil, err
 	}
@@ -58,7 +59,7 @@ func NewServiceBuilder(log logger.Logger, cfg *analytics.Config, webMetrics web.
 		return nil, err
 	}
 	return &ServicesBuilder{
-		Auth:          auth.NewAuth(log, cfg.JWTSecret, cfg.JWTSpotSecret, pgconn, nil),
+		Auth:          auth.NewAuth(log, cfg.JWTSecret, cfg.JWTSpotSecret, pgconn, nil, api.NoPrefix),
 		RateLimiter:   limiter.NewUserRateLimiter(10, 30, 1*time.Minute, 5*time.Minute),
 		AuditTrail:    audiTrail,
 		CardsAPI:      cardsHandlers,

backend/pkg/db/clickhouse/bulk.go

@@ -18,13 +18,14 @@ type Bulk interface {
 }
 
 type bulkImpl struct {
-	conn   driver.Conn
-	table  string
-	query  string
-	values [][]interface{}
+	conn    driver.Conn
+	metrics database.Database
+	table   string
+	query   string
+	values  [][]interface{}
 }
 
-func NewBulk(conn driver.Conn, table, query string) (Bulk, error) {
+func NewBulk(conn driver.Conn, metrics database.Database, table, query string) (Bulk, error) {
 	switch {
 	case conn == nil:
 		return nil, errors.New("clickhouse connection is empty")
@@ -34,10 +35,11 @@ func NewBulk(conn driver.Conn, table, query string) (Bulk, error) {
 		return nil, errors.New("query is empty")
 	}
 	return &bulkImpl{
-		conn:   conn,
-		table:  table,
-		query:  query,
-		values: make([][]interface{}, 0),
+		conn:    conn,
+		metrics: metrics,
+		table:   table,
+		query:   query,
+		values:  make([][]interface{}, 0),
 	}, nil
 }
 
@@ -60,8 +62,8 @@ func (b *bulkImpl) Send() error {
 	}
 	err = batch.Send()
 	// Save bulk metrics
-	database.RecordBulkElements(float64(len(b.values)), "ch", b.table)
-	database.RecordBulkInsertDuration(float64(time.Now().Sub(start).Milliseconds()), "ch", b.table)
+	b.metrics.RecordBulkElements(float64(len(b.values)), "ch", b.table)
+	b.metrics.RecordBulkInsertDuration(float64(time.Now().Sub(start).Milliseconds()), "ch", b.table)
 	// Prepare values slice for a new data
 	b.values = make([][]interface{}, 0)
 	return err

backend/pkg/db/clickhouse/connector.go

@@ -18,6 +18,7 @@ import (
 	"openreplay/backend/pkg/db/types"
 	"openreplay/backend/pkg/hashid"
 	"openreplay/backend/pkg/messages"
+	"openreplay/backend/pkg/metrics/database"
 	"openreplay/backend/pkg/sessions"
 	"openreplay/backend/pkg/url"
 )
@@ -57,13 +58,14 @@ func NewTask() *task {
 
 type connectorImpl struct {
 	conn       driver.Conn
+	metrics    database.Database
 	batches    map[string]Bulk //driver.Batch
 	workerTask chan *task
 	done       chan struct{}
 	finished   chan struct{}
 }
 
-func NewConnector(cfg common.Clickhouse) Connector {
+func NewConnector(cfg common.Clickhouse, metrics database.Database) Connector {
 	conn, err := clickhouse.Open(&clickhouse.Options{
 		Addr: []string{cfg.GetTrimmedURL()},
 		Auth: clickhouse.Auth{
@@ -84,6 +86,7 @@ func NewConnector(cfg common.Clickhouse) Connector {
 
 	c := &connectorImpl{
 		conn:       conn,
+		metrics:    metrics,
 		batches:    make(map[string]Bulk, 20),
 		workerTask: make(chan *task, 1),
 		done:       make(chan struct{}),
@@ -94,7 +97,7 @@ func NewConnector(cfg common.Clickhouse) Connector {
 }
 
 func (c *connectorImpl) newBatch(name, query string) error {
-	batch, err := NewBulk(c.conn, name, query)
+	batch, err := NewBulk(c.conn, c.metrics, name, query)
 	if err != nil {
 		return fmt.Errorf("can't create new batch: %s", err)
 	}
@@ -103,25 +106,25 @@ func (c *connectorImpl) newBatch(name, query string) error {
 }
 
 var batches = map[string]string{
-	"sessions":        "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, timezone, utm_source, utm_medium, utm_campaign) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?, ?, ?)",
+	"sessions":        "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, platform, timezone, utm_source, utm_medium, utm_campaign) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?)",
 	"autocompletes":   "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, SUBSTR(?, 1, 8000))",
-	"pages":           `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"clicks":          `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"inputs":          `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$duration_s", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"errors":          `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"performance":     `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"requests":        `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$duration_s", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"custom":          `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?,	?, ?)`,
-	"graphql":         `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"issuesEvents":    `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", issue_type, issue_id, "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"pages":           `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"clicks":          `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"inputs":          `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$duration_s", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"errors":          `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", error_id, "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"performance":     `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"requests":        `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$duration_s", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"custom":          `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"graphql":         `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"issuesEvents":    `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", issue_type, issue_id, "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 	"issues":          "INSERT INTO experimental.issues (project_id, issue_id, type, context_string) VALUES (?, ?, ?, ?)",
 	"mobile_sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, platform, timezone) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?)",
-	"mobile_custom":   `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"mobile_clicks":   `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"mobile_swipes":   `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"mobile_inputs":   `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"mobile_requests": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	"mobile_crashes":  `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"mobile_custom":   `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"mobile_clicks":   `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"mobile_swipes":   `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"mobile_inputs":   `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"mobile_requests": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	"mobile_crashes":  `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$device", "$os_version", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 }
 
 func (c *connectorImpl) Prepare() error {
@@ -212,6 +215,7 @@ func (c *connectorImpl) InsertWebSession(session *sessions.Session) error {
 		session.Metadata8,
 		session.Metadata9,
 		session.Metadata10,
+		"web",
 		session.Timezone,
 		session.UtmSource,
 		session.UtmMedium,
@@ -243,8 +247,10 @@ func (c *connectorImpl) InsertWebInputDuration(session *sessions.Session, msg *m
 		return nil
 	}
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"label":           msg.Label,
-		"hesitation_time": nullableUint32(uint32(msg.HesitationTime)),
+		"label":            msg.Label,
+		"hesitation_time":  nullableUint32(uint32(msg.HesitationTime)),
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal input event: %s", err)
@@ -259,6 +265,8 @@ func (c *connectorImpl) InsertWebInputDuration(session *sessions.Session, msg *m
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		nullableUint16(uint16(msg.InputDuration)),
 		jsonString,
 	); err != nil {
@@ -275,12 +283,14 @@ func (c *connectorImpl) InsertMouseThrashing(session *sessions.Session, msg *mes
 		return fmt.Errorf("can't extract url parts: %s", err)
 	}
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"issue_id":     issueID,
-		"issue_type":   "mouse_thrashing",
-		"url":          cropString(msg.Url),
-		"url_host":     host,
-		"url_path":     path,
-		"url_hostpath": hostpath,
+		"issue_id":         issueID,
+		"issue_type":       "mouse_thrashing",
+		"url":              cropString(msg.Url),
+		"url_host":         host,
+		"url_path":         path,
+		"url_hostpath":     hostpath,
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal issue event: %s", err)
@@ -295,6 +305,8 @@ func (c *connectorImpl) InsertMouseThrashing(session *sessions.Session, msg *mes
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		"mouse_thrashing",
 		issueID,
 		jsonString,
@@ -327,12 +339,14 @@ func (c *connectorImpl) InsertIssue(session *sessions.Session, msg *messages.Iss
 		return fmt.Errorf("can't extract url parts: %s", err)
 	}
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"issue_id":     issueID,
-		"issue_type":   msg.Type,
-		"url":          cropString(msg.Url),
-		"url_host":     host,
-		"url_path":     path,
-		"url_hostpath": hostpath,
+		"issue_id":         issueID,
+		"issue_type":       msg.Type,
+		"url":              cropString(msg.Url),
+		"url_host":         host,
+		"url_path":         path,
+		"url_hostpath":     hostpath,
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal issue event: %s", err)
@@ -347,6 +361,8 @@ func (c *connectorImpl) InsertIssue(session *sessions.Session, msg *messages.Iss
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		msg.Type,
 		issueID,
 		jsonString,
@@ -418,6 +434,8 @@ func (c *connectorImpl) InsertWebPageEvent(session *sessions.Session, msg *messa
 		"dom_building_time":              domBuildingTime,
 		"dom_content_loaded_event_time":  domContentLoadedEventTime,
 		"load_event_time":                loadEventTime,
+		"user_device":                    session.UserDevice,
+		"user_device_type":               session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal page event: %s", err)
@@ -432,6 +450,8 @@ func (c *connectorImpl) InsertWebPageEvent(session *sessions.Session, msg *messa
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		cropString(msg.URL),
 		jsonString,
 	); err != nil {
@@ -465,15 +485,17 @@ func (c *connectorImpl) InsertWebClickEvent(session *sessions.Session, msg *mess
 		return fmt.Errorf("can't extract url parts: %s", err)
 	}
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"label":           msg.Label,
-		"hesitation_time": nullableUint32(uint32(msg.HesitationTime)),
-		"selector":        msg.Selector,
-		"normalized_x":    nX,
-		"normalized_y":    nY,
-		"url":             cropString(msg.Url),
-		"url_host":        host,
-		"url_path":        path,
-		"url_hostpath":    hostpath,
+		"label":            msg.Label,
+		"hesitation_time":  nullableUint32(uint32(msg.HesitationTime)),
+		"selector":         msg.Selector,
+		"normalized_x":     nX,
+		"normalized_y":     nY,
+		"url":              cropString(msg.Url),
+		"url_host":         host,
+		"url_path":         path,
+		"url_hostpath":     hostpath,
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal click event: %s", err)
@@ -488,6 +510,8 @@ func (c *connectorImpl) InsertWebClickEvent(session *sessions.Session, msg *mess
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		cropString(msg.Url),
 		jsonString,
 	); err != nil {
@@ -498,11 +522,6 @@ func (c *connectorImpl) InsertWebClickEvent(session *sessions.Session, msg *mess
 }
 
 func (c *connectorImpl) InsertWebErrorEvent(session *sessions.Session, msg *types.ErrorEvent) error {
-	keys, values := make([]string, 0, len(msg.Tags)), make([]*string, 0, len(msg.Tags))
-	for k, v := range msg.Tags {
-		keys = append(keys, k)
-		values = append(values, v)
-	}
 	// Check error source before insert to avoid panic from clickhouse lib
 	switch msg.Source {
 	case "js_exception", "bugsnag", "cloudwatch", "datadog", "elasticsearch", "newrelic", "rollbar", "sentry", "stackdriver", "sumologic":
@@ -511,12 +530,11 @@ func (c *connectorImpl) InsertWebErrorEvent(session *sessions.Session, msg *type
 	}
 	msgID, _ := msg.ID(session.ProjectID)
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"source":            msg.Source,
-		"name":              nullableString(msg.Name),
-		"message":           msg.Message,
-		"error_id":          msgID,
-		"error_tags_keys":   keys,
-		"error_tags_values": values,
+		"source":           msg.Source,
+		"name":             nullableString(msg.Name),
+		"message":          msg.Message,
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal error event: %s", err)
@@ -531,6 +549,9 @@ func (c *connectorImpl) InsertWebErrorEvent(session *sessions.Session, msg *type
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
+		msgID,
 		jsonString,
 	); err != nil {
 		c.checkError("errors", err)
@@ -562,6 +583,8 @@ func (c *connectorImpl) InsertWebPerformanceTrackAggr(session *sessions.Session,
 		"min_used_js_heap_size":  msg.MinUsedJSHeapSize,
 		"avg_used_js_heap_size":  msg.AvgUsedJSHeapSize,
 		"max_used_js_heap_size":  msg.MaxUsedJSHeapSize,
+		"user_device":            session.UserDevice,
+		"user_device_type":       session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal performance event: %s", err)
@@ -576,6 +599,8 @@ func (c *connectorImpl) InsertWebPerformanceTrackAggr(session *sessions.Session,
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		jsonString,
 	); err != nil {
 		c.checkError("performance", err)
@@ -599,16 +624,18 @@ func (c *connectorImpl) InsertRequest(session *sessions.Session, msg *messages.N
 		return fmt.Errorf("can't extract url parts: %s", err)
 	}
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"request_body":  request,
-		"response_body": response,
-		"status":        uint16(msg.Status),
-		"method":        url.EnsureMethod(msg.Method),
-		"success":       msg.Status < 400,
-		"transfer_size": uint32(msg.TransferredBodySize),
-		"url":           cropString(msg.URL),
-		"url_host":      host,
-		"url_path":      path,
-		"url_hostpath":  hostpath,
+		"request_body":     request,
+		"response_body":    response,
+		"status":           uint16(msg.Status),
+		"method":           url.EnsureMethod(msg.Method),
+		"success":          msg.Status < 400,
+		"transfer_size":    uint32(msg.TransferredBodySize),
+		"url":              cropString(msg.URL),
+		"url_host":         host,
+		"url_path":         path,
+		"url_hostpath":     hostpath,
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal request event: %s", err)
@@ -623,6 +650,8 @@ func (c *connectorImpl) InsertRequest(session *sessions.Session, msg *messages.N
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		nullableUint16(uint16(msg.Duration)),
 		jsonString,
 	); err != nil {
@@ -634,8 +663,10 @@ func (c *connectorImpl) InsertRequest(session *sessions.Session, msg *messages.N
 
 func (c *connectorImpl) InsertCustom(session *sessions.Session, msg *messages.CustomEvent) error {
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"name":    msg.Name,
-		"payload": msg.Payload,
+		"name":             msg.Name,
+		"payload":          msg.Payload,
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal custom event: %s", err)
@@ -650,6 +681,8 @@ func (c *connectorImpl) InsertCustom(session *sessions.Session, msg *messages.Cu
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		jsonString,
 	); err != nil {
 		c.checkError("custom", err)
@@ -660,9 +693,11 @@ func (c *connectorImpl) InsertCustom(session *sessions.Session, msg *messages.Cu
 
 func (c *connectorImpl) InsertGraphQL(session *sessions.Session, msg *messages.GraphQL) error {
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"name":          msg.OperationName,
-		"request_body":  nullableString(msg.Variables),
-		"response_body": nullableString(msg.Response),
+		"name":             msg.OperationName,
+		"request_body":     nullableString(msg.Variables),
+		"response_body":    nullableString(msg.Response),
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal graphql event: %s", err)
@@ -677,6 +712,8 @@ func (c *connectorImpl) InsertGraphQL(session *sessions.Session, msg *messages.G
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		jsonString,
 	); err != nil {
 		c.checkError("graphql", err)
@@ -724,7 +761,7 @@ func (c *connectorImpl) InsertMobileSession(session *sessions.Session) error {
 		session.Metadata8,
 		session.Metadata9,
 		session.Metadata10,
-		"ios",
+		"mobile",
 		session.Timezone,
 	); err != nil {
 		c.checkError("mobile_sessions", err)
@@ -735,8 +772,10 @@ func (c *connectorImpl) InsertMobileSession(session *sessions.Session) error {
 
 func (c *connectorImpl) InsertMobileCustom(session *sessions.Session, msg *messages.MobileEvent) error {
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"name":    msg.Name,
-		"payload": msg.Payload,
+		"name":             msg.Name,
+		"payload":          msg.Payload,
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal mobile custom event: %s", err)
@@ -751,6 +790,8 @@ func (c *connectorImpl) InsertMobileCustom(session *sessions.Session, msg *messa
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		jsonString,
 	); err != nil {
 		c.checkError("mobile_custom", err)
@@ -764,7 +805,9 @@ func (c *connectorImpl) InsertMobileClick(session *sessions.Session, msg *messag
 		return nil
 	}
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"label": msg.Label,
+		"label":            msg.Label,
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal mobile clicks event: %s", err)
@@ -779,6 +822,8 @@ func (c *connectorImpl) InsertMobileClick(session *sessions.Session, msg *messag
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		jsonString,
 	); err != nil {
 		c.checkError("mobile_clicks", err)
@@ -792,8 +837,10 @@ func (c *connectorImpl) InsertMobileSwipe(session *sessions.Session, msg *messag
 		return nil
 	}
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"label":     msg.Label,
-		"direction": nullableString(msg.Direction),
+		"label":            msg.Label,
+		"direction":        nullableString(msg.Direction),
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal mobile swipe event: %s", err)
@@ -808,6 +855,8 @@ func (c *connectorImpl) InsertMobileSwipe(session *sessions.Session, msg *messag
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		jsonString,
 	); err != nil {
 		c.checkError("mobile_swipes", err)
@@ -821,7 +870,9 @@ func (c *connectorImpl) InsertMobileInput(session *sessions.Session, msg *messag
 		return nil
 	}
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"label": msg.Label,
+		"label":            msg.Label,
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal mobile input event: %s", err)
@@ -836,6 +887,8 @@ func (c *connectorImpl) InsertMobileInput(session *sessions.Session, msg *messag
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		jsonString,
 	); err != nil {
 		c.checkError("mobile_inputs", err)
@@ -855,13 +908,15 @@ func (c *connectorImpl) InsertMobileRequest(session *sessions.Session, msg *mess
 		response = &msg.Response
 	}
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"url":           cropString(msg.URL),
-		"request_body":  request,
-		"response_body": response,
-		"status":        uint16(msg.Status),
-		"method":        url.EnsureMethod(msg.Method),
-		"duration":      uint16(msg.Duration),
-		"success":       msg.Status < 400,
+		"url":              cropString(msg.URL),
+		"request_body":     request,
+		"response_body":    response,
+		"status":           uint16(msg.Status),
+		"method":           url.EnsureMethod(msg.Method),
+		"duration":         uint16(msg.Duration),
+		"success":          msg.Status < 400,
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal mobile request event: %s", err)
@@ -876,6 +931,8 @@ func (c *connectorImpl) InsertMobileRequest(session *sessions.Session, msg *mess
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		jsonString,
 	); err != nil {
 		c.checkError("mobile_requests", err)
@@ -886,9 +943,11 @@ func (c *connectorImpl) InsertMobileRequest(session *sessions.Session, msg *mess
 
 func (c *connectorImpl) InsertMobileCrash(session *sessions.Session, msg *messages.MobileCrash) error {
 	jsonString, err := json.Marshal(map[string]interface{}{
-		"name":       msg.Name,
-		"reason":     msg.Reason,
-		"stacktrace": msg.Stacktrace,
+		"name":             msg.Name,
+		"reason":           msg.Reason,
+		"stacktrace":       msg.Stacktrace,
+		"user_device":      session.UserDevice,
+		"user_device_type": session.UserDeviceType,
 	})
 	if err != nil {
 		return fmt.Errorf("can't marshal mobile crash event: %s", err)
@@ -903,6 +962,8 @@ func (c *connectorImpl) InsertMobileCrash(session *sessions.Session, msg *messag
 		eventTime.Unix(),
 		session.UserUUID,
 		true,
+		session.Platform,
+		session.UserOSVersion,
 		jsonString,
 	); err != nil {
 		c.checkError("mobile_crashes", err)

backend/pkg/db/postgres/batch/batches.go

@@ -52,6 +52,7 @@ func NewBatchesTask(size int) *batchesTask {
 type BatchSet struct {
 	log        logger.Logger
 	c          pool.Pool
+	metrics    database.Database
 	ctx        context.Context
 	batches    map[uint64]*SessionBatch
 	workerTask chan *batchesTask
@@ -59,10 +60,11 @@ type BatchSet struct {
 	finished   chan struct{}
 }
 
-func NewBatchSet(log logger.Logger, c pool.Pool) *BatchSet {
+func NewBatchSet(log logger.Logger, c pool.Pool, metrics database.Database) *BatchSet {
 	bs := &BatchSet{
 		log:        log,
 		c:          c,
+		metrics:    metrics,
 		ctx:        context.Background(),
 		batches:    make(map[uint64]*SessionBatch),
 		workerTask: make(chan *batchesTask, 1),
@@ -104,7 +106,7 @@ func (conn *BatchSet) Stop() {
 func (conn *BatchSet) sendBatches(t *batchesTask) {
 	for _, batch := range t.batches {
 		// Record batch size
-		database.RecordBatchElements(float64(batch.Len()))
+		conn.metrics.RecordBatchElements(float64(batch.Len()))
 
 		start := time.Now()
 
@@ -120,7 +122,7 @@ func (conn *BatchSet) sendBatches(t *batchesTask) {
 			}
 		}
 		br.Close() // returns err
-		database.RecordBatchInsertDuration(float64(time.Now().Sub(start).Milliseconds()))
+		conn.metrics.RecordBatchInsertDuration(float64(time.Now().Sub(start).Milliseconds()))
 	}
 }
 

backend/pkg/db/postgres/bulk.go

@@ -24,6 +24,7 @@ type Bulk interface {
 
 type bulkImpl struct {
 	conn      pool.Pool
+	metrics   database.Database
 	table     string
 	columns   string
 	template  string
@@ -75,12 +76,12 @@ func (b *bulkImpl) send() error {
 		return fmt.Errorf("send bulk err: %s", err)
 	}
 	// Save bulk metrics
-	database.RecordBulkElements(float64(size), "pg", b.table)
-	database.RecordBulkInsertDuration(float64(time.Now().Sub(start).Milliseconds()), "pg", b.table)
+	b.metrics.RecordBulkElements(float64(size), "pg", b.table)
+	b.metrics.RecordBulkInsertDuration(float64(time.Now().Sub(start).Milliseconds()), "pg", b.table)
 	return nil
 }
 
-func NewBulk(conn pool.Pool, table, columns, template string, setSize, sizeLimit int) (Bulk, error) {
+func NewBulk(conn pool.Pool, metrics database.Database, table, columns, template string, setSize, sizeLimit int) (Bulk, error) {
 	switch {
 	case conn == nil:
 		return nil, errors.New("db conn is empty")
@@ -97,6 +98,7 @@ func NewBulk(conn pool.Pool, table, columns, template string, setSize, sizeLimit
 	}
 	return &bulkImpl{
 		conn:      conn,
+		metrics:   metrics,
 		table:     table,
 		columns:   columns,
 		template:  template,

backend/pkg/db/postgres/bulks.go

@@ -2,6 +2,7 @@ package postgres
 
 import (
 	"context"
+	"openreplay/backend/pkg/metrics/database"
 
 	"openreplay/backend/pkg/db/postgres/pool"
 	"openreplay/backend/pkg/logger"
@@ -21,6 +22,7 @@ type BulkSet struct {
 	log               logger.Logger
 	c                 pool.Pool
 	ctx               context.Context
+	metrics           database.Database
 	autocompletes     Bulk
 	requests          Bulk
 	customEvents      Bulk
@@ -43,10 +45,11 @@ type BulkSet struct {
 	finished          chan struct{}
 }
 
-func NewBulkSet(log logger.Logger, c pool.Pool) *BulkSet {
+func NewBulkSet(log logger.Logger, c pool.Pool, metrics database.Database) *BulkSet {
 	bs := &BulkSet{
 		log:        log,
 		c:          c,
+		metrics:    metrics,
 		ctx:        context.Background(),
 		workerTask: make(chan *bulksTask, 1),
 		done:       make(chan struct{}),
@@ -100,7 +103,7 @@ func (conn *BulkSet) Get(name string) Bulk {
 
 func (conn *BulkSet) initBulks() {
 	var err error
-	conn.autocompletes, err = NewBulk(conn.c,
+	conn.autocompletes, err = NewBulk(conn.c, conn.metrics,
 		"autocomplete",
 		"(value, type, project_id)",
 		"($%d, $%d, $%d)",
@@ -108,7 +111,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create autocomplete bulk: %s", err)
 	}
-	conn.requests, err = NewBulk(conn.c,
+	conn.requests, err = NewBulk(conn.c, conn.metrics,
 		"events_common.requests",
 		"(session_id, timestamp, seq_index, url, duration, success)",
 		"($%d, $%d, $%d, LEFT($%d, 8000), $%d, $%d)",
@@ -116,7 +119,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create requests bulk: %s", err)
 	}
-	conn.customEvents, err = NewBulk(conn.c,
+	conn.customEvents, err = NewBulk(conn.c, conn.metrics,
 		"events_common.customs",
 		"(session_id, timestamp, seq_index, name, payload)",
 		"($%d, $%d, $%d, LEFT($%d, 2000), $%d)",
@@ -124,7 +127,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create customEvents bulk: %s", err)
 	}
-	conn.webPageEvents, err = NewBulk(conn.c,
+	conn.webPageEvents, err = NewBulk(conn.c, conn.metrics,
 		"events.pages",
 		"(session_id, message_id, timestamp, referrer, base_referrer, host, path, query, dom_content_loaded_time, "+
 			"load_time, response_end, first_paint_time, first_contentful_paint_time, speed_index, visually_complete, "+
@@ -136,7 +139,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webPageEvents bulk: %s", err)
 	}
-	conn.webInputDurations, err = NewBulk(conn.c,
+	conn.webInputDurations, err = NewBulk(conn.c, conn.metrics,
 		"events.inputs",
 		"(session_id, message_id, timestamp, label, hesitation, duration)",
 		"($%d, $%d, $%d, NULLIF(LEFT($%d, 2000),''), $%d, $%d)",
@@ -144,7 +147,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webInputDurations bulk: %s", err)
 	}
-	conn.webGraphQL, err = NewBulk(conn.c,
+	conn.webGraphQL, err = NewBulk(conn.c, conn.metrics,
 		"events.graphql",
 		"(session_id, timestamp, message_id, name, request_body, response_body)",
 		"($%d, $%d, $%d, LEFT($%d, 2000), $%d, $%d)",
@@ -152,7 +155,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webGraphQL bulk: %s", err)
 	}
-	conn.webErrors, err = NewBulk(conn.c,
+	conn.webErrors, err = NewBulk(conn.c, conn.metrics,
 		"errors",
 		"(error_id, project_id, source, name, message, payload)",
 		"($%d, $%d, $%d, $%d, $%d, $%d::jsonb)",
@@ -160,7 +163,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webErrors bulk: %s", err)
 	}
-	conn.webErrorEvents, err = NewBulk(conn.c,
+	conn.webErrorEvents, err = NewBulk(conn.c, conn.metrics,
 		"events.errors",
 		"(session_id, message_id, timestamp, error_id)",
 		"($%d, $%d, $%d, $%d)",
@@ -168,7 +171,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webErrorEvents bulk: %s", err)
 	}
-	conn.webErrorTags, err = NewBulk(conn.c,
+	conn.webErrorTags, err = NewBulk(conn.c, conn.metrics,
 		"public.errors_tags",
 		"(session_id, message_id, error_id, key, value)",
 		"($%d, $%d, $%d, $%d, $%d)",
@@ -176,7 +179,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webErrorTags bulk: %s", err)
 	}
-	conn.webIssues, err = NewBulk(conn.c,
+	conn.webIssues, err = NewBulk(conn.c, conn.metrics,
 		"issues",
 		"(project_id, issue_id, type, context_string)",
 		"($%d, $%d, $%d, $%d)",
@@ -184,7 +187,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webIssues bulk: %s", err)
 	}
-	conn.webIssueEvents, err = NewBulk(conn.c,
+	conn.webIssueEvents, err = NewBulk(conn.c, conn.metrics,
 		"events_common.issues",
 		"(session_id, issue_id, timestamp, seq_index, payload)",
 		"($%d, $%d, $%d, $%d, CAST($%d AS jsonb))",
@@ -192,7 +195,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webIssueEvents bulk: %s", err)
 	}
-	conn.webCustomEvents, err = NewBulk(conn.c,
+	conn.webCustomEvents, err = NewBulk(conn.c, conn.metrics,
 		"events_common.customs",
 		"(session_id, seq_index, timestamp, name, payload, level)",
 		"($%d, $%d, $%d, LEFT($%d, 2000), $%d, $%d)",
@@ -200,7 +203,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webCustomEvents bulk: %s", err)
 	}
-	conn.webClickEvents, err = NewBulk(conn.c,
+	conn.webClickEvents, err = NewBulk(conn.c, conn.metrics,
 		"events.clicks",
 		"(session_id, message_id, timestamp, label, selector, url, path, hesitation)",
 		"($%d, $%d, $%d, NULLIF(LEFT($%d, 2000), ''), LEFT($%d, 8000), LEFT($%d, 2000), LEFT($%d, 2000), $%d)",
@@ -208,7 +211,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webClickEvents bulk: %s", err)
 	}
-	conn.webClickXYEvents, err = NewBulk(conn.c,
+	conn.webClickXYEvents, err = NewBulk(conn.c, conn.metrics,
 		"events.clicks",
 		"(session_id, message_id, timestamp, label, selector, url, path, hesitation, normalized_x, normalized_y)",
 		"($%d, $%d, $%d, NULLIF(LEFT($%d, 2000), ''), LEFT($%d, 8000), LEFT($%d, 2000), LEFT($%d, 2000), $%d, $%d, $%d)",
@@ -216,7 +219,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webClickEvents bulk: %s", err)
 	}
-	conn.webNetworkRequest, err = NewBulk(conn.c,
+	conn.webNetworkRequest, err = NewBulk(conn.c, conn.metrics,
 		"events_common.requests",
 		"(session_id, timestamp, seq_index, url, host, path, query, request_body, response_body, status_code, method, duration, success, transfer_size)",
 		"($%d, $%d, $%d, LEFT($%d, 8000), LEFT($%d, 300), LEFT($%d, 2000), LEFT($%d, 8000), $%d, $%d, $%d::smallint, NULLIF($%d, '')::http_method, $%d, $%d, $%d)",
@@ -224,7 +227,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webNetworkRequest bulk: %s", err)
 	}
-	conn.webCanvasNodes, err = NewBulk(conn.c,
+	conn.webCanvasNodes, err = NewBulk(conn.c, conn.metrics,
 		"events.canvas_recordings",
 		"(session_id, recording_id, timestamp)",
 		"($%d, $%d, $%d)",
@@ -232,7 +235,7 @@ func (conn *BulkSet) initBulks() {
 	if err != nil {
 		conn.log.Fatal(conn.ctx, "can't create webCanvasNodes bulk: %s", err)
 	}
-	conn.webTagTriggers, err = NewBulk(conn.c,
+	conn.webTagTriggers, err = NewBulk(conn.c, conn.metrics,
 		"events.tags",
 		"(session_id, timestamp, seq_index, tag_id)",
 		"($%d, $%d, $%d, $%d)",

backend/pkg/db/postgres/connector.go

@@ -2,6 +2,7 @@ package postgres
 
 import (
 	"context"
+	"openreplay/backend/pkg/metrics/database"
 
 	"openreplay/backend/pkg/db/postgres/batch"
 	"openreplay/backend/pkg/db/postgres/pool"
@@ -22,7 +23,7 @@ type Conn struct {
 	chConn  CH
 }
 
-func NewConn(log logger.Logger, pool pool.Pool, ch CH) *Conn {
+func NewConn(log logger.Logger, pool pool.Pool, ch CH, metrics database.Database) *Conn {
 	if pool == nil {
 		log.Fatal(context.Background(), "pg pool is empty")
 	}
@@ -30,8 +31,8 @@ func NewConn(log logger.Logger, pool pool.Pool, ch CH) *Conn {
 		log:     log,
 		Pool:    pool,
 		chConn:  ch,
-		bulks:   NewBulkSet(log, pool),
-		batches: batch.NewBatchSet(log, pool),
+		bulks:   NewBulkSet(log, pool, metrics),
+		batches: batch.NewBatchSet(log, pool, metrics),
 	}
 }
 

backend/pkg/db/postgres/events.go

@@ -181,11 +181,6 @@ func (conn *Conn) InsertWebErrorEvent(sess *sessions.Session, e *types.ErrorEven
 	if err := conn.bulks.Get("webErrorEvents").Append(sess.SessionID, truncSqIdx(e.MessageID), e.Timestamp, errorID); err != nil {
 		conn.log.Error(sessCtx, "insert web error event err: %s", err)
 	}
-	for key, value := range e.Tags {
-		if err := conn.bulks.Get("webErrorTags").Append(sess.SessionID, truncSqIdx(e.MessageID), errorID, key, value); err != nil {
-			conn.log.Error(sessCtx, "insert web error token err: %s", err)
-		}
-	}
 	return nil
 }
 

backend/pkg/db/postgres/pool/pool.go

@@ -23,58 +23,12 @@ type Pool interface {
 }
 
 type poolImpl struct {
-	url  string
-	conn *pgxpool.Pool
+	url     string
+	conn    *pgxpool.Pool
+	metrics database.Database
 }
 
-func (p *poolImpl) Query(sql string, args ...interface{}) (pgx.Rows, error) {
-	start := time.Now()
-	res, err := p.conn.Query(getTimeoutContext(), sql, args...)
-	method, table := methodName(sql)
-	database.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), method, table)
-	database.IncreaseTotalRequests(method, table)
-	return res, err
-}
-
-func (p *poolImpl) QueryRow(sql string, args ...interface{}) pgx.Row {
-	start := time.Now()
-	res := p.conn.QueryRow(getTimeoutContext(), sql, args...)
-	method, table := methodName(sql)
-	database.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), method, table)
-	database.IncreaseTotalRequests(method, table)
-	return res
-}
-
-func (p *poolImpl) Exec(sql string, arguments ...interface{}) error {
-	start := time.Now()
-	_, err := p.conn.Exec(getTimeoutContext(), sql, arguments...)
-	method, table := methodName(sql)
-	database.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), method, table)
-	database.IncreaseTotalRequests(method, table)
-	return err
-}
-
-func (p *poolImpl) SendBatch(b *pgx.Batch) pgx.BatchResults {
-	start := time.Now()
-	res := p.conn.SendBatch(getTimeoutContext(), b)
-	database.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "sendBatch", "")
-	database.IncreaseTotalRequests("sendBatch", "")
-	return res
-}
-
-func (p *poolImpl) Begin() (*Tx, error) {
-	start := time.Now()
-	tx, err := p.conn.Begin(context.Background())
-	database.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "begin", "")
-	database.IncreaseTotalRequests("begin", "")
-	return &Tx{tx}, err
-}
-
-func (p *poolImpl) Close() {
-	p.conn.Close()
-}
-
-func New(url string) (Pool, error) {
+func New(metrics database.Database, url string) (Pool, error) {
 	if url == "" {
 		return nil, errors.New("pg connection url is empty")
 	}
@@ -83,24 +37,73 @@ func New(url string) (Pool, error) {
 		return nil, fmt.Errorf("pgxpool.Connect error: %v", err)
 	}
 	res := &poolImpl{
-		url:  url,
-		conn: conn,
+		url:     url,
+		conn:    conn,
+		metrics: metrics,
 	}
 	return res, nil
 }
 
+func (p *poolImpl) Query(sql string, args ...interface{}) (pgx.Rows, error) {
+	start := time.Now()
+	res, err := p.conn.Query(getTimeoutContext(), sql, args...)
+	method, table := methodName(sql)
+	p.metrics.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), method, table)
+	p.metrics.IncreaseTotalRequests(method, table)
+	return res, err
+}
+
+func (p *poolImpl) QueryRow(sql string, args ...interface{}) pgx.Row {
+	start := time.Now()
+	res := p.conn.QueryRow(getTimeoutContext(), sql, args...)
+	method, table := methodName(sql)
+	p.metrics.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), method, table)
+	p.metrics.IncreaseTotalRequests(method, table)
+	return res
+}
+
+func (p *poolImpl) Exec(sql string, arguments ...interface{}) error {
+	start := time.Now()
+	_, err := p.conn.Exec(getTimeoutContext(), sql, arguments...)
+	method, table := methodName(sql)
+	p.metrics.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), method, table)
+	p.metrics.IncreaseTotalRequests(method, table)
+	return err
+}
+
+func (p *poolImpl) SendBatch(b *pgx.Batch) pgx.BatchResults {
+	start := time.Now()
+	res := p.conn.SendBatch(getTimeoutContext(), b)
+	p.metrics.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "sendBatch", "")
+	p.metrics.IncreaseTotalRequests("sendBatch", "")
+	return res
+}
+
+func (p *poolImpl) Begin() (*Tx, error) {
+	start := time.Now()
+	tx, err := p.conn.Begin(context.Background())
+	p.metrics.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "begin", "")
+	p.metrics.IncreaseTotalRequests("begin", "")
+	return &Tx{tx, p.metrics}, err
+}
+
+func (p *poolImpl) Close() {
+	p.conn.Close()
+}
+
 // TX - start
 
 type Tx struct {
 	pgx.Tx
+	metrics database.Database
 }
 
 func (tx *Tx) TxExec(sql string, args ...interface{}) error {
 	start := time.Now()
 	_, err := tx.Exec(context.Background(), sql, args...)
 	method, table := methodName(sql)
-	database.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), method, table)
-	database.IncreaseTotalRequests(method, table)
+	tx.metrics.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), method, table)
+	tx.metrics.IncreaseTotalRequests(method, table)
 	return err
 }
 
@@ -108,24 +111,24 @@ func (tx *Tx) TxQueryRow(sql string, args ...interface{}) pgx.Row {
 	start := time.Now()
 	res := tx.QueryRow(context.Background(), sql, args...)
 	method, table := methodName(sql)
-	database.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), method, table)
-	database.IncreaseTotalRequests(method, table)
+	tx.metrics.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), method, table)
+	tx.metrics.IncreaseTotalRequests(method, table)
 	return res
 }
 
 func (tx *Tx) TxRollback() error {
 	start := time.Now()
 	err := tx.Rollback(context.Background())
-	database.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "rollback", "")
-	database.IncreaseTotalRequests("rollback", "")
+	tx.metrics.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "rollback", "")
+	tx.metrics.IncreaseTotalRequests("rollback", "")
 	return err
 }
 
 func (tx *Tx) TxCommit() error {
 	start := time.Now()
 	err := tx.Commit(context.Background())
-	database.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "commit", "")
-	database.IncreaseTotalRequests("commit", "")
+	tx.metrics.RecordRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "commit", "")
+	tx.metrics.IncreaseTotalRequests("commit", "")
 	return err
 }
 

backend/pkg/db/types/error-event.go

@@ -61,7 +61,6 @@ func parseTags(tagsJSON string) (tags map[string]*string, err error) {
 }
 
 func WrapJSException(m *JSException) (*ErrorEvent, error) {
-	meta, err := parseTags(m.Metadata)
 	return &ErrorEvent{
 		MessageID:  m.Meta().Index,
 		Timestamp:  m.Meta().Timestamp,
@@ -69,9 +68,8 @@ func WrapJSException(m *JSException) (*ErrorEvent, error) {
 		Name:       m.Name,
 		Message:    m.Message,
 		Payload:    m.Payload,
-		Tags:       meta,
 		OriginType: m.TypeID(),
-	}, err
+	}, nil
 }
 
 func WrapIntegrationEvent(m *IntegrationEvent) *ErrorEvent {

backend/pkg/integrations/builder.go

@@ -2,6 +2,7 @@ package integrations
 
 import (
 	"openreplay/backend/pkg/integrations/service"
+	"openreplay/backend/pkg/metrics/database"
 	"openreplay/backend/pkg/metrics/web"
 	"openreplay/backend/pkg/server/tracer"
 	"time"
@@ -23,7 +24,7 @@ type ServiceBuilder struct {
 	IntegrationsAPI api.Handlers
 }
 
-func NewServiceBuilder(log logger.Logger, cfg *integrations.Config, webMetrics web.Web, pgconn pool.Pool) (*ServiceBuilder, error) {
+func NewServiceBuilder(log logger.Logger, cfg *integrations.Config, webMetrics web.Web, dbMetrics database.Database, pgconn pool.Pool) (*ServiceBuilder, error) {
 	objStore, err := store.NewStore(&cfg.ObjectsConfig)
 	if err != nil {
 		return nil, err
@@ -37,12 +38,12 @@ func NewServiceBuilder(log logger.Logger, cfg *integrations.Config, webMetrics w
 	if err != nil {
 		return nil, err
 	}
-	auditrail, err := tracer.NewTracer(log, pgconn)
+	auditrail, err := tracer.NewTracer(log, pgconn, dbMetrics)
 	if err != nil {
 		return nil, err
 	}
 	builder := &ServiceBuilder{
-		Auth:            auth.NewAuth(log, cfg.JWTSecret, "", pgconn, nil),
+		Auth:            auth.NewAuth(log, cfg.JWTSecret, "", pgconn, nil, api.NoPrefix),
 		RateLimiter:     limiter.NewUserRateLimiter(10, 30, 1*time.Minute, 5*time.Minute),
 		AuditTrail:      auditrail,
 		IntegrationsAPI: handlers,

backend/pkg/messages/iterator-sink.go

@@ -8,11 +8,13 @@ import (
 type sinkIteratorImpl struct {
 	coreIterator MessageIterator
 	handler      MessageHandler
+	metrics      sink.Sink
 }
 
-func NewSinkMessageIterator(log logger.Logger, messageHandler MessageHandler, messageFilter []int, autoDecode bool) MessageIterator {
+func NewSinkMessageIterator(log logger.Logger, messageHandler MessageHandler, messageFilter []int, autoDecode bool, metrics sink.Sink) MessageIterator {
 	iter := &sinkIteratorImpl{
 		handler: messageHandler,
+		metrics: metrics,
 	}
 	iter.coreIterator = NewMessageIterator(log, iter.handle, messageFilter, autoDecode)
 	return iter
@@ -23,8 +25,8 @@ func (i *sinkIteratorImpl) handle(message Message) {
 }
 
 func (i *sinkIteratorImpl) Iterate(batchData []byte, batchInfo *BatchInfo) {
-	sink.RecordBatchSize(float64(len(batchData)))
-	sink.IncreaseTotalBatches()
+	i.metrics.RecordBatchSize(float64(len(batchData)))
+	i.metrics.IncreaseTotalBatches()
 	// Call core iterator
 	i.coreIterator.Iterate(batchData, batchInfo)
 	// Send batch end signal

backend/pkg/metrics/analytics/analytics.go

@@ -1,22 +0,0 @@
-package analytics
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-
-	"openreplay/backend/pkg/metrics/common"
-)
-
-var cardCreated = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "card",
-		Name:      "created",
-		Help:      "Histogram for tracking card creation",
-		Buckets:   common.DefaultBuckets,
-	},
-)
-
-func List() []prometheus.Collector {
-	return []prometheus.Collector{
-		cardCreated,
-	}
-}

backend/pkg/metrics/assets/metrics.go

@@ -2,71 +2,22 @@ package assets
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
-	"openreplay/backend/pkg/metrics/common"
-	"strconv"
 )
 
-var assetsProcessedSessions = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "assets",
-		Name:      "processed_total",
-		Help:      "A counter displaying the total count of processed assets.",
-	},
-)
-
-func IncreaseProcessesSessions() {
-	assetsProcessedSessions.Inc()
+type Assets interface {
+	IncreaseProcessesSessions()
+	IncreaseSavedSessions()
+	RecordDownloadDuration(durMillis float64, code int)
+	RecordUploadDuration(durMillis float64, isFailed bool)
+	List() []prometheus.Collector
 }
 
-var assetsSavedSessions = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "assets",
-		Name:      "saved_total",
-		Help:      "A counter displaying the total number of cached assets.",
-	},
-)
+type assetsImpl struct{}
 
-func IncreaseSavedSessions() {
-	assetsSavedSessions.Inc()
-}
+func New(serviceName string) Assets { return &assetsImpl{} }
 
-var assetsDownloadDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "assets",
-		Name:      "download_duration_seconds",
-		Help:      "A histogram displaying the duration of downloading for each asset in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"response_code"},
-)
-
-func RecordDownloadDuration(durMillis float64, code int) {
-	assetsDownloadDuration.WithLabelValues(strconv.Itoa(code)).Observe(durMillis / 1000.0)
-}
-
-var assetsUploadDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "assets",
-		Name:      "upload_s3_duration_seconds",
-		Help:      "A histogram displaying the duration of uploading to s3 for each asset in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"failed"},
-)
-
-func RecordUploadDuration(durMillis float64, isFailed bool) {
-	failed := "false"
-	if isFailed {
-		failed = "true"
-	}
-	assetsUploadDuration.WithLabelValues(failed).Observe(durMillis / 1000.0)
-}
-
-func List() []prometheus.Collector {
-	return []prometheus.Collector{
-		assetsProcessedSessions,
-		assetsSavedSessions,
-		assetsDownloadDuration,
-		assetsUploadDuration,
-	}
-}
+func (a *assetsImpl) List() []prometheus.Collector                          { return []prometheus.Collector{} }
+func (a *assetsImpl) IncreaseProcessesSessions()                            {}
+func (a *assetsImpl) IncreaseSavedSessions()                                {}
+func (a *assetsImpl) RecordDownloadDuration(durMillis float64, code int)    {}
+func (a *assetsImpl) RecordUploadDuration(durMillis float64, isFailed bool) {}

backend/pkg/metrics/canvas/metrics.go

@@ -2,7 +2,6 @@ package canvas
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
-	"openreplay/backend/pkg/metrics/common"
 )
 
 type Canvas interface {
@@ -18,175 +17,17 @@ type Canvas interface {
 	List() []prometheus.Collector
 }
 
-type canvasImpl struct {
-	canvasesImageSize            prometheus.Histogram
-	canvasesTotalSavedImages     prometheus.Counter
-	canvasesImagesPerCanvas      prometheus.Histogram
-	canvasesCanvasesPerSession   prometheus.Histogram
-	canvasesPreparingDuration    prometheus.Histogram
-	canvasesTotalCreatedArchives prometheus.Counter
-	canvasesArchivingDuration    prometheus.Histogram
-	canvasesArchiveSize          prometheus.Histogram
-	canvasesUploadingDuration    prometheus.Histogram
-}
+type canvasImpl struct{}
 
-func New(serviceName string) Canvas {
-	return &canvasImpl{
-		canvasesImageSize:            newImageSizeMetric(serviceName),
-		canvasesTotalSavedImages:     newTotalSavedImages(serviceName),
-		canvasesImagesPerCanvas:      newImagesPerCanvas(serviceName),
-		canvasesCanvasesPerSession:   newCanvasesPerSession(serviceName),
-		canvasesPreparingDuration:    newPreparingDuration(serviceName),
-		canvasesTotalCreatedArchives: newTotalCreatedArchives(serviceName),
-		canvasesArchivingDuration:    newArchivingDuration(serviceName),
-		canvasesArchiveSize:          newArchiveSize(serviceName),
-		canvasesUploadingDuration:    newUploadingDuration(serviceName),
-	}
-}
+func New(serviceName string) Canvas { return &canvasImpl{} }
 
-func (c *canvasImpl) List() []prometheus.Collector {
-	return []prometheus.Collector{
-		c.canvasesImageSize,
-		c.canvasesTotalSavedImages,
-		c.canvasesImagesPerCanvas,
-		c.canvasesCanvasesPerSession,
-		c.canvasesPreparingDuration,
-		c.canvasesTotalCreatedArchives,
-		c.canvasesArchivingDuration,
-		c.canvasesArchiveSize,
-		c.canvasesUploadingDuration,
-	}
-}
-
-func newImageSizeMetric(serviceName string) prometheus.Histogram {
-	return prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: serviceName,
-			Name:      "image_size_bytes",
-			Help:      "A histogram displaying the size of each canvas image in bytes.",
-			Buckets:   common.DefaultSizeBuckets,
-		},
-	)
-}
-
-func (c *canvasImpl) RecordCanvasImageSize(size float64) {
-	c.canvasesImageSize.Observe(size)
-}
-
-func newTotalSavedImages(serviceName string) prometheus.Counter {
-	return prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: serviceName,
-			Name:      "total_saved_images",
-			Help:      "A counter displaying the total number of saved images.",
-		},
-	)
-}
-
-func (c *canvasImpl) IncreaseTotalSavedImages() {
-	c.canvasesTotalSavedImages.Inc()
-}
-
-func newImagesPerCanvas(serviceName string) prometheus.Histogram {
-	return prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: serviceName,
-			Name:      "images_per_canvas",
-			Help:      "A histogram displaying the number of images per canvas.",
-			Buckets:   common.DefaultBuckets,
-		},
-	)
-}
-
-func (c *canvasImpl) RecordImagesPerCanvas(number float64) {
-	c.canvasesImagesPerCanvas.Observe(number)
-}
-
-func newCanvasesPerSession(serviceName string) prometheus.Histogram {
-	return prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: serviceName,
-			Name:      "canvases_per_session",
-			Help:      "A histogram displaying the number of canvases per session.",
-			Buckets:   common.DefaultBuckets,
-		},
-	)
-}
-
-func (c *canvasImpl) RecordCanvasesPerSession(number float64) {
-	c.canvasesCanvasesPerSession.Observe(number)
-}
-
-func newPreparingDuration(serviceName string) prometheus.Histogram {
-	return prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: serviceName,
-			Name:      "preparing_duration_seconds",
-			Help:      "A histogram displaying the duration of preparing the list of canvases for each session in seconds.",
-			Buckets:   common.DefaultDurationBuckets,
-		},
-	)
-}
-
-func (c *canvasImpl) RecordPreparingDuration(duration float64) {
-	c.canvasesPreparingDuration.Observe(duration)
-}
-
-func newTotalCreatedArchives(serviceName string) prometheus.Counter {
-	return prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: serviceName,
-			Name:      "total_created_archives",
-			Help:      "A counter displaying the total number of created canvas archives.",
-		},
-	)
-}
-
-func (c *canvasImpl) IncreaseTotalCreatedArchives() {
-	c.canvasesTotalCreatedArchives.Inc()
-}
-
-func newArchivingDuration(serviceName string) prometheus.Histogram {
-	return prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: serviceName,
-			Name:      "archiving_duration_seconds",
-			Help:      "A histogram displaying the duration of archiving for each canvas in seconds.",
-			Buckets:   common.DefaultDurationBuckets,
-		},
-	)
-}
-
-func (c *canvasImpl) RecordArchivingDuration(duration float64) {
-	c.canvasesArchivingDuration.Observe(duration)
-}
-
-func newArchiveSize(serviceName string) prometheus.Histogram {
-	return prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: serviceName,
-			Name:      "archive_size_bytes",
-			Help:      "A histogram displaying the size of each canvas archive in bytes.",
-			Buckets:   common.DefaultSizeBuckets,
-		},
-	)
-}
-
-func (c *canvasImpl) RecordArchiveSize(size float64) {
-	c.canvasesArchiveSize.Observe(size)
-}
-
-func newUploadingDuration(serviceName string) prometheus.Histogram {
-	return prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: serviceName,
-			Name:      "uploading_duration_seconds",
-			Help:      "A histogram displaying the duration of uploading for each canvas in seconds.",
-			Buckets:   common.DefaultDurationBuckets,
-		},
-	)
-}
-
-func (c *canvasImpl) RecordUploadingDuration(duration float64) {
-	c.canvasesUploadingDuration.Observe(duration)
-}
+func (c *canvasImpl) List() []prometheus.Collector             { return []prometheus.Collector{} }
+func (c *canvasImpl) RecordCanvasImageSize(size float64)       {}
+func (c *canvasImpl) IncreaseTotalSavedImages()                {}
+func (c *canvasImpl) RecordImagesPerCanvas(number float64)     {}
+func (c *canvasImpl) RecordCanvasesPerSession(number float64)  {}
+func (c *canvasImpl) RecordPreparingDuration(duration float64) {}
+func (c *canvasImpl) IncreaseTotalCreatedArchives()            {}
+func (c *canvasImpl) RecordArchivingDuration(duration float64) {}
+func (c *canvasImpl) RecordArchiveSize(size float64)           {}
+func (c *canvasImpl) RecordUploadingDuration(duration float64) {}

backend/pkg/metrics/database/metrics.go

@@ -2,141 +2,32 @@ package database
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
-	"openreplay/backend/pkg/metrics/common"
 )
 
-var dbBatchElements = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "db",
-		Name:      "batch_size_elements",
-		Help:      "A histogram displaying the number of SQL commands in each batch.",
-		Buckets:   common.DefaultBuckets,
-	},
-)
-
-func RecordBatchElements(number float64) {
-	dbBatchElements.Observe(number)
+type Database interface {
+	RecordBatchElements(number float64)
+	RecordBatchInsertDuration(durMillis float64)
+	RecordBulkSize(size float64, db, table string)
+	RecordBulkElements(size float64, db, table string)
+	RecordBulkInsertDuration(durMillis float64, db, table string)
+	RecordRequestDuration(durMillis float64, method, table string)
+	IncreaseTotalRequests(method, table string)
+	IncreaseRedisRequests(method, table string)
+	RecordRedisRequestDuration(durMillis float64, method, table string)
+	List() []prometheus.Collector
 }
 
-var dbBatchInsertDuration = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "db",
-		Name:      "batch_insert_duration_seconds",
-		Help:      "A histogram displaying the duration of batch inserts in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-)
+type databaseImpl struct{}
 
-func RecordBatchInsertDuration(durMillis float64) {
-	dbBatchInsertDuration.Observe(durMillis / 1000.0)
-}
+func New(serviceName string) Database { return &databaseImpl{} }
 
-var dbBulkSize = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "db",
-		Name:      "bulk_size_bytes",
-		Help:      "A histogram displaying the bulk size in bytes.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-	[]string{"db", "table"},
-)
-
-func RecordBulkSize(size float64, db, table string) {
-	dbBulkSize.WithLabelValues(db, table).Observe(size)
-}
-
-var dbBulkElements = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "db",
-		Name:      "bulk_size_elements",
-		Help:      "A histogram displaying the size of data set in each bulk.",
-		Buckets:   common.DefaultBuckets,
-	},
-	[]string{"db", "table"},
-)
-
-func RecordBulkElements(size float64, db, table string) {
-	dbBulkElements.WithLabelValues(db, table).Observe(size)
-}
-
-var dbBulkInsertDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "db",
-		Name:      "bulk_insert_duration_seconds",
-		Help:      "A histogram displaying the duration of bulk inserts in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"db", "table"},
-)
-
-func RecordBulkInsertDuration(durMillis float64, db, table string) {
-	dbBulkInsertDuration.WithLabelValues(db, table).Observe(durMillis / 1000.0)
-}
-
-var dbRequestDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "db",
-		Name:      "request_duration_seconds",
-		Help:      "A histogram displaying the duration of each sql request in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"method", "table"},
-)
-
-func RecordRequestDuration(durMillis float64, method, table string) {
-	dbRequestDuration.WithLabelValues(method, table).Observe(durMillis / 1000.0)
-}
-
-var dbTotalRequests = prometheus.NewCounterVec(
-	prometheus.CounterOpts{
-		Namespace: "db",
-		Name:      "requests_total",
-		Help:      "A counter showing the total number of all SQL requests.",
-	},
-	[]string{"method", "table"},
-)
-
-func IncreaseTotalRequests(method, table string) {
-	dbTotalRequests.WithLabelValues(method, table).Inc()
-}
-
-var cacheRedisRequests = prometheus.NewCounterVec(
-	prometheus.CounterOpts{
-		Namespace: "cache",
-		Name:      "redis_requests_total",
-		Help:      "A counter showing the total number of all Redis requests.",
-	},
-	[]string{"method", "table"},
-)
-
-func IncreaseRedisRequests(method, table string) {
-	cacheRedisRequests.WithLabelValues(method, table).Inc()
-}
-
-var cacheRedisRequestDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "cache",
-		Name:      "redis_request_duration_seconds",
-		Help:      "A histogram displaying the duration of each Redis request in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"method", "table"},
-)
-
-func RecordRedisRequestDuration(durMillis float64, method, table string) {
-	cacheRedisRequestDuration.WithLabelValues(method, table).Observe(durMillis / 1000.0)
-}
-
-func List() []prometheus.Collector {
-	return []prometheus.Collector{
-		dbBatchElements,
-		dbBatchInsertDuration,
-		dbBulkSize,
-		dbBulkElements,
-		dbBulkInsertDuration,
-		dbRequestDuration,
-		dbTotalRequests,
-		cacheRedisRequests,
-		cacheRedisRequestDuration,
-	}
-}
+func (d *databaseImpl) List() []prometheus.Collector                                       { return []prometheus.Collector{} }
+func (d *databaseImpl) RecordBatchElements(number float64)                                 {}
+func (d *databaseImpl) RecordBatchInsertDuration(durMillis float64)                        {}
+func (d *databaseImpl) RecordBulkSize(size float64, db, table string)                      {}
+func (d *databaseImpl) RecordBulkElements(size float64, db, table string)                  {}
+func (d *databaseImpl) RecordBulkInsertDuration(durMillis float64, db, table string)       {}
+func (d *databaseImpl) RecordRequestDuration(durMillis float64, method, table string)      {}
+func (d *databaseImpl) IncreaseTotalRequests(method, table string)                         {}
+func (d *databaseImpl) IncreaseRedisRequests(method, table string)                         {}
+func (d *databaseImpl) RecordRedisRequestDuration(durMillis float64, method, table string) {}

backend/pkg/metrics/ender/metrics.go

@@ -2,50 +2,20 @@ package ender
 
 import "github.com/prometheus/client_golang/prometheus"
 
-var enderActiveSessions = prometheus.NewGauge(
-	prometheus.GaugeOpts{
-		Namespace: "ender",
-		Name:      "sessions_active",
-		Help:      "A gauge displaying the number of active (live) sessions.",
-	},
-)
-
-func IncreaseActiveSessions() {
-	enderActiveSessions.Inc()
+type Ender interface {
+	IncreaseActiveSessions()
+	DecreaseActiveSessions()
+	IncreaseClosedSessions()
+	IncreaseTotalSessions()
+	List() []prometheus.Collector
 }
 
-func DecreaseActiveSessions() {
-	enderActiveSessions.Dec()
-}
+type enderImpl struct{}
 
-var enderClosedSessions = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "ender",
-		Name:      "sessions_closed",
-		Help:      "A counter displaying the number of closed sessions (sent SessionEnd).",
-	},
-)
+func New(serviceName string) Ender { return &enderImpl{} }
 
-func IncreaseClosedSessions() {
-	enderClosedSessions.Inc()
-}
-
-var enderTotalSessions = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "ender",
-		Name:      "sessions_total",
-		Help:      "A counter displaying the number of all processed sessions.",
-	},
-)
-
-func IncreaseTotalSessions() {
-	enderTotalSessions.Inc()
-}
-
-func List() []prometheus.Collector {
-	return []prometheus.Collector{
-		enderActiveSessions,
-		enderClosedSessions,
-		enderTotalSessions,
-	}
-}
+func (e *enderImpl) List() []prometheus.Collector { return []prometheus.Collector{} }
+func (e *enderImpl) IncreaseActiveSessions()      {}
+func (e *enderImpl) DecreaseActiveSessions()      {}
+func (e *enderImpl) IncreaseClosedSessions()      {}
+func (e *enderImpl) IncreaseTotalSessions()       {}

backend/pkg/metrics/heuristics/metrics.go

@@ -2,65 +2,16 @@ package heuristics
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
-	"openreplay/backend/pkg/metrics/common"
-	"strconv"
 )
 
-var heuristicsTotalEvents = prometheus.NewCounterVec(
-	prometheus.CounterOpts{
-		Namespace: "heuristics",
-		Name:      "events_total",
-		Help:      "A counter displaying the number of all processed events",
-	},
-	[]string{"type"},
-)
-
-func IncreaseTotalEvents(eventType string) {
-	heuristicsTotalEvents.WithLabelValues(eventType).Inc()
+type Heuristics interface {
+	IncreaseTotalEvents(eventType string)
+	List() []prometheus.Collector
 }
 
-var heuristicsRequestSize = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "heuristics",
-		Name:      "request_size_bytes",
-		Help:      "A histogram displaying the size of each HTTP request in bytes.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-	[]string{"url", "response_code"},
-)
+type heuristicsImpl struct{}
 
-func RecordRequestSize(size float64, url string, code int) {
-	heuristicsRequestSize.WithLabelValues(url, strconv.Itoa(code)).Observe(size)
-}
+func New(serviceName string) Heuristics { return &heuristicsImpl{} }
 
-var heuristicsRequestDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "heuristics",
-		Name:      "request_duration_seconds",
-		Help:      "A histogram displaying the duration of each HTTP request in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"url", "response_code"},
-)
-
-func RecordRequestDuration(durMillis float64, url string, code int) {
-	heuristicsRequestDuration.WithLabelValues(url, strconv.Itoa(code)).Observe(durMillis / 1000.0)
-}
-
-var heuristicsTotalRequests = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "heuristics",
-		Name:      "requests_total",
-		Help:      "A counter displaying the number all HTTP requests.",
-	},
-)
-
-func IncreaseTotalRequests() {
-	heuristicsTotalRequests.Inc()
-}
-
-func List() []prometheus.Collector {
-	return []prometheus.Collector{
-		heuristicsTotalEvents,
-	}
-}
+func (h *heuristicsImpl) List() []prometheus.Collector         { return []prometheus.Collector{} }
+func (h *heuristicsImpl) IncreaseTotalEvents(eventType string) {}

backend/pkg/metrics/images/metrics.go

@@ -2,154 +2,32 @@ package images
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
-	"openreplay/backend/pkg/metrics/common"
 )
 
-var storageSessionSize = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "session_size_bytes",
-		Help:      "A histogram displaying the size of each session file in bytes prior to any manipulation.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionSize(fileSize float64, fileType string) {
-	storageSessionSize.WithLabelValues(fileType).Observe(fileSize)
+type Images interface {
+	RecordOriginalArchiveSize(size float64)
+	RecordOriginalArchiveExtractionDuration(duration float64)
+	IncreaseTotalSavedArchives()
+	RecordSavingImageDuration(duration float64)
+	IncreaseTotalSavedImages()
+	IncreaseTotalCreatedArchives()
+	RecordArchivingDuration(duration float64)
+	RecordArchiveSize(size float64)
+	RecordUploadingDuration(duration float64)
+	List() []prometheus.Collector
 }
 
-var storageTotalSessions = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "storage",
-		Name:      "sessions_total",
-		Help:      "A counter displaying the total number of all processed sessions.",
-	},
-)
+type imagesImpl struct{}
 
-func IncreaseStorageTotalSessions() {
-	storageTotalSessions.Inc()
-}
+func New(serviceName string) Images { return &imagesImpl{} }
 
-var storageSkippedSessionSize = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "session_size_bytes",
-		Help:      "A histogram displaying the size of each skipped session file in bytes.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSkippedSessionSize(fileSize float64, fileType string) {
-	storageSkippedSessionSize.WithLabelValues(fileType).Observe(fileSize)
-}
-
-var storageTotalSkippedSessions = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "storage",
-		Name:      "sessions_skipped_total",
-		Help:      "A counter displaying the total number of all skipped sessions because of the size limits.",
-	},
-)
-
-func IncreaseStorageTotalSkippedSessions() {
-	storageTotalSkippedSessions.Inc()
-}
-
-var storageSessionReadDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "read_duration_seconds",
-		Help:      "A histogram displaying the duration of reading for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionReadDuration(durMillis float64, fileType string) {
-	storageSessionReadDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionSortDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "sort_duration_seconds",
-		Help:      "A histogram displaying the duration of sorting for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionSortDuration(durMillis float64, fileType string) {
-	storageSessionSortDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionEncryptionDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "encryption_duration_seconds",
-		Help:      "A histogram displaying the duration of encoding for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionEncryptionDuration(durMillis float64, fileType string) {
-	storageSessionEncryptionDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionCompressDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "compress_duration_seconds",
-		Help:      "A histogram displaying the duration of compressing for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionCompressDuration(durMillis float64, fileType string) {
-	storageSessionCompressDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionUploadDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "upload_duration_seconds",
-		Help:      "A histogram displaying the duration of uploading to s3 for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionUploadDuration(durMillis float64, fileType string) {
-	storageSessionUploadDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionCompressionRatio = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "compression_ratio",
-		Help:      "A histogram displaying the compression ratio of mob files for each session.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionCompressionRatio(ratio float64, fileType string) {
-	storageSessionCompressionRatio.WithLabelValues(fileType).Observe(ratio)
-}
-
-func List() []prometheus.Collector {
-	return []prometheus.Collector{
-		storageSessionSize,
-		storageTotalSessions,
-		storageSessionReadDuration,
-		storageSessionSortDuration,
-		storageSessionEncryptionDuration,
-		storageSessionCompressDuration,
-		storageSessionUploadDuration,
-		storageSessionCompressionRatio,
-	}
-}
+func (i *imagesImpl) List() []prometheus.Collector                             { return []prometheus.Collector{} }
+func (i *imagesImpl) RecordOriginalArchiveSize(size float64)                   {}
+func (i *imagesImpl) RecordOriginalArchiveExtractionDuration(duration float64) {}
+func (i *imagesImpl) IncreaseTotalSavedArchives()                              {}
+func (i *imagesImpl) RecordSavingImageDuration(duration float64)               {}
+func (i *imagesImpl) IncreaseTotalSavedImages()                                {}
+func (i *imagesImpl) IncreaseTotalCreatedArchives()                            {}
+func (i *imagesImpl) RecordArchivingDuration(duration float64)                 {}
+func (i *imagesImpl) RecordArchiveSize(size float64)                           {}
+func (i *imagesImpl) RecordUploadingDuration(duration float64)                 {}

backend/pkg/metrics/sink/metrics.go

@@ -2,184 +2,40 @@ package sink
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
-	"openreplay/backend/pkg/metrics/common"
 )
 
-var sinkMessageSize = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "sink",
-		Name:      "message_size_bytes",
-		Help:      "A histogram displaying the size of each message in bytes.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-)
-
-func RecordMessageSize(size float64) {
-	sinkMessageSize.Observe(size)
+type Sink interface {
+	RecordMessageSize(size float64)
+	IncreaseWrittenMessages()
+	IncreaseTotalMessages()
+	RecordBatchSize(size float64)
+	IncreaseTotalBatches()
+	RecordWrittenBytes(size float64, fileType string)
+	IncreaseTotalWrittenBytes(size float64, fileType string)
+	IncreaseCachedAssets()
+	DecreaseCachedAssets()
+	IncreaseSkippedAssets()
+	IncreaseTotalAssets()
+	RecordAssetSize(size float64)
+	RecordProcessAssetDuration(durMillis float64)
+	List() []prometheus.Collector
 }
 
-var sinkWrittenMessages = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "sink",
-		Name:      "messages_written",
-		Help:      "A counter displaying the total number of all written messages.",
-	},
-)
+type sinkImpl struct{}
 
-func IncreaseWrittenMessages() {
-	sinkWrittenMessages.Inc()
-}
+func New(serviceName string) Sink { return &sinkImpl{} }
 
-var sinkTotalMessages = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "sink",
-		Name:      "messages_total",
-		Help:      "A counter displaying the total number of all processed messages.",
-	},
-)
-
-func IncreaseTotalMessages() {
-	sinkTotalMessages.Inc()
-}
-
-var sinkBatchSize = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "sink",
-		Name:      "batch_size_bytes",
-		Help:      "A histogram displaying the size of each batch in bytes.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-)
-
-func RecordBatchSize(size float64) {
-	sinkBatchSize.Observe(size)
-}
-
-var sinkTotalBatches = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "sink",
-		Name:      "batches_total",
-		Help:      "A counter displaying the total number of all written batches.",
-	},
-)
-
-func IncreaseTotalBatches() {
-	sinkTotalBatches.Inc()
-}
-
-var sinkWrittenBytes = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "sink",
-		Name:      "written_bytes",
-		Help:      "A histogram displaying the size of buffer in bytes written to session file.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordWrittenBytes(size float64, fileType string) {
-	if size == 0 {
-		return
-	}
-	sinkWrittenBytes.WithLabelValues(fileType).Observe(size)
-	IncreaseTotalWrittenBytes(size, fileType)
-}
-
-var sinkTotalWrittenBytes = prometheus.NewCounterVec(
-	prometheus.CounterOpts{
-		Namespace: "sink",
-		Name:      "written_bytes_total",
-		Help:      "A counter displaying the total number of bytes written to all session files.",
-	},
-	[]string{"file_type"},
-)
-
-func IncreaseTotalWrittenBytes(size float64, fileType string) {
-	if size == 0 {
-		return
-	}
-	sinkTotalWrittenBytes.WithLabelValues(fileType).Add(size)
-}
-
-var sinkCachedAssets = prometheus.NewGauge(
-	prometheus.GaugeOpts{
-		Namespace: "sink",
-		Name:      "assets_cached",
-		Help:      "A gauge displaying the current number of cached assets.",
-	},
-)
-
-func IncreaseCachedAssets() {
-	sinkCachedAssets.Inc()
-}
-
-func DecreaseCachedAssets() {
-	sinkCachedAssets.Dec()
-}
-
-var sinkSkippedAssets = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "sink",
-		Name:      "assets_skipped",
-		Help:      "A counter displaying the total number of all skipped assets.",
-	},
-)
-
-func IncreaseSkippedAssets() {
-	sinkSkippedAssets.Inc()
-}
-
-var sinkTotalAssets = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "sink",
-		Name:      "assets_total",
-		Help:      "A counter displaying the total number of all processed assets.",
-	},
-)
-
-func IncreaseTotalAssets() {
-	sinkTotalAssets.Inc()
-}
-
-var sinkAssetSize = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "sink",
-		Name:      "asset_size_bytes",
-		Help:      "A histogram displaying the size of each asset in bytes.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-)
-
-func RecordAssetSize(size float64) {
-	sinkAssetSize.Observe(size)
-}
-
-var sinkProcessAssetDuration = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "sink",
-		Name:      "asset_process_duration_seconds",
-		Help:      "A histogram displaying the duration of processing for each asset in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-)
-
-func RecordProcessAssetDuration(durMillis float64) {
-	sinkProcessAssetDuration.Observe(durMillis / 1000.0)
-}
-
-func List() []prometheus.Collector {
-	return []prometheus.Collector{
-		sinkMessageSize,
-		sinkWrittenMessages,
-		sinkTotalMessages,
-		sinkBatchSize,
-		sinkTotalBatches,
-		sinkWrittenBytes,
-		sinkTotalWrittenBytes,
-		sinkCachedAssets,
-		sinkSkippedAssets,
-		sinkTotalAssets,
-		sinkAssetSize,
-		sinkProcessAssetDuration,
-	}
-}
+func (s *sinkImpl) List() []prometheus.Collector                            { return []prometheus.Collector{} }
+func (s *sinkImpl) RecordMessageSize(size float64)                          {}
+func (s *sinkImpl) IncreaseWrittenMessages()                                {}
+func (s *sinkImpl) IncreaseTotalMessages()                                  {}
+func (s *sinkImpl) RecordBatchSize(size float64)                            {}
+func (s *sinkImpl) IncreaseTotalBatches()                                   {}
+func (s *sinkImpl) RecordWrittenBytes(size float64, fileType string)        {}
+func (s *sinkImpl) IncreaseTotalWrittenBytes(size float64, fileType string) {}
+func (s *sinkImpl) IncreaseCachedAssets()                                   {}
+func (s *sinkImpl) DecreaseCachedAssets()                                   {}
+func (s *sinkImpl) IncreaseSkippedAssets()                                  {}
+func (s *sinkImpl) IncreaseTotalAssets()                                    {}
+func (s *sinkImpl) RecordAssetSize(size float64)                            {}
+func (s *sinkImpl) RecordProcessAssetDuration(durMillis float64)            {}

backend/pkg/metrics/spot/spot.go

@@ -2,148 +2,34 @@ package spot
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
-
-	"openreplay/backend/pkg/metrics/common"
 )
 
-var spotOriginalVideoSize = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "spot",
-		Name:      "original_video_size_bytes",
-		Help:      "A histogram displaying the size of each original video in bytes.",
-		Buckets:   common.VideoSizeBuckets,
-	},
-)
-
-func RecordOriginalVideoSize(size float64) {
-	spotOriginalVideoSize.Observe(size)
+type Spot interface {
+	RecordOriginalVideoSize(size float64)
+	RecordCroppedVideoSize(size float64)
+	IncreaseVideosTotal()
+	IncreaseVideosCropped()
+	IncreaseVideosTranscoded()
+	RecordOriginalVideoDownloadDuration(durMillis float64)
+	RecordCroppingDuration(durMillis float64)
+	RecordCroppedVideoUploadDuration(durMillis float64)
+	RecordTranscodingDuration(durMillis float64)
+	RecordTranscodedVideoUploadDuration(durMillis float64)
+	List() []prometheus.Collector
 }
 
-var spotCroppedVideoSize = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "spot",
-		Name:      "cropped_video_size_bytes",
-		Help:      "A histogram displaying the size of each cropped video in bytes.",
-		Buckets:   common.VideoSizeBuckets,
-	},
-)
+type spotImpl struct{}
 
-func RecordCroppedVideoSize(size float64) {
-	spotCroppedVideoSize.Observe(size)
-}
+func New(serviceName string) Spot { return &spotImpl{} }
 
-var spotVideosTotal = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "spot",
-		Name:      "videos_total",
-		Help:      "A counter displaying the total number of all processed videos.",
-	},
-)
-
-func IncreaseVideosTotal() {
-	spotVideosTotal.Inc()
-}
-
-var spotVideosCropped = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "spot",
-		Name:      "videos_cropped_total",
-		Help:      "A counter displaying the total number of all cropped videos.",
-	},
-)
-
-func IncreaseVideosCropped() {
-	spotVideosCropped.Inc()
-}
-
-var spotVideosTranscoded = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "spot",
-		Name:      "videos_transcoded_total",
-		Help:      "A counter displaying the total number of all transcoded videos.",
-	},
-)
-
-func IncreaseVideosTranscoded() {
-	spotVideosTranscoded.Inc()
-}
-
-var spotOriginalVideoDownloadDuration = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "spot",
-		Name:      "original_video_download_duration_seconds",
-		Help:      "A histogram displaying the duration of downloading each original video in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-)
-
-func RecordOriginalVideoDownloadDuration(durMillis float64) {
-	spotOriginalVideoDownloadDuration.Observe(durMillis / 1000.0)
-}
-
-var spotCroppingDuration = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "spot",
-		Name:      "cropping_duration_seconds",
-		Help:      "A histogram displaying the duration of cropping each video in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-)
-
-func RecordCroppingDuration(durMillis float64) {
-	spotCroppingDuration.Observe(durMillis / 1000.0)
-}
-
-var spotCroppedVideoUploadDuration = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "spot",
-		Name:      "cropped_video_upload_duration_seconds",
-		Help:      "A histogram displaying the duration of uploading each cropped video in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-)
-
-func RecordCroppedVideoUploadDuration(durMillis float64) {
-	spotCroppedVideoUploadDuration.Observe(durMillis / 1000.0)
-}
-
-var spotTranscodingDuration = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "spot",
-		Name:      "transcoding_duration_seconds",
-		Help:      "A histogram displaying the duration of transcoding each video in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-)
-
-func RecordTranscodingDuration(durMillis float64) {
-	spotTranscodingDuration.Observe(durMillis / 1000.0)
-}
-
-var spotTranscodedVideoUploadDuration = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: "spot",
-		Name:      "transcoded_video_upload_duration_seconds",
-		Help:      "A histogram displaying the duration of uploading each transcoded video in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-)
-
-func RecordTranscodedVideoUploadDuration(durMillis float64) {
-	spotTranscodedVideoUploadDuration.Observe(durMillis / 1000.0)
-}
-
-func List() []prometheus.Collector {
-	return []prometheus.Collector{
-		spotOriginalVideoSize,
-		spotCroppedVideoSize,
-		spotVideosTotal,
-		spotVideosCropped,
-		spotVideosTranscoded,
-		spotOriginalVideoDownloadDuration,
-		spotCroppingDuration,
-		spotCroppedVideoUploadDuration,
-		spotTranscodingDuration,
-		spotTranscodedVideoUploadDuration,
-	}
-}
+func (s *spotImpl) List() []prometheus.Collector                          { return []prometheus.Collector{} }
+func (s *spotImpl) RecordOriginalVideoSize(size float64)                  {}
+func (s *spotImpl) RecordCroppedVideoSize(size float64)                   {}
+func (s *spotImpl) IncreaseVideosTotal()                                  {}
+func (s *spotImpl) IncreaseVideosCropped()                                {}
+func (s *spotImpl) IncreaseVideosTranscoded()                             {}
+func (s *spotImpl) RecordOriginalVideoDownloadDuration(durMillis float64) {}
+func (s *spotImpl) RecordCroppingDuration(durMillis float64)              {}
+func (s *spotImpl) RecordCroppedVideoUploadDuration(durMillis float64)    {}
+func (s *spotImpl) RecordTranscodingDuration(durMillis float64)           {}
+func (s *spotImpl) RecordTranscodedVideoUploadDuration(durMillis float64) {}

backend/pkg/metrics/storage/metrics.go

@@ -2,154 +2,34 @@ package storage
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
-	"openreplay/backend/pkg/metrics/common"
 )
 
-var storageSessionSize = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "session_size_bytes",
-		Help:      "A histogram displaying the size of each session file in bytes prior to any manipulation.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionSize(fileSize float64, fileType string) {
-	storageSessionSize.WithLabelValues(fileType).Observe(fileSize)
+type Storage interface {
+	RecordSessionSize(fileSize float64, fileType string)
+	IncreaseStorageTotalSessions()
+	RecordSkippedSessionSize(fileSize float64, fileType string)
+	IncreaseStorageTotalSkippedSessions()
+	RecordSessionReadDuration(durMillis float64, fileType string)
+	RecordSessionSortDuration(durMillis float64, fileType string)
+	RecordSessionEncryptionDuration(durMillis float64, fileType string)
+	RecordSessionCompressDuration(durMillis float64, fileType string)
+	RecordSessionUploadDuration(durMillis float64, fileType string)
+	RecordSessionCompressionRatio(ratio float64, fileType string)
+	List() []prometheus.Collector
 }
 
-var storageTotalSessions = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "storage",
-		Name:      "sessions_total",
-		Help:      "A counter displaying the total number of all processed sessions.",
-	},
-)
+type storageImpl struct{}
 
-func IncreaseStorageTotalSessions() {
-	storageTotalSessions.Inc()
-}
+func New(serviceName string) Storage { return &storageImpl{} }
 
-var storageSkippedSessionSize = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "session_size_bytes",
-		Help:      "A histogram displaying the size of each skipped session file in bytes.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSkippedSessionSize(fileSize float64, fileType string) {
-	storageSkippedSessionSize.WithLabelValues(fileType).Observe(fileSize)
-}
-
-var storageTotalSkippedSessions = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "storage",
-		Name:      "sessions_skipped_total",
-		Help:      "A counter displaying the total number of all skipped sessions because of the size limits.",
-	},
-)
-
-func IncreaseStorageTotalSkippedSessions() {
-	storageTotalSkippedSessions.Inc()
-}
-
-var storageSessionReadDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "read_duration_seconds",
-		Help:      "A histogram displaying the duration of reading for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionReadDuration(durMillis float64, fileType string) {
-	storageSessionReadDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionSortDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "sort_duration_seconds",
-		Help:      "A histogram displaying the duration of sorting for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionSortDuration(durMillis float64, fileType string) {
-	storageSessionSortDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionEncryptionDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "encryption_duration_seconds",
-		Help:      "A histogram displaying the duration of encoding for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionEncryptionDuration(durMillis float64, fileType string) {
-	storageSessionEncryptionDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionCompressDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "compress_duration_seconds",
-		Help:      "A histogram displaying the duration of compressing for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionCompressDuration(durMillis float64, fileType string) {
-	storageSessionCompressDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionUploadDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "upload_duration_seconds",
-		Help:      "A histogram displaying the duration of uploading to s3 for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionUploadDuration(durMillis float64, fileType string) {
-	storageSessionUploadDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionCompressionRatio = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "compression_ratio",
-		Help:      "A histogram displaying the compression ratio of mob files for each session.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionCompressionRatio(ratio float64, fileType string) {
-	storageSessionCompressionRatio.WithLabelValues(fileType).Observe(ratio)
-}
-
-func List() []prometheus.Collector {
-	return []prometheus.Collector{
-		storageSessionSize,
-		storageTotalSessions,
-		storageSessionReadDuration,
-		storageSessionSortDuration,
-		storageSessionEncryptionDuration,
-		storageSessionCompressDuration,
-		storageSessionUploadDuration,
-		storageSessionCompressionRatio,
-	}
-}
+func (s *storageImpl) List() []prometheus.Collector                                       { return []prometheus.Collector{} }
+func (s *storageImpl) RecordSessionSize(fileSize float64, fileType string)                {}
+func (s *storageImpl) IncreaseStorageTotalSessions()                                      {}
+func (s *storageImpl) RecordSkippedSessionSize(fileSize float64, fileType string)         {}
+func (s *storageImpl) IncreaseStorageTotalSkippedSessions()                               {}
+func (s *storageImpl) RecordSessionReadDuration(durMillis float64, fileType string)       {}
+func (s *storageImpl) RecordSessionSortDuration(durMillis float64, fileType string)       {}
+func (s *storageImpl) RecordSessionEncryptionDuration(durMillis float64, fileType string) {}
+func (s *storageImpl) RecordSessionCompressDuration(durMillis float64, fileType string)   {}
+func (s *storageImpl) RecordSessionUploadDuration(durMillis float64, fileType string)     {}
+func (s *storageImpl) RecordSessionCompressionRatio(ratio float64, fileType string)       {}

backend/pkg/metrics/videostorage/metrics.go

@@ -1,155 +0,0 @@
-package videostorage
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"openreplay/backend/pkg/metrics/common"
-)
-
-var storageSessionSize = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "session_size_bytes",
-		Help:      "A histogram displaying the size of each session file in bytes prior to any manipulation.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionSize(fileSize float64, fileType string) {
-	storageSessionSize.WithLabelValues(fileType).Observe(fileSize)
-}
-
-var storageTotalSessions = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "storage",
-		Name:      "sessions_total",
-		Help:      "A counter displaying the total number of all processed sessions.",
-	},
-)
-
-func IncreaseStorageTotalSessions() {
-	storageTotalSessions.Inc()
-}
-
-var storageSkippedSessionSize = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "session_size_bytes",
-		Help:      "A histogram displaying the size of each skipped session file in bytes.",
-		Buckets:   common.DefaultSizeBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSkippedSessionSize(fileSize float64, fileType string) {
-	storageSkippedSessionSize.WithLabelValues(fileType).Observe(fileSize)
-}
-
-var storageTotalSkippedSessions = prometheus.NewCounter(
-	prometheus.CounterOpts{
-		Namespace: "storage",
-		Name:      "sessions_skipped_total",
-		Help:      "A counter displaying the total number of all skipped sessions because of the size limits.",
-	},
-)
-
-func IncreaseStorageTotalSkippedSessions() {
-	storageTotalSkippedSessions.Inc()
-}
-
-var storageSessionReadDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "read_duration_seconds",
-		Help:      "A histogram displaying the duration of reading for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionReadDuration(durMillis float64, fileType string) {
-	storageSessionReadDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionSortDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "sort_duration_seconds",
-		Help:      "A histogram displaying the duration of sorting for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionSortDuration(durMillis float64, fileType string) {
-	storageSessionSortDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionEncryptionDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "encryption_duration_seconds",
-		Help:      "A histogram displaying the duration of encoding for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionEncryptionDuration(durMillis float64, fileType string) {
-	storageSessionEncryptionDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionCompressDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "compress_duration_seconds",
-		Help:      "A histogram displaying the duration of compressing for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionCompressDuration(durMillis float64, fileType string) {
-	storageSessionCompressDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionUploadDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "upload_duration_seconds",
-		Help:      "A histogram displaying the duration of uploading to s3 for each session in seconds.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionUploadDuration(durMillis float64, fileType string) {
-	storageSessionUploadDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
-}
-
-var storageSessionCompressionRatio = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "storage",
-		Name:      "compression_ratio",
-		Help:      "A histogram displaying the compression ratio of mob files for each session.",
-		Buckets:   common.DefaultDurationBuckets,
-	},
-	[]string{"file_type"},
-)
-
-func RecordSessionCompressionRatio(ratio float64, fileType string) {
-	storageSessionCompressionRatio.WithLabelValues(fileType).Observe(ratio)
-}
-
-func List() []prometheus.Collector {
-	return []prometheus.Collector{
-		storageSessionSize,
-		storageTotalSessions,
-		storageSessionReadDuration,
-		storageSessionSortDuration,
-		storageSessionEncryptionDuration,
-		storageSessionCompressDuration,
-		storageSessionUploadDuration,
-		storageSessionCompressionRatio,
-	}
-}

backend/pkg/metrics/web/metrics.go

@@ -1,11 +1,7 @@
 package web
 
 import (
-	"strconv"
-
 	"github.com/prometheus/client_golang/prometheus"
-
-	"openreplay/backend/pkg/metrics/common"
 )
 
 type Web interface {
@@ -15,70 +11,11 @@ type Web interface {
 	List() []prometheus.Collector
 }
 
-type webImpl struct {
-	httpRequestSize     *prometheus.HistogramVec
-	httpRequestDuration *prometheus.HistogramVec
-	httpTotalRequests   prometheus.Counter
-}
+type webImpl struct{}
 
-func New(serviceName string) Web {
-	return &webImpl{
-		httpRequestSize:     newRequestSizeMetric(serviceName),
-		httpRequestDuration: newRequestDurationMetric(serviceName),
-		httpTotalRequests:   newTotalRequestsMetric(serviceName),
-	}
-}
+func New(serviceName string) Web { return &webImpl{} }
 
-func (w *webImpl) List() []prometheus.Collector {
-	return []prometheus.Collector{
-		w.httpRequestSize,
-		w.httpRequestDuration,
-		w.httpTotalRequests,
-	}
-}
-
-func newRequestSizeMetric(serviceName string) *prometheus.HistogramVec {
-	return prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Namespace: serviceName,
-			Name:      "request_size_bytes",
-			Help:      "A histogram displaying the size of each HTTP request in bytes.",
-			Buckets:   common.DefaultSizeBuckets,
-		},
-		[]string{"url", "response_code"},
-	)
-}
-
-func (w *webImpl) RecordRequestSize(size float64, url string, code int) {
-	w.httpRequestSize.WithLabelValues(url, strconv.Itoa(code)).Observe(size)
-}
-
-func newRequestDurationMetric(serviceName string) *prometheus.HistogramVec {
-	return prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Namespace: serviceName,
-			Name:      "request_duration_seconds",
-			Help:      "A histogram displaying the duration of each HTTP request in seconds.",
-			Buckets:   common.DefaultDurationBuckets,
-		},
-		[]string{"url", "response_code"},
-	)
-}
-
-func (w *webImpl) RecordRequestDuration(durMillis float64, url string, code int) {
-	w.httpRequestDuration.WithLabelValues(url, strconv.Itoa(code)).Observe(durMillis / 1000.0)
-}
-
-func newTotalRequestsMetric(serviceName string) prometheus.Counter {
-	return prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: serviceName,
-			Name:      "requests_total",
-			Help:      "A counter displaying the number all HTTP requests.",
-		},
-	)
-}
-
-func (w *webImpl) IncreaseTotalRequests() {
-	w.httpTotalRequests.Inc()
-}
+func (w *webImpl) List() []prometheus.Collector                                  { return []prometheus.Collector{} }
+func (w *webImpl) RecordRequestSize(size float64, url string, code int)          {}
+func (w *webImpl) RecordRequestDuration(durMillis float64, url string, code int) {}
+func (w *webImpl) IncreaseTotalRequests()                                        {}

backend/pkg/projects/cache.go

@@ -11,6 +11,8 @@ import (
 	"openreplay/backend/pkg/metrics/database"
 )
 
+var ErrDisabledCache = errors.New("cache is disabled")
+
 type Cache interface {
 	Set(project *Project) error
 	GetByID(projectID uint32) (*Project, error)
@@ -18,10 +20,16 @@ type Cache interface {
 }
 
 type cacheImpl struct {
-	db *redis.Client
+	db      *redis.Client
+	metrics database.Database
 }
 
-var ErrDisabledCache = errors.New("cache is disabled")
+func NewCache(db *redis.Client, metrics database.Database) Cache {
+	return &cacheImpl{
+		db:      db,
+		metrics: metrics,
+	}
+}
 
 func (c *cacheImpl) Set(project *Project) error {
 	if c.db == nil {
@@ -38,8 +46,8 @@ func (c *cacheImpl) Set(project *Project) error {
 	if _, err = c.db.Redis.Set(fmt.Sprintf("project:key:%s", project.ProjectKey), projectBytes, time.Minute*10).Result(); err != nil {
 		return err
 	}
-	database.RecordRedisRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "set", "project")
-	database.IncreaseRedisRequests("set", "project")
+	c.metrics.RecordRedisRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "set", "project")
+	c.metrics.IncreaseRedisRequests("set", "project")
 	return nil
 }
 
@@ -52,8 +60,8 @@ func (c *cacheImpl) GetByID(projectID uint32) (*Project, error) {
 	if err != nil {
 		return nil, err
 	}
-	database.RecordRedisRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "get", "project")
-	database.IncreaseRedisRequests("get", "project")
+	c.metrics.RecordRedisRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "get", "project")
+	c.metrics.IncreaseRedisRequests("get", "project")
 	project := &Project{}
 	if err = json.Unmarshal([]byte(result), project); err != nil {
 		return nil, err
@@ -70,15 +78,11 @@ func (c *cacheImpl) GetByKey(projectKey string) (*Project, error) {
 	if err != nil {
 		return nil, err
 	}
-	database.RecordRedisRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "get", "project")
-	database.IncreaseRedisRequests("get", "project")
+	c.metrics.RecordRedisRequestDuration(float64(time.Now().Sub(start).Milliseconds()), "get", "project")
+	c.metrics.IncreaseRedisRequests("get", "project")
 	project := &Project{}
 	if err = json.Unmarshal([]byte(result), project); err != nil {
 		return nil, err
 	}
 	return project, nil
 }
-
-func NewCache(db *redis.Client) Cache {
-	return &cacheImpl{db: db}
-}

backend/pkg/projects/projects.go

@@ -9,6 +9,7 @@ import (
 	"openreplay/backend/pkg/db/postgres/pool"
 	"openreplay/backend/pkg/db/redis"
 	"openreplay/backend/pkg/logger"
+	"openreplay/backend/pkg/metrics/database"
 )
 
 type Projects interface {
@@ -24,8 +25,8 @@ type projectsImpl struct {
 	projectsByKeys cache.Cache
 }
 
-func New(log logger.Logger, db pool.Pool, redis *redis.Client) Projects {
-	cl := NewCache(redis)
+func New(log logger.Logger, db pool.Pool, redis *redis.Client, metrics database.Database) Projects {
+	cl := NewCache(redis, metrics)
 	return &projectsImpl{
 		log:            log,
 		db:             db,

backend/pkg/server/auth/auth.go

@@ -24,15 +24,17 @@ type authImpl struct {
 	spotSecret string
 	pgconn     pool.Pool
 	keys       keys.Keys
+	prefix     string
 }
 
-func NewAuth(log logger.Logger, jwtSecret, jwtSpotSecret string, conn pool.Pool, keys keys.Keys) Auth {
+func NewAuth(log logger.Logger, jwtSecret, jwtSpotSecret string, conn pool.Pool, keys keys.Keys, prefix string) Auth {
 	return &authImpl{
 		log:        log,
 		secret:     jwtSecret,
 		spotSecret: jwtSpotSecret,
 		pgconn:     conn,
 		keys:       keys,
+		prefix:     prefix,
 	}
 }
 

backend/pkg/server/auth/middleware.go

@@ -36,9 +36,9 @@ func (e *authImpl) isExtensionRequest(r *http.Request) bool {
 	if err != nil {
 		e.log.Error(r.Context(), "failed to get path template: %s", err)
 	} else {
-		if pathTemplate == "/v1/ping" ||
-			(pathTemplate == "/v1/spots" && r.Method == "POST") ||
-			(pathTemplate == "/v1/spots/{id}/uploaded" && r.Method == "POST") {
+		if pathTemplate == e.prefix+"/v1/ping" ||
+			(pathTemplate == e.prefix+"/v1/spots" && r.Method == "POST") ||
+			(pathTemplate == e.prefix+"/v1/spots/{id}/uploaded" && r.Method == "POST") {
 			return true
 		}
 	}
@@ -53,9 +53,9 @@ func (e *authImpl) isSpotWithKeyRequest(r *http.Request) bool {
 	if err != nil {
 		return false
 	}
-	getSpotPrefix := "/v1/spots/{id}"            // GET
-	addCommentPrefix := "/v1/spots/{id}/comment" // POST
-	getStatusPrefix := "/v1/spots/{id}/status"   // GET
+	getSpotPrefix := e.prefix + "/v1/spots/{id}"            // GET
+	addCommentPrefix := e.prefix + "/v1/spots/{id}/comment" // POST
+	getStatusPrefix := e.prefix + "/v1/spots/{id}/status"   // GET
 	if (pathTemplate == getSpotPrefix && r.Method == "GET") ||
 		(pathTemplate == addCommentPrefix && r.Method == "POST") ||
 		(pathTemplate == getStatusPrefix && r.Method == "GET") {