feat(images): renamed + small improvements

backend/cmd/canvases/main.go

@@ -13,7 +13,7 @@ import (
 	"openreplay/backend/pkg/logger"
 	"openreplay/backend/pkg/messages"
 	"openreplay/backend/pkg/metrics"
-	storageMetrics "openreplay/backend/pkg/metrics/imagestorage"
+	storageMetrics "openreplay/backend/pkg/metrics/images"
 	"openreplay/backend/pkg/objectstorage/store"
 	"openreplay/backend/pkg/queue"
 )
@@ -34,7 +34,7 @@ func main() {
 
 	srv, err := canvases.New(cfg, log, objStore, producer)
 	if err != nil {
-		log.Fatal(ctx, "can't init canvas service: %s", err)
+		log.Fatal(ctx, "can't init canvases service: %s", err)
 	}
 
 	canvasConsumer := queue.NewConsumer(
@@ -97,7 +97,7 @@ func main() {
 		cfg.MessageSizeLimit,
 	)
 
-	log.Info(ctx, "canvas handler service started")
+	log.Info(ctx, "canvases service started")
 
 	sigchan := make(chan os.Signal, 1)
 	signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)

backend/cmd/images/main.go

@@ -9,12 +9,12 @@ import (
 	"syscall"
 	"time"
 
-	config "openreplay/backend/internal/config/imagestorage"
-	"openreplay/backend/internal/screenshot-handler"
+	config "openreplay/backend/internal/config/images"
+	"openreplay/backend/internal/images"
 	"openreplay/backend/pkg/logger"
 	"openreplay/backend/pkg/messages"
 	"openreplay/backend/pkg/metrics"
-	storageMetrics "openreplay/backend/pkg/metrics/imagestorage"
+	storageMetrics "openreplay/backend/pkg/metrics/images"
 	"openreplay/backend/pkg/objectstorage/store"
 	"openreplay/backend/pkg/queue"
 )
@@ -30,9 +30,9 @@ func main() {
 		log.Fatal(ctx, "can't init object storage: %s", err)
 	}
 
-	srv, err := screenshot_handler.New(cfg, log, objStore)
+	srv, err := images.New(cfg, log, objStore)
 	if err != nil {
-		log.Fatal(ctx, "can't init storage service: %s", err)
+		log.Fatal(ctx, "can't init images service: %s", err)
 	}
 
 	workDir := cfg.FSDir
@@ -74,7 +74,7 @@ func main() {
 		cfg.MessageSizeLimit,
 	)
 
-	log.Info(ctx, "Image storage service started")
+	log.Info(ctx, "Images service started")
 
 	sigchan := make(chan os.Signal, 1)
 	signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)

backend/internal/config/images/config.go

@@ -1,4 +1,4 @@
-package imagestorage
+package images
 
 import (
 	"openreplay/backend/internal/config/common"

backend/internal/images/service.go

@@ -1,4 +1,4 @@
-package screenshot_handler
+package images
 
 import (
 	"archive/tar"
@@ -6,16 +6,17 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"openreplay/backend/pkg/logger"
-	"openreplay/backend/pkg/objectstorage"
-	"openreplay/backend/pkg/pool"
 	"os"
 	"os/exec"
 	"strconv"
 	"time"
 
 	gzip "github.com/klauspost/pgzip"
-	config "openreplay/backend/internal/config/imagestorage"
+
+	config "openreplay/backend/internal/config/images"
+	"openreplay/backend/pkg/logger"
+	"openreplay/backend/pkg/objectstorage"
+	"openreplay/backend/pkg/pool"
 )
 
 type saveTask struct {
@@ -43,6 +44,10 @@ func New(cfg *config.Config, log logger.Logger, objStorage objectstorage.ObjectS
 	switch {
 	case cfg == nil:
 		return nil, fmt.Errorf("config is empty")
+	case log == nil:
+		return nil, fmt.Errorf("logger is empty")
+	case objStorage == nil:
+		return nil, fmt.Errorf("objStorage is empty")
 	}
 	s := &ImageStorage{
 		cfg:        cfg,
@@ -50,7 +55,7 @@ func New(cfg *config.Config, log logger.Logger, objStorage objectstorage.ObjectS
 		objStorage: objStorage,
 	}
 	s.saverPool = pool.NewPool(4, 8, s.writeToDisk)
-	s.uploaderPool = pool.NewPool(4, 4, s.sendToS3)
+	s.uploaderPool = pool.NewPool(8, 8, s.sendToS3)
 	return s, nil
 }
 
@@ -100,12 +105,11 @@ func (v *ImageStorage) writeToDisk(payload interface{}) {
 	if v.cfg.ScreenshotsDir != "" {
 		path += v.cfg.ScreenshotsDir + "/"
 	}
-
 	path += strconv.FormatUint(task.sessionID, 10) + "/"
 
 	// Ensure the directory exists
 	if err := os.MkdirAll(path, 0755); err != nil {
-		v.log.Fatal(task.ctx, "Error creating directories: %v", err)
+		v.log.Fatal(task.ctx, "error creating directories: %v", err)
 	}
 
 	// Write images to disk
@@ -118,7 +122,12 @@ func (v *ImageStorage) writeToDisk(payload interface{}) {
 		if _, err := io.Copy(outFile, img); err != nil {
 			v.log.Error(task.ctx, "can't copy file: %s", err.Error())
 		}
-		outFile.Close()
+		if outFile == nil {
+			continue
+		}
+		if err := outFile.Close(); err != nil {
+			v.log.Warn(task.ctx, "can't close file: %s", err.Error())
+		}
 		saved++
 	}
 	v.log.Info(task.ctx, "saved %d images to disk", saved)
@@ -126,10 +135,6 @@ func (v *ImageStorage) writeToDisk(payload interface{}) {
 }
 
 func (v *ImageStorage) PackScreenshots(ctx context.Context, sessID uint64, filesPath string) error {
-	// Temporarily disabled for tests
-	if v.objStorage == nil {
-		return fmt.Errorf("object storage is empty")
-	}
 	start := time.Now()
 	sessionID := strconv.FormatUint(sessID, 10)
 	selector := fmt.Sprintf("%s*.jpeg", filesPath)

backend/pkg/metrics/images/metrics.go

@@ -1,4 +1,4 @@
-package imagestorage
+package images
 
 import (
 	"github.com/prometheus/client_golang/prometheus"

scripts/helmcharts/openreplay/charts/images/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-name: imagestorage
+name: images
 description: A Helm chart for Kubernetes
 
 # A chart can be either an 'application' or a 'library' chart.

scripts/helmcharts/openreplay/charts/images/templates/NOTES.txt

@@ -6,16 +6,16 @@
   {{- end }}
 {{- end }}
 {{- else if contains "NodePort" .Values.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "imagestorage.fullname" . }})
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "images.fullname" . }})
   export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
   echo http://$NODE_IP:$NODE_PORT
 {{- else if contains "LoadBalancer" .Values.service.type }}
      NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "imagestorage.fullname" . }}'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "imagestorage.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "images.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "images.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
   echo http://$SERVICE_IP:{{ .Values.service.port }}
 {{- else if contains "ClusterIP" .Values.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "imagestorage.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "images.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
   export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
   echo "Visit http://127.0.0.1:8080 to use your application"
   kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT

scripts/helmcharts/openreplay/charts/images/templates/_helpers.tpl

@@ -1,7 +1,7 @@
 {{/*
 Expand the name of the chart.
 */}}
-{{- define "imagestorage.name" -}}
+{{- define "images.name" -}}
 {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
@@ -10,7 +10,7 @@ Create a default fully qualified app name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
 If release name contains chart name it will be used as a full name.
 */}}
-{{- define "imagestorage.fullname" -}}
+{{- define "images.fullname" -}}
 {{- if .Values.fullnameOverride }}
 {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
 {{- else }}
@@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name.
 {{/*
 Create chart name and version as used by the chart label.
 */}}
-{{- define "imagestorage.chart" -}}
+{{- define "images.chart" -}}
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
 {{/*
 Common labels
 */}}
-{{- define "imagestorage.labels" -}}
-helm.sh/chart: {{ include "imagestorage.chart" . }}
-{{ include "imagestorage.selectorLabels" . }}
+{{- define "images.labels" -}}
+helm.sh/chart: {{ include "images.chart" . }}
+{{ include "images.selectorLabels" . }}
 {{- if .Chart.AppVersion }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
@@ -48,17 +48,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{/*
 Selector labels
 */}}
-{{- define "imagestorage.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "imagestorage.name" . }}
+{{- define "images.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "images.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
 {{/*
 Create the name of the service account to use
 */}}
-{{- define "imagestorage.serviceAccountName" -}}
+{{- define "images.serviceAccountName" -}}
 {{- if .Values.serviceAccount.create }}
-{{- default (include "imagestorage.fullname" .) .Values.serviceAccount.name }}
+{{- default (include "images.fullname" .) .Values.serviceAccount.name }}
 {{- else }}
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}

scripts/helmcharts/openreplay/charts/images/templates/deployment.yaml

@@ -1,17 +1,17 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ include "imagestorage.fullname" . }}
+  name: {{ include "images.fullname" . }}
   namespace: {{ .Release.Namespace }}
   labels:
-    {{- include "imagestorage.labels" . | nindent 4 }}
+    {{- include "images.labels" . | nindent 4 }}
 spec:
   {{- if not .Values.autoscaling.enabled }}
   replicas: {{ .Values.replicaCount }}
   {{- end }}
   selector:
     matchLabels:
-      {{- include "imagestorage.selectorLabels" . | nindent 6 }}
+      {{- include "images.selectorLabels" . | nindent 6 }}
   template:
     metadata:
       {{- with .Values.podAnnotations }}
@@ -19,13 +19,13 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
-        {{- include "imagestorage.selectorLabels" . | nindent 8 }}
+        {{- include "images.selectorLabels" . | nindent 8 }}
     spec:
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      serviceAccountName: {{ include "imagestorage.serviceAccountName" . }}
+      serviceAccountName: {{ include "images.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       shareProcessNamespace: true

scripts/helmcharts/openreplay/charts/images/templates/hpa.yaml

@@ -2,15 +2,15 @@
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
-  name: {{ include "imagestorage.fullname" . }}
+  name: {{ include "images.fullname" . }}
   namespace: {{ .Release.Namespace }}
   labels:
-    {{- include "imagestorage.labels" . | nindent 4 }}
+    {{- include "images.labels" . | nindent 4 }}
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
-    name: {{ include "imagestorage.fullname" . }}
+    name: {{ include "images.fullname" . }}
   minReplicas: {{ .Values.autoscaling.minReplicas }}
   maxReplicas: {{ .Values.autoscaling.maxReplicas }}
   metrics:

scripts/helmcharts/openreplay/charts/images/templates/ingress.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.ingress.enabled -}}
-{{- $fullName := include "imagestorage.fullname" . -}}
+{{- $fullName := include "images.fullname" . -}}
 {{- $svcPort := .Values.service.ports.http -}}
 {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
   {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
@@ -18,7 +18,7 @@ metadata:
   name: {{ $fullName }}
   namespace: {{ .Release.Namespace }}
   labels:
-    {{- include "imagestorage.labels" . | nindent 4 }}
+    {{- include "images.labels" . | nindent 4 }}
   {{- with .Values.ingress.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}

scripts/helmcharts/openreplay/charts/images/templates/service.yaml

@@ -1,10 +1,10 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "imagestorage.fullname" . }}
+  name: {{ include "images.fullname" . }}
   namespace: {{ .Release.Namespace }}
   labels:
-    {{- include "imagestorage.labels" . | nindent 4 }}
+    {{- include "images.labels" . | nindent 4 }}
 spec:
   type: {{ .Values.service.type }}
   ports:
@@ -15,4 +15,4 @@ spec:
       name: {{ $key }}
     {{- end}}
   selector:
-    {{- include "imagestorage.selectorLabels" . | nindent 4 }}
+    {{- include "images.selectorLabels" . | nindent 4 }}

scripts/helmcharts/openreplay/charts/images/templates/serviceMonitor.yaml

@@ -2,10 +2,10 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
-  name: {{ include "imagestorage.fullname" . }}
+  name: {{ include "images.fullname" . }}
   namespace: {{ .Release.Namespace }}
   labels:
-    {{- include "imagestorage.labels" . | nindent 4 }}
+    {{- include "images.labels" . | nindent 4 }}
   {{- if .Values.serviceMonitor.additionalLabels }}
     {{- toYaml .Values.serviceMonitor.additionalLabels | nindent 4 }}
   {{- end }}
@@ -14,5 +14,5 @@ spec:
     {{- .Values.serviceMonitor.scrapeConfigs | toYaml | nindent 4 }}
   selector:
     matchLabels:
-      {{- include "imagestorage.selectorLabels" . | nindent 6 }}
+      {{- include "images.selectorLabels" . | nindent 6 }}
 {{- end }}

scripts/helmcharts/openreplay/charts/images/templates/serviceaccount.yaml

@@ -2,10 +2,10 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: {{ include "imagestorage.serviceAccountName" . }}
+  name: {{ include "images.serviceAccountName" . }}
   namespace: {{ .Release.Namespace }}
   labels:
-    {{- include "imagestorage.labels" . | nindent 4 }}
+    {{- include "images.labels" . | nindent 4 }}
   {{- with .Values.serviceAccount.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}

scripts/helmcharts/openreplay/charts/images/templates/tests/test-connection.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "images.fullname" . }}-test-connection"
+  labels:
+    {{- include "images.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ include "images.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never

scripts/helmcharts/openreplay/charts/images/values.yaml

@@ -5,14 +5,14 @@
 replicaCount: 1
 
 image:
-  repository: "{{ .Values.global.openReplayContainerRegistry }}/imagestorage"
+  repository: "{{ .Values.global.openReplayContainerRegistry }}/images"
   pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
   tag: ""
 
 imagePullSecrets: []
-nameOverride: "imagestorage"
-fullnameOverride: "imagestorage-openreplay"
+nameOverride: "images"
+fullnameOverride: "images-openreplay"
 
 serviceAccount:
   # Specifies whether a service account should be created

scripts/helmcharts/openreplay/charts/imagestorage/templates/tests/test-connection.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: "{{ include "imagestorage.fullname" . }}-test-connection"
-  labels:
-    {{- include "imagestorage.labels" . | nindent 4 }}
-  annotations:
-    "helm.sh/hook": test
-spec:
-  containers:
-    - name: wget
-      image: busybox
-      command: ['wget']
-      args: ['{{ include "imagestorage.fullname" . }}:{{ .Values.service.port }}']
-  restartPolicy: Never

scripts/helmcharts/openreplay/file

@@ -6,7 +6,5 @@ charts/heuristics/values.yaml
 charts/integrations/values.yaml
 charts/sink/values.yaml
 charts/storage/values.yaml
-charts/imagestorage/values.yaml
-charts/canvas-handler/values.yaml
-charts/canvas-maker/values.yaml
-charts/videostorage/values.yaml
+charts/images/values.yaml
+charts/canvases/values.yaml