Ux testing backend (#1709)

* feat(backend): added ux-testing support

* feat(backend): added ux-testing module

* feat(http): added bucket name for http service

* feat(backend): fixed small typos in http router

---------

Co-authored-by: Alexander <zavorotynskiy@pm.me>

backend/internal/config/http/config.go

@@ -3,6 +3,7 @@ package http
 import (
 	"openreplay/backend/internal/config/common"
 	"openreplay/backend/internal/config/configurator"
+	"openreplay/backend/internal/config/objectstorage"
 	"openreplay/backend/internal/config/redis"
 	"openreplay/backend/pkg/env"
 	"time"
@@ -12,6 +13,7 @@ type Config struct {
 	common.Config
 	common.Postgres
 	redis.Redis
+	objectstorage.ObjectsConfig
 	HTTPHost                string        `env:"HTTP_HOST,default="`
 	HTTPPort                string        `env:"HTTP_PORT,required"`
 	HTTPTimeout             time.Duration `env:"HTTP_TIMEOUT,default=60s"`

backend/internal/http/router/handlers-web.go

@@ -4,12 +4,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/gorilla/mux"
 	"io"
 	"log"
 	"math/rand"
 	"net/http"
 	"openreplay/backend/pkg/featureflags"
 	"openreplay/backend/pkg/sessions"
+	"openreplay/backend/pkg/uxtesting"
 	"strconv"
 	"time"
 
@@ -364,3 +366,125 @@ func (e *Router) featureFlagsHandlerWeb(w http.ResponseWriter, r *http.Request)
 	}
 	ResponseWithJSON(w, resp, startTime, r.URL.Path, bodySize)
 }
+
+func (e *Router) getUXTestInfo(w http.ResponseWriter, r *http.Request) {
+	startTime := time.Now()
+	bodySize := 0
+
+	// Check authorization
+	_, err := e.services.Tokenizer.ParseFromHTTPRequest(r)
+	if err != nil {
+		ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+
+	// Get taskID
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	// Get task info
+	info, err := e.services.UXTesting.GetInfo(id)
+	if err != nil {
+		ResponseWithError(w, http.StatusInternalServerError, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+	type TaskInfoResponse struct {
+		Task *uxtesting.UXTestInfo `json:"test"`
+	}
+	ResponseWithJSON(w, &TaskInfoResponse{Task: info}, startTime, r.URL.Path, bodySize)
+}
+
+func (e *Router) sendUXTestSignal(w http.ResponseWriter, r *http.Request) {
+	startTime := time.Now()
+	bodySize := 0
+
+	// Check authorization
+	sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r)
+	if err != nil {
+		ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+
+	bodyBytes, err := e.readBody(w, r, e.cfg.JsonSizeLimit)
+	if err != nil {
+		log.Printf("error while reading request body: %s", err)
+		ResponseWithError(w, http.StatusRequestEntityTooLarge, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+	bodySize = len(bodyBytes)
+
+	// Parse request body
+	req := &uxtesting.TestSignal{}
+
+	if err := json.Unmarshal(bodyBytes, req); err != nil {
+		ResponseWithError(w, http.StatusBadRequest, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+	req.SessionID = sessionData.ID
+
+	// Save test signal
+	if err := e.services.UXTesting.SetTestSignal(req); err != nil {
+		ResponseWithError(w, http.StatusBadRequest, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+	ResponseOK(w, startTime, r.URL.Path, bodySize)
+}
+
+func (e *Router) sendUXTaskSignal(w http.ResponseWriter, r *http.Request) {
+	startTime := time.Now()
+	bodySize := 0
+
+	// Check authorization
+	sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r)
+	if err != nil {
+		ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+
+	bodyBytes, err := e.readBody(w, r, e.cfg.JsonSizeLimit)
+	if err != nil {
+		log.Printf("error while reading request body: %s", err)
+		ResponseWithError(w, http.StatusRequestEntityTooLarge, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+	bodySize = len(bodyBytes)
+
+	// Parse request body
+	req := &uxtesting.TaskSignal{}
+
+	if err := json.Unmarshal(bodyBytes, req); err != nil {
+		ResponseWithError(w, http.StatusBadRequest, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+	req.SessionID = sessionData.ID
+
+	// Save test signal
+	if err := e.services.UXTesting.SetTaskSignal(req); err != nil {
+		ResponseWithError(w, http.StatusBadRequest, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+	ResponseOK(w, startTime, r.URL.Path, bodySize)
+}
+
+func (e *Router) getUXUploadUrl(w http.ResponseWriter, r *http.Request) {
+	startTime := time.Now()
+	bodySize := 0
+
+	// Check authorization
+	sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r)
+	if err != nil {
+		ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+
+	key := fmt.Sprintf("%d/ux_webcam_record.mp4", sessionData.ID)
+	url, err := e.services.ObjStorage.GetPreSignedUploadUrl(key)
+	if err != nil {
+		ResponseWithError(w, http.StatusInternalServerError, err, startTime, r.URL.Path, bodySize)
+		return
+	}
+	type UrlResponse struct {
+		URL string `json:"url"`
+	}
+	ResponseWithJSON(w, &UrlResponse{URL: url}, startTime, r.URL.Path, bodySize)
+}

backend/internal/http/router/router.go

@@ -101,14 +101,20 @@ func (e *Router) init() {
 	e.router.HandleFunc("/", e.root)
 
 	handlers := map[string]func(http.ResponseWriter, *http.Request){
-		"/v1/web/not-started":   e.notStartedHandlerWeb,
-		"/v1/web/start":         e.startSessionHandlerWeb,
-		"/v1/web/i":             e.pushMessagesHandlerWeb,
-		"/v1/web/feature-flags": e.featureFlagsHandlerWeb,
-		"/v1/mobile/start":      e.startSessionHandlerIOS,
-		"/v1/mobile/i":          e.pushMessagesHandlerIOS,
-		"/v1/mobile/late":       e.pushLateMessagesHandlerIOS,
-		"/v1/mobile/images":     e.imagesUploadHandlerIOS,
+		"/v1/web/not-started":      e.notStartedHandlerWeb,
+		"/v1/web/start":            e.startSessionHandlerWeb,
+		"/v1/web/i":                e.pushMessagesHandlerWeb,
+		"/v1/web/feature-flags":    e.featureFlagsHandlerWeb,
+		"/v1/mobile/start":         e.startSessionHandlerIOS,
+		"/v1/mobile/i":             e.pushMessagesHandlerIOS,
+		"/v1/mobile/late":          e.pushLateMessagesHandlerIOS,
+		"/v1/mobile/images":        e.imagesUploadHandlerIOS,
+		"/v1/web/uxt/signals/test": e.sendUXTestSignal,
+		"/v1/web/uxt/signals/task": e.sendUXTaskSignal,
+	}
+	getHandlers := map[string]func(http.ResponseWriter, *http.Request){
+		"/v1/web/uxt/test/{id}":  e.getUXTestInfo,
+		"/v1/web/uxt/upload-url": e.getUXUploadUrl,
 	}
 	prefix := "/ingest"
 
@@ -116,6 +122,10 @@ func (e *Router) init() {
 		e.router.HandleFunc(path, handler).Methods("POST", "OPTIONS")
 		e.router.HandleFunc(prefix+path, handler).Methods("POST", "OPTIONS")
 	}
+	for path, handler := range getHandlers {
+		e.router.HandleFunc(path, handler).Methods("GET", "OPTIONS")
+		e.router.HandleFunc(prefix+path, handler).Methods("GET", "OPTIONS")
+	}
 
 	// CORS middleware
 	e.router.Use(e.corsMiddleware)
@@ -130,7 +140,7 @@ func (e *Router) corsMiddleware(next http.Handler) http.Handler {
 		if e.cfg.UseAccessControlHeaders {
 			// Prepare headers for preflight requests
 			w.Header().Set("Access-Control-Allow-Origin", "*")
-			w.Header().Set("Access-Control-Allow-Methods", "POST")
+			w.Header().Set("Access-Control-Allow-Methods", "POST,GET")
 			w.Header().Set("Access-Control-Allow-Headers", "Content-Type,Authorization,Content-Encoding")
 		}
 		if r.Method == http.MethodOptions {

backend/internal/http/services/services.go

@@ -1,6 +1,7 @@
 package services
 
 import (
+	"log"
 	"openreplay/backend/internal/config/http"
 	"openreplay/backend/internal/http/geoip"
 	"openreplay/backend/internal/http/uaparser"
@@ -8,10 +9,13 @@ import (
 	"openreplay/backend/pkg/db/redis"
 	"openreplay/backend/pkg/featureflags"
 	"openreplay/backend/pkg/flakeid"
+	"openreplay/backend/pkg/objectstorage"
+	"openreplay/backend/pkg/objectstorage/store"
 	"openreplay/backend/pkg/projects"
 	"openreplay/backend/pkg/queue/types"
 	"openreplay/backend/pkg/sessions"
 	"openreplay/backend/pkg/token"
+	"openreplay/backend/pkg/uxtesting"
 )
 
 type ServicesBuilder struct {
@@ -23,10 +27,17 @@ type ServicesBuilder struct {
 	UaParser     *uaparser.UAParser
 	GeoIP        geoip.GeoParser
 	Tokenizer    *token.Tokenizer
+	ObjStorage   objectstorage.ObjectStorage
+	UXTesting    uxtesting.UXTesting
 }
 
 func New(cfg *http.Config, producer types.Producer, pgconn pool.Pool, redis *redis.Client) (*ServicesBuilder, error) {
 	projs := projects.New(pgconn, redis)
+	// ObjectStorage client to generate pre-signed upload urls
+	objStore, err := store.NewStore(&cfg.ObjectsConfig)
+	if err != nil {
+		log.Fatalf("can't init object storage: %s", err)
+	}
 	return &ServicesBuilder{
 		Projects:     projs,
 		Sessions:     sessions.New(pgconn, projs, redis),
@@ -36,5 +47,7 @@ func New(cfg *http.Config, producer types.Producer, pgconn pool.Pool, redis *red
 		UaParser:     uaparser.NewUAParser(cfg.UAParserFile),
 		GeoIP:        geoip.New(cfg.MaxMinDBFile),
 		Flaker:       flakeid.NewFlaker(cfg.WorkerID),
+		ObjStorage:   objStore,
+		UXTesting:    uxtesting.New(pgconn),
 	}, nil
 }

backend/pkg/objectstorage/objectstorage.go

@@ -18,4 +18,5 @@ type ObjectStorage interface {
 	Get(key string) (io.ReadCloser, error)
 	Exists(key string) bool
 	GetCreationTime(key string) *time.Time
+	GetPreSignedUploadUrl(key string) (string, error)
 }

backend/pkg/objectstorage/s3/s3.go

@@ -194,6 +194,18 @@ func (s *storageImpl) GetFrequentlyUsedKeys(projectID uint64) ([]string, error)
 	return keyList, nil
 }
 
+func (s *storageImpl) GetPreSignedUploadUrl(key string) (string, error) {
+	req, _ := s.svc.PutObjectRequest(&s3.PutObjectInput{
+		Bucket: aws.String(*s.bucket),
+		Key:    aws.String(key),
+	})
+	urlStr, err := req.Presign(15 * time.Minute)
+	if err != nil {
+		return "", err
+	}
+	return urlStr, nil
+}
+
 func loadFileTag() string {
 	// Load file tag from env
 	key := "retention"

backend/pkg/uxtesting/uxtesting.go

@@ -0,0 +1,127 @@
+package uxtesting
+
+import (
+	"openreplay/backend/pkg/db/postgres/pool"
+)
+
+type UXTesting interface {
+	GetInfo(testID string) (*UXTestInfo, error)
+	SetTestSignal(testSignal *TestSignal) error
+	SetTaskSignal(taskSignal *TaskSignal) error
+}
+
+type uxTestingImpl struct {
+	db pool.Pool
+}
+
+func New(db pool.Pool) UXTesting {
+	return &uxTestingImpl{
+		db: db,
+	}
+}
+
+type UXTestInfo struct {
+	Title        string        `json:"title"`
+	Description  string        `json:"description"`
+	StartingPath string        `json:"startingPath"`
+	Status       string        `json:"status"`
+	ReqMic       bool          `json:"reqMic"`
+	ReqCamera    bool          `json:"reqCamera"`
+	Guidelines   string        `json:"guidelines"`
+	Conclusion   string        `json:"conclusion"`
+	Tasks        []interface{} `json:"tasks"`
+}
+
+func (u *uxTestingImpl) GetInfo(testID string) (*UXTestInfo, error) {
+	info := &UXTestInfo{}
+	var description, startingPath, guidelines, conclusion *string
+	err := u.db.QueryRow(`
+		SELECT
+			ut_tests.title,
+			ut_tests.description,
+			ut_tests.starting_path,
+			ut_tests.status,
+			ut_tests.require_mic,
+			ut_tests.require_camera,
+			ut_tests.guidelines,
+			ut_tests.conclusion_message,
+			json_agg(
+					json_build_object(
+							'task_id', ut_tests_tasks.task_id,
+					    	'title', ut_tests_tasks.title,
+							'description', ut_tests_tasks.description,
+							'allow_typing', ut_tests_tasks.allow_typing
+						)
+				) AS tasks
+		FROM
+			ut_tests
+				JOIN
+			ut_tests_tasks ON ut_tests.test_id = ut_tests_tasks.test_id
+		WHERE ut_tests.test_id = $1
+		GROUP BY
+			ut_tests.test_id;
+	`, testID).Scan(&info.Title, &description, &startingPath, &info.Status, &info.ReqMic, &info.ReqCamera,
+		&guidelines, &conclusion, &info.Tasks)
+	if err != nil {
+		return nil, err
+	}
+	if description != nil {
+		info.Description = *description
+	}
+	if startingPath != nil {
+		info.StartingPath = *startingPath
+	}
+	if guidelines != nil {
+		info.Guidelines = *guidelines
+	}
+	if conclusion != nil {
+		info.Conclusion = *conclusion
+	}
+	return info, nil
+}
+
+type TestSignal struct {
+	SessionID uint64 `json:"sessionID"`
+	TestID    int    `json:"testID"`
+	Status    string `json:"status"`
+	Timestamp uint64 `json:"timestamp"`
+	Duration  uint64 `json:"duration"`
+}
+
+func (u *uxTestingImpl) SetTestSignal(signal *TestSignal) error {
+	if err := u.db.Exec(`
+		INSERT INTO ut_tests_signals (
+			session_id, test_id, status, timestamp, duration
+		) VALUES (
+			$1, $2, $3, $4, $5
+		)`,
+		signal.SessionID, signal.TestID, signal.Status, signal.Timestamp, signal.Duration,
+	); err != nil {
+		return err
+	}
+	return nil
+}
+
+type TaskSignal struct {
+	SessionID uint64 `json:"sessionID"`
+	TestID    int    `json:"testID"`
+	TaskID    int    `json:"taskID"`
+	Status    string `json:"status"`
+	Answer    string `json:"answer"`
+	Timestamp uint64 `json:"timestamp"`
+	Duration  uint64 `json:"duration"`
+}
+
+func (u *uxTestingImpl) SetTaskSignal(signal *TaskSignal) error {
+	if err := u.db.Exec(`
+		INSERT INTO ut_tests_signals (
+			session_id, test_id, task_id, status, comment, timestamp, duration
+		) VALUES (
+			$1, $2, $3, $4, $5, $6, $7
+		)`,
+		signal.SessionID, signal.TestID, signal.TaskID, signal.Status, signal.Answer, signal.Timestamp, signal.Duration,
+	); err != nil {
+		return err
+	}
+	return nil
+}

scripts/helmcharts/openreplay/charts/http/values.yaml

@@ -102,6 +102,7 @@ autoscaling:
 env:
   TOKEN_SECRET: secret_token_string # TODO: generate on buld
   CACHE_ASSETS: true
+  BUCKET_NAME: 'uxtesting-records'
 
 
 nodeSelector: {}