From f321fccc11c8ab5411c7bd2415de7d9049d6f458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=B5=84=E2=B5=8E=E2=B5=89=E2=B5=94=E2=B5=93=E2=B5=9B?= Date: Mon, 27 Nov 2023 15:58:36 +0100 Subject: [PATCH] Ux testing backend (#1709) * feat(backend): added ux-testing support * feat(backend): added ux-testing module * feat(http): added bucket name for http service * feat(backend): fixed small typos in http router --------- Co-authored-by: Alexander --- backend/internal/config/http/config.go | 2 + backend/internal/http/router/handlers-web.go | 124 +++++++++++++++++ backend/internal/http/router/router.go | 28 ++-- backend/internal/http/services/services.go | 13 ++ backend/pkg/objectstorage/objectstorage.go | 1 + backend/pkg/objectstorage/s3/s3.go | 12 ++ backend/pkg/uxtesting/uxtesting.go | 127 ++++++++++++++++++ .../openreplay/charts/http/values.yaml | 1 + 8 files changed, 299 insertions(+), 9 deletions(-) create mode 100644 backend/pkg/uxtesting/uxtesting.go diff --git a/backend/internal/config/http/config.go b/backend/internal/config/http/config.go index 76c1662d6..b27aa0dc0 100644 --- a/backend/internal/config/http/config.go +++ b/backend/internal/config/http/config.go @@ -3,6 +3,7 @@ package http import ( "openreplay/backend/internal/config/common" "openreplay/backend/internal/config/configurator" + "openreplay/backend/internal/config/objectstorage" "openreplay/backend/internal/config/redis" "openreplay/backend/pkg/env" "time" @@ -12,6 +13,7 @@ type Config struct { common.Config common.Postgres redis.Redis + objectstorage.ObjectsConfig HTTPHost string `env:"HTTP_HOST,default="` HTTPPort string `env:"HTTP_PORT,required"` HTTPTimeout time.Duration `env:"HTTP_TIMEOUT,default=60s"` diff --git a/backend/internal/http/router/handlers-web.go b/backend/internal/http/router/handlers-web.go index 21c484e86..c0f47e37a 100644 --- a/backend/internal/http/router/handlers-web.go +++ b/backend/internal/http/router/handlers-web.go @@ -4,12 +4,14 @@ import ( "encoding/json" "errors" "fmt" + "github.com/gorilla/mux" "io" "log" "math/rand" "net/http" "openreplay/backend/pkg/featureflags" "openreplay/backend/pkg/sessions" + "openreplay/backend/pkg/uxtesting" "strconv" "time" @@ -364,3 +366,125 @@ func (e *Router) featureFlagsHandlerWeb(w http.ResponseWriter, r *http.Request) } ResponseWithJSON(w, resp, startTime, r.URL.Path, bodySize) } + +func (e *Router) getUXTestInfo(w http.ResponseWriter, r *http.Request) { + startTime := time.Now() + bodySize := 0 + + // Check authorization + _, err := e.services.Tokenizer.ParseFromHTTPRequest(r) + if err != nil { + ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, bodySize) + return + } + + // Get taskID + vars := mux.Vars(r) + id := vars["id"] + + // Get task info + info, err := e.services.UXTesting.GetInfo(id) + if err != nil { + ResponseWithError(w, http.StatusInternalServerError, err, startTime, r.URL.Path, bodySize) + return + } + type TaskInfoResponse struct { + Task *uxtesting.UXTestInfo `json:"test"` + } + ResponseWithJSON(w, &TaskInfoResponse{Task: info}, startTime, r.URL.Path, bodySize) +} + +func (e *Router) sendUXTestSignal(w http.ResponseWriter, r *http.Request) { + startTime := time.Now() + bodySize := 0 + + // Check authorization + sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r) + if err != nil { + ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, bodySize) + return + } + + bodyBytes, err := e.readBody(w, r, e.cfg.JsonSizeLimit) + if err != nil { + log.Printf("error while reading request body: %s", err) + ResponseWithError(w, http.StatusRequestEntityTooLarge, err, startTime, r.URL.Path, bodySize) + return + } + bodySize = len(bodyBytes) + + // Parse request body + req := &uxtesting.TestSignal{} + + if err := json.Unmarshal(bodyBytes, req); err != nil { + ResponseWithError(w, http.StatusBadRequest, err, startTime, r.URL.Path, bodySize) + return + } + req.SessionID = sessionData.ID + + // Save test signal + if err := e.services.UXTesting.SetTestSignal(req); err != nil { + ResponseWithError(w, http.StatusBadRequest, err, startTime, r.URL.Path, bodySize) + return + } + ResponseOK(w, startTime, r.URL.Path, bodySize) +} + +func (e *Router) sendUXTaskSignal(w http.ResponseWriter, r *http.Request) { + startTime := time.Now() + bodySize := 0 + + // Check authorization + sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r) + if err != nil { + ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, bodySize) + return + } + + bodyBytes, err := e.readBody(w, r, e.cfg.JsonSizeLimit) + if err != nil { + log.Printf("error while reading request body: %s", err) + ResponseWithError(w, http.StatusRequestEntityTooLarge, err, startTime, r.URL.Path, bodySize) + return + } + bodySize = len(bodyBytes) + + // Parse request body + req := &uxtesting.TaskSignal{} + + if err := json.Unmarshal(bodyBytes, req); err != nil { + ResponseWithError(w, http.StatusBadRequest, err, startTime, r.URL.Path, bodySize) + return + } + req.SessionID = sessionData.ID + + // Save test signal + if err := e.services.UXTesting.SetTaskSignal(req); err != nil { + ResponseWithError(w, http.StatusBadRequest, err, startTime, r.URL.Path, bodySize) + return + } + ResponseOK(w, startTime, r.URL.Path, bodySize) +} + +func (e *Router) getUXUploadUrl(w http.ResponseWriter, r *http.Request) { + startTime := time.Now() + bodySize := 0 + + // Check authorization + sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r) + if err != nil { + ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, bodySize) + return + } + + key := fmt.Sprintf("%d/ux_webcam_record.mp4", sessionData.ID) + url, err := e.services.ObjStorage.GetPreSignedUploadUrl(key) + if err != nil { + ResponseWithError(w, http.StatusInternalServerError, err, startTime, r.URL.Path, bodySize) + return + } + type UrlResponse struct { + URL string `json:"url"` + } + ResponseWithJSON(w, &UrlResponse{URL: url}, startTime, r.URL.Path, bodySize) +} diff --git a/backend/internal/http/router/router.go b/backend/internal/http/router/router.go index e1eacc65d..67d3c4562 100644 --- a/backend/internal/http/router/router.go +++ b/backend/internal/http/router/router.go @@ -101,14 +101,20 @@ func (e *Router) init() { e.router.HandleFunc("/", e.root) handlers := map[string]func(http.ResponseWriter, *http.Request){ - "/v1/web/not-started": e.notStartedHandlerWeb, - "/v1/web/start": e.startSessionHandlerWeb, - "/v1/web/i": e.pushMessagesHandlerWeb, - "/v1/web/feature-flags": e.featureFlagsHandlerWeb, - "/v1/mobile/start": e.startSessionHandlerIOS, - "/v1/mobile/i": e.pushMessagesHandlerIOS, - "/v1/mobile/late": e.pushLateMessagesHandlerIOS, - "/v1/mobile/images": e.imagesUploadHandlerIOS, + "/v1/web/not-started": e.notStartedHandlerWeb, + "/v1/web/start": e.startSessionHandlerWeb, + "/v1/web/i": e.pushMessagesHandlerWeb, + "/v1/web/feature-flags": e.featureFlagsHandlerWeb, + "/v1/mobile/start": e.startSessionHandlerIOS, + "/v1/mobile/i": e.pushMessagesHandlerIOS, + "/v1/mobile/late": e.pushLateMessagesHandlerIOS, + "/v1/mobile/images": e.imagesUploadHandlerIOS, + "/v1/web/uxt/signals/test": e.sendUXTestSignal, + "/v1/web/uxt/signals/task": e.sendUXTaskSignal, + } + getHandlers := map[string]func(http.ResponseWriter, *http.Request){ + "/v1/web/uxt/test/{id}": e.getUXTestInfo, + "/v1/web/uxt/upload-url": e.getUXUploadUrl, } prefix := "/ingest" @@ -116,6 +122,10 @@ func (e *Router) init() { e.router.HandleFunc(path, handler).Methods("POST", "OPTIONS") e.router.HandleFunc(prefix+path, handler).Methods("POST", "OPTIONS") } + for path, handler := range getHandlers { + e.router.HandleFunc(path, handler).Methods("GET", "OPTIONS") + e.router.HandleFunc(prefix+path, handler).Methods("GET", "OPTIONS") + } // CORS middleware e.router.Use(e.corsMiddleware) @@ -130,7 +140,7 @@ func (e *Router) corsMiddleware(next http.Handler) http.Handler { if e.cfg.UseAccessControlHeaders { // Prepare headers for preflight requests w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "POST") + w.Header().Set("Access-Control-Allow-Methods", "POST,GET") w.Header().Set("Access-Control-Allow-Headers", "Content-Type,Authorization,Content-Encoding") } if r.Method == http.MethodOptions { diff --git a/backend/internal/http/services/services.go b/backend/internal/http/services/services.go index 0c2892905..e8578d429 100644 --- a/backend/internal/http/services/services.go +++ b/backend/internal/http/services/services.go @@ -1,6 +1,7 @@ package services import ( + "log" "openreplay/backend/internal/config/http" "openreplay/backend/internal/http/geoip" "openreplay/backend/internal/http/uaparser" @@ -8,10 +9,13 @@ import ( "openreplay/backend/pkg/db/redis" "openreplay/backend/pkg/featureflags" "openreplay/backend/pkg/flakeid" + "openreplay/backend/pkg/objectstorage" + "openreplay/backend/pkg/objectstorage/store" "openreplay/backend/pkg/projects" "openreplay/backend/pkg/queue/types" "openreplay/backend/pkg/sessions" "openreplay/backend/pkg/token" + "openreplay/backend/pkg/uxtesting" ) type ServicesBuilder struct { @@ -23,10 +27,17 @@ type ServicesBuilder struct { UaParser *uaparser.UAParser GeoIP geoip.GeoParser Tokenizer *token.Tokenizer + ObjStorage objectstorage.ObjectStorage + UXTesting uxtesting.UXTesting } func New(cfg *http.Config, producer types.Producer, pgconn pool.Pool, redis *redis.Client) (*ServicesBuilder, error) { projs := projects.New(pgconn, redis) + // ObjectStorage client to generate pre-signed upload urls + objStore, err := store.NewStore(&cfg.ObjectsConfig) + if err != nil { + log.Fatalf("can't init object storage: %s", err) + } return &ServicesBuilder{ Projects: projs, Sessions: sessions.New(pgconn, projs, redis), @@ -36,5 +47,7 @@ func New(cfg *http.Config, producer types.Producer, pgconn pool.Pool, redis *red UaParser: uaparser.NewUAParser(cfg.UAParserFile), GeoIP: geoip.New(cfg.MaxMinDBFile), Flaker: flakeid.NewFlaker(cfg.WorkerID), + ObjStorage: objStore, + UXTesting: uxtesting.New(pgconn), }, nil } diff --git a/backend/pkg/objectstorage/objectstorage.go b/backend/pkg/objectstorage/objectstorage.go index 2504ba02c..63537d774 100644 --- a/backend/pkg/objectstorage/objectstorage.go +++ b/backend/pkg/objectstorage/objectstorage.go @@ -18,4 +18,5 @@ type ObjectStorage interface { Get(key string) (io.ReadCloser, error) Exists(key string) bool GetCreationTime(key string) *time.Time + GetPreSignedUploadUrl(key string) (string, error) } diff --git a/backend/pkg/objectstorage/s3/s3.go b/backend/pkg/objectstorage/s3/s3.go index 27e68698d..476413d8a 100644 --- a/backend/pkg/objectstorage/s3/s3.go +++ b/backend/pkg/objectstorage/s3/s3.go @@ -194,6 +194,18 @@ func (s *storageImpl) GetFrequentlyUsedKeys(projectID uint64) ([]string, error) return keyList, nil } +func (s *storageImpl) GetPreSignedUploadUrl(key string) (string, error) { + req, _ := s.svc.PutObjectRequest(&s3.PutObjectInput{ + Bucket: aws.String(*s.bucket), + Key: aws.String(key), + }) + urlStr, err := req.Presign(15 * time.Minute) + if err != nil { + return "", err + } + return urlStr, nil +} + func loadFileTag() string { // Load file tag from env key := "retention" diff --git a/backend/pkg/uxtesting/uxtesting.go b/backend/pkg/uxtesting/uxtesting.go new file mode 100644 index 000000000..5c0b1fc9a --- /dev/null +++ b/backend/pkg/uxtesting/uxtesting.go @@ -0,0 +1,127 @@ +package uxtesting + +import ( + "openreplay/backend/pkg/db/postgres/pool" +) + +type UXTesting interface { + GetInfo(testID string) (*UXTestInfo, error) + SetTestSignal(testSignal *TestSignal) error + SetTaskSignal(taskSignal *TaskSignal) error +} + +type uxTestingImpl struct { + db pool.Pool +} + +func New(db pool.Pool) UXTesting { + return &uxTestingImpl{ + db: db, + } +} + +type UXTestInfo struct { + Title string `json:"title"` + Description string `json:"description"` + StartingPath string `json:"startingPath"` + Status string `json:"status"` + ReqMic bool `json:"reqMic"` + ReqCamera bool `json:"reqCamera"` + Guidelines string `json:"guidelines"` + Conclusion string `json:"conclusion"` + Tasks []interface{} `json:"tasks"` +} + +func (u *uxTestingImpl) GetInfo(testID string) (*UXTestInfo, error) { + info := &UXTestInfo{} + var description, startingPath, guidelines, conclusion *string + err := u.db.QueryRow(` + SELECT + ut_tests.title, + ut_tests.description, + ut_tests.starting_path, + ut_tests.status, + ut_tests.require_mic, + ut_tests.require_camera, + ut_tests.guidelines, + ut_tests.conclusion_message, + json_agg( + json_build_object( + 'task_id', ut_tests_tasks.task_id, + 'title', ut_tests_tasks.title, + 'description', ut_tests_tasks.description, + 'allow_typing', ut_tests_tasks.allow_typing + ) + ) AS tasks + FROM + ut_tests + JOIN + ut_tests_tasks ON ut_tests.test_id = ut_tests_tasks.test_id + WHERE ut_tests.test_id = $1 + GROUP BY + ut_tests.test_id; + `, testID).Scan(&info.Title, &description, &startingPath, &info.Status, &info.ReqMic, &info.ReqCamera, + &guidelines, &conclusion, &info.Tasks) + if err != nil { + return nil, err + } + if description != nil { + info.Description = *description + } + if startingPath != nil { + info.StartingPath = *startingPath + } + if guidelines != nil { + info.Guidelines = *guidelines + } + if conclusion != nil { + info.Conclusion = *conclusion + } + return info, nil +} + +type TestSignal struct { + SessionID uint64 `json:"sessionID"` + TestID int `json:"testID"` + Status string `json:"status"` + Timestamp uint64 `json:"timestamp"` + Duration uint64 `json:"duration"` +} + +func (u *uxTestingImpl) SetTestSignal(signal *TestSignal) error { + if err := u.db.Exec(` + INSERT INTO ut_tests_signals ( + session_id, test_id, status, timestamp, duration + ) VALUES ( + $1, $2, $3, $4, $5 + )`, + signal.SessionID, signal.TestID, signal.Status, signal.Timestamp, signal.Duration, + ); err != nil { + return err + } + return nil +} + +type TaskSignal struct { + SessionID uint64 `json:"sessionID"` + TestID int `json:"testID"` + TaskID int `json:"taskID"` + Status string `json:"status"` + Answer string `json:"answer"` + Timestamp uint64 `json:"timestamp"` + Duration uint64 `json:"duration"` +} + +func (u *uxTestingImpl) SetTaskSignal(signal *TaskSignal) error { + if err := u.db.Exec(` + INSERT INTO ut_tests_signals ( + session_id, test_id, task_id, status, comment, timestamp, duration + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7 + )`, + signal.SessionID, signal.TestID, signal.TaskID, signal.Status, signal.Answer, signal.Timestamp, signal.Duration, + ); err != nil { + return err + } + return nil +} diff --git a/scripts/helmcharts/openreplay/charts/http/values.yaml b/scripts/helmcharts/openreplay/charts/http/values.yaml index ab63af8f6..2a8e9df73 100644 --- a/scripts/helmcharts/openreplay/charts/http/values.yaml +++ b/scripts/helmcharts/openreplay/charts/http/values.yaml @@ -102,6 +102,7 @@ autoscaling: env: TOKEN_SECRET: secret_token_string # TODO: generate on buld CACHE_ASSETS: true + BUCKET_NAME: 'uxtesting-records' nodeSelector: {}