openreplay/backend/internal/http/router/handlers-mobile.go
Alexander d22e4e138e
Backend for mobile tracker support (#1456)
* feat(backend): handlers for mobile messages

* feat(backend): new service template

* feat(backend): save mobile session start and send batches to kafka

* feat(backend): added mobile support to sink, ender and storage

* helm(videostorage): added helm chart for a new service videostorage

* fix(backend): added pointer to streing for userBrowser (because it's null for mobile sessions)

* feat(backend): added MsgIOSBatchMeta handler to message iterator's logic

* feat(backend): added ios ts parser to ender

* feat(backend): enabled producing batch of messages to queue

* feat(backend): removed iosstart from mob files

* feat(backend): added new ios message types

* feat(backend): added iosStart and iosEnd

* fix(backend): fixed log issue

* feat(backend): send tar.gz archives to special queue topic

* feat(backend): read raw archives from kafka

* fix(backend): added missing file

* fix(backend): removed the second file reading

* fix(backend): fixed wrong queue topic name

* fix(backend): fixed mobile trigger topic name

* feat(backend): added tar.gz extractor and iOSSessionEnd handler

* feat(backend): debug logs on message uploading

* fix(backend): added raw-images topic consumption

* feat(backend): now sink send iosSessionEnd to video-storage

* feat(backend): added dir creator for new sessions

* feat(backend): added correct command to execute

* feat(backend): added overwrite option

* feat(backend): added s3 uploader for video session replays

* feat(backend): new consumer group for mobile sessions

* feat(backend): debug logs for uploader

* feat(backend): removed unused log

* feat(backend): fixed s3 key for video replays

* feat(backend): removed debug code

* feat(backend): fixed video-storage message filter

* fix(backend): added mobileSessionEnd to SessionEnd converter

* feat(backend): added first version if db support for mobile events

* fix(backend): added swipe events to mob file

* feat(backend): added swipe event to pg

* feat(backend): split logic into 2 services: image-storage and video-storage

* feat(backend): added helm chart for image-storage service

* fix(backend): fixed table name for mobile taps

* feat(backend): added metadata handler for mobile message parser + fix message filters

* feat(backend): added iosRawTopic to DB message consumer

* fix(backend): removed value from mobile inputs

* feat(backend): removed debug log from iterator

* feat(backend): added new apple devices to iOS device parser

* fix(backend): added real projectID instead of 0

* feat(backend): extended a list of simulators for device detector

* feat(backend): updated networkCall mobile message

* fix(backend): added new way to define is network call successed or not

* feat(backend): added timezone support for mobile start request

* feat(backend): added 2 mobile events Input and Click to mob file

* feat(backend): refactored image storage

* feat(backend): video storage with 2 workers

* feat(backend): added project's platform support

* feat(backend): added memory size field for mobile start request

* feat(backend): changed video preset for ultrafast

* feat(backend): added debug log to http /late handler

* feat(backend): added debug log to db service for iosCrash messages

* feat(backend): added tapRage event handler to heuristics

* feat(backend): changed table and field names for ios crashes

* feat(backend): added payload for tapRage events

* feat(backend): added TapRage events insert to DB

* feat(backend): added fps value to /mobile/start response

* feat(backend): added image quality parameter to /mobile/start response

* feat(backend): added ScreenLeave handler

* feat(backend): removed screenEnter and screenLeave events, added new viewComponent event

---------

Co-authored-by: rjshrjndrn <rjshrjndrn@gmail.com>
2023-10-09 15:02:20 +02:00

224 lines
7.6 KiB
Go

package router
import (
"encoding/json"
"errors"
"io/ioutil"
"log"
"math/rand"
"net/http"
"openreplay/backend/internal/http/ios"
"openreplay/backend/internal/http/util"
"openreplay/backend/internal/http/uuid"
"openreplay/backend/pkg/db/postgres"
"openreplay/backend/pkg/messages"
"openreplay/backend/pkg/sessions"
"openreplay/backend/pkg/token"
"strconv"
"time"
)
func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
req := &StartIOSSessionRequest{}
if r.Body == nil {
ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty"), startTime, r.URL.Path, 0)
return
}
body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit)
defer body.Close()
if err := json.NewDecoder(body).Decode(req); err != nil {
ResponseWithError(w, http.StatusBadRequest, err, startTime, r.URL.Path, 0)
return
}
if req.ProjectKey == nil {
ResponseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required"), startTime, r.URL.Path, 0)
return
}
p, err := e.services.Projects.GetProjectByKey(*req.ProjectKey)
if err != nil {
if postgres.IsNoRowsErr(err) {
ResponseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or is not active"), startTime, r.URL.Path, 0)
} else {
ResponseWithError(w, http.StatusInternalServerError, err, startTime, r.URL.Path, 0) // TODO: send error here only on staging
}
return
}
// Check if the project supports mobile sessions
if !p.IsMobile() {
ResponseWithError(w, http.StatusForbidden, errors.New("project doesn't support mobile sessions"), startTime, r.URL.Path, 0)
return
}
userUUID := uuid.GetUUID(req.UserUUID)
tokenData, err := e.services.Tokenizer.Parse(req.Token)
if err != nil { // Starting the new one
dice := byte(rand.Intn(100)) // [0, 100)
if dice >= p.SampleRate {
ResponseWithError(w, http.StatusForbidden, errors.New("cancel"), startTime, r.URL.Path, 0)
return
}
ua := e.services.UaParser.ParseFromHTTPRequest(r)
if ua == nil {
ResponseWithError(w, http.StatusForbidden, errors.New("browser not recognized"), startTime, r.URL.Path, 0)
return
}
sessionID, err := e.services.Flaker.Compose(uint64(startTime.UnixMilli()))
if err != nil {
ResponseWithError(w, http.StatusInternalServerError, err, startTime, r.URL.Path, 0)
return
}
// TODO: if EXPIRED => send message for two sessions association
expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond)
tokenData = &token.TokenData{sessionID, 0, expTime.UnixMilli()}
geoInfo := e.ExtractGeoData(r)
if err := e.services.Sessions.Add(&sessions.Session{
SessionID: sessionID,
Platform: "ios",
Timestamp: req.Timestamp,
Timezone: req.Timezone,
ProjectID: p.ProjectID,
TrackerVersion: req.TrackerVersion,
RevID: req.RevID,
UserUUID: userUUID,
UserOS: "IOS",
UserOSVersion: req.UserOSVersion,
UserDevice: ios.MapIOSDevice(req.UserDevice),
UserDeviceType: ios.GetIOSDeviceType(req.UserDevice),
UserCountry: geoInfo.Country,
UserState: geoInfo.State,
UserCity: geoInfo.City,
UserDeviceMemorySize: req.DeviceMemory,
UserDeviceHeapSize: req.DeviceMemory,
}); err != nil {
log.Printf("failed to add mobile session to DB: %v", err)
}
sessStart := &messages.IOSSessionStart{
Timestamp: req.Timestamp,
ProjectID: uint64(p.ProjectID),
TrackerVersion: req.TrackerVersion,
RevID: req.RevID,
UserUUID: userUUID,
UserOS: "IOS",
UserOSVersion: req.UserOSVersion,
UserDevice: ios.MapIOSDevice(req.UserDevice),
UserDeviceType: ios.GetIOSDeviceType(req.UserDevice),
UserCountry: geoInfo.Pack(),
}
log.Printf("mobile session start: %+v", sessStart)
if err := e.services.Producer.Produce(e.cfg.TopicRawIOS, tokenData.ID, sessStart.Encode()); err != nil {
log.Printf("failed to produce mobile session start message: %v", err)
}
}
ResponseWithJSON(w, &StartIOSSessionResponse{
Token: e.services.Tokenizer.Compose(*tokenData),
UserUUID: userUUID,
SessionID: strconv.FormatUint(tokenData.ID, 10),
BeaconSizeLimit: e.cfg.BeaconSizeLimit,
ImageQuality: "standard", // Pull from project settings (low, standard, high)
FrameRate: 3, // Pull from project settings
}, startTime, r.URL.Path, 0)
}
func (e *Router) pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r)
if err != nil {
ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, 0)
return
}
e.pushMessages(w, r, sessionData.ID, e.cfg.TopicRawIOS)
}
func (e *Router) pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r)
if err != nil && err != token.EXPIRED {
ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, 0)
return
}
// Check timestamps here?
e.pushMessages(w, r, sessionData.ID, e.cfg.TopicRawIOS)
}
func (e *Router) imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
log.Printf("recieved imagerequest")
sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r)
if err != nil { // Should accept expired token?
ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, 0)
return
}
if r.Body == nil {
ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty"), startTime, r.URL.Path, 0)
return
}
r.Body = http.MaxBytesReader(w, r.Body, e.cfg.FileSizeLimit)
defer r.Body.Close()
err = r.ParseMultipartForm(5 * 1e6) // ~5Mb
if err == http.ErrNotMultipart || err == http.ErrMissingBoundary {
ResponseWithError(w, http.StatusUnsupportedMediaType, err, startTime, r.URL.Path, 0)
return
// } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB
} else if err != nil {
ResponseWithError(w, http.StatusInternalServerError, err, startTime, r.URL.Path, 0) // TODO: send error here only on staging
return
}
if r.MultipartForm == nil {
ResponseWithError(w, http.StatusInternalServerError, errors.New("Multipart not parsed"), startTime, r.URL.Path, 0)
return
}
if len(r.MultipartForm.Value["projectKey"]) == 0 {
ResponseWithError(w, http.StatusBadRequest, errors.New("projectKey parameter missing"), startTime, r.URL.Path, 0) // status for missing/wrong parameter?
return
}
//prefix := r.MultipartForm.Value["projectKey"][0] + "/" + strconv.FormatUint(sessionData.ID, 10) + "/"
for _, fileHeaderList := range r.MultipartForm.File {
for _, fileHeader := range fileHeaderList {
file, err := fileHeader.Open()
if err != nil {
continue // TODO: send server error or accumulate successful files
}
//key := prefix + fileHeader.Filename
data, err := ioutil.ReadAll(file)
if err != nil {
log.Fatalf("failed reading data: %s", err)
}
log.Printf("Uploading image... %v, len: %d", util.SafeString(fileHeader.Filename), len(data))
if err := e.services.Producer.Produce(e.cfg.TopicRawImages, sessionData.ID, data); err != nil {
log.Printf("failed to produce mobile session start message: %v", err)
}
log.Printf("Image uploaded")
//go func() { //TODO: mime type from header
// log.Printf("Uploading image... %v", file)
// //if err := e.services.Storage.Upload(file, key, "image/jpeg", false); err != nil {
// // log.Printf("Upload ios screen error. %v", err)
// //}
//}()
}
}
w.WriteHeader(http.StatusOK)
}