* feat(backend): handlers for mobile messages * feat(backend): new service template * feat(backend): save mobile session start and send batches to kafka * feat(backend): added mobile support to sink, ender and storage * helm(videostorage): added helm chart for a new service videostorage * fix(backend): added pointer to streing for userBrowser (because it's null for mobile sessions) * feat(backend): added MsgIOSBatchMeta handler to message iterator's logic * feat(backend): added ios ts parser to ender * feat(backend): enabled producing batch of messages to queue * feat(backend): removed iosstart from mob files * feat(backend): added new ios message types * feat(backend): added iosStart and iosEnd * fix(backend): fixed log issue * feat(backend): send tar.gz archives to special queue topic * feat(backend): read raw archives from kafka * fix(backend): added missing file * fix(backend): removed the second file reading * fix(backend): fixed wrong queue topic name * fix(backend): fixed mobile trigger topic name * feat(backend): added tar.gz extractor and iOSSessionEnd handler * feat(backend): debug logs on message uploading * fix(backend): added raw-images topic consumption * feat(backend): now sink send iosSessionEnd to video-storage * feat(backend): added dir creator for new sessions * feat(backend): added correct command to execute * feat(backend): added overwrite option * feat(backend): added s3 uploader for video session replays * feat(backend): new consumer group for mobile sessions * feat(backend): debug logs for uploader * feat(backend): removed unused log * feat(backend): fixed s3 key for video replays * feat(backend): removed debug code * feat(backend): fixed video-storage message filter * fix(backend): added mobileSessionEnd to SessionEnd converter * feat(backend): added first version if db support for mobile events * fix(backend): added swipe events to mob file * feat(backend): added swipe event to pg * feat(backend): split logic into 2 services: image-storage and video-storage * feat(backend): added helm chart for image-storage service * fix(backend): fixed table name for mobile taps * feat(backend): added metadata handler for mobile message parser + fix message filters * feat(backend): added iosRawTopic to DB message consumer * fix(backend): removed value from mobile inputs * feat(backend): removed debug log from iterator * feat(backend): added new apple devices to iOS device parser * fix(backend): added real projectID instead of 0 * feat(backend): extended a list of simulators for device detector * feat(backend): updated networkCall mobile message * fix(backend): added new way to define is network call successed or not * feat(backend): added timezone support for mobile start request * feat(backend): added 2 mobile events Input and Click to mob file * feat(backend): refactored image storage * feat(backend): video storage with 2 workers * feat(backend): added project's platform support * feat(backend): added memory size field for mobile start request * feat(backend): changed video preset for ultrafast * feat(backend): added debug log to http /late handler * feat(backend): added debug log to db service for iosCrash messages * feat(backend): added tapRage event handler to heuristics * feat(backend): changed table and field names for ios crashes * feat(backend): added payload for tapRage events * feat(backend): added TapRage events insert to DB * feat(backend): added fps value to /mobile/start response * feat(backend): added image quality parameter to /mobile/start response * feat(backend): added ScreenLeave handler * feat(backend): removed screenEnter and screenLeave events, added new viewComponent event --------- Co-authored-by: rjshrjndrn <rjshrjndrn@gmail.com>
224 lines
7.6 KiB
Go
224 lines
7.6 KiB
Go
package router
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"io/ioutil"
|
|
"log"
|
|
"math/rand"
|
|
"net/http"
|
|
"openreplay/backend/internal/http/ios"
|
|
"openreplay/backend/internal/http/util"
|
|
"openreplay/backend/internal/http/uuid"
|
|
"openreplay/backend/pkg/db/postgres"
|
|
"openreplay/backend/pkg/messages"
|
|
"openreplay/backend/pkg/sessions"
|
|
"openreplay/backend/pkg/token"
|
|
"strconv"
|
|
"time"
|
|
)
|
|
|
|
func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) {
|
|
startTime := time.Now()
|
|
req := &StartIOSSessionRequest{}
|
|
|
|
if r.Body == nil {
|
|
ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty"), startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit)
|
|
defer body.Close()
|
|
|
|
if err := json.NewDecoder(body).Decode(req); err != nil {
|
|
ResponseWithError(w, http.StatusBadRequest, err, startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
|
|
if req.ProjectKey == nil {
|
|
ResponseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required"), startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
|
|
p, err := e.services.Projects.GetProjectByKey(*req.ProjectKey)
|
|
if err != nil {
|
|
if postgres.IsNoRowsErr(err) {
|
|
ResponseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or is not active"), startTime, r.URL.Path, 0)
|
|
} else {
|
|
ResponseWithError(w, http.StatusInternalServerError, err, startTime, r.URL.Path, 0) // TODO: send error here only on staging
|
|
}
|
|
return
|
|
}
|
|
|
|
// Check if the project supports mobile sessions
|
|
if !p.IsMobile() {
|
|
ResponseWithError(w, http.StatusForbidden, errors.New("project doesn't support mobile sessions"), startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
|
|
userUUID := uuid.GetUUID(req.UserUUID)
|
|
tokenData, err := e.services.Tokenizer.Parse(req.Token)
|
|
|
|
if err != nil { // Starting the new one
|
|
dice := byte(rand.Intn(100)) // [0, 100)
|
|
if dice >= p.SampleRate {
|
|
ResponseWithError(w, http.StatusForbidden, errors.New("cancel"), startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
|
|
ua := e.services.UaParser.ParseFromHTTPRequest(r)
|
|
if ua == nil {
|
|
ResponseWithError(w, http.StatusForbidden, errors.New("browser not recognized"), startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
sessionID, err := e.services.Flaker.Compose(uint64(startTime.UnixMilli()))
|
|
if err != nil {
|
|
ResponseWithError(w, http.StatusInternalServerError, err, startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
// TODO: if EXPIRED => send message for two sessions association
|
|
expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond)
|
|
tokenData = &token.TokenData{sessionID, 0, expTime.UnixMilli()}
|
|
|
|
geoInfo := e.ExtractGeoData(r)
|
|
|
|
if err := e.services.Sessions.Add(&sessions.Session{
|
|
SessionID: sessionID,
|
|
Platform: "ios",
|
|
Timestamp: req.Timestamp,
|
|
Timezone: req.Timezone,
|
|
ProjectID: p.ProjectID,
|
|
TrackerVersion: req.TrackerVersion,
|
|
RevID: req.RevID,
|
|
UserUUID: userUUID,
|
|
UserOS: "IOS",
|
|
UserOSVersion: req.UserOSVersion,
|
|
UserDevice: ios.MapIOSDevice(req.UserDevice),
|
|
UserDeviceType: ios.GetIOSDeviceType(req.UserDevice),
|
|
UserCountry: geoInfo.Country,
|
|
UserState: geoInfo.State,
|
|
UserCity: geoInfo.City,
|
|
UserDeviceMemorySize: req.DeviceMemory,
|
|
UserDeviceHeapSize: req.DeviceMemory,
|
|
}); err != nil {
|
|
log.Printf("failed to add mobile session to DB: %v", err)
|
|
}
|
|
|
|
sessStart := &messages.IOSSessionStart{
|
|
Timestamp: req.Timestamp,
|
|
ProjectID: uint64(p.ProjectID),
|
|
TrackerVersion: req.TrackerVersion,
|
|
RevID: req.RevID,
|
|
UserUUID: userUUID,
|
|
UserOS: "IOS",
|
|
UserOSVersion: req.UserOSVersion,
|
|
UserDevice: ios.MapIOSDevice(req.UserDevice),
|
|
UserDeviceType: ios.GetIOSDeviceType(req.UserDevice),
|
|
UserCountry: geoInfo.Pack(),
|
|
}
|
|
log.Printf("mobile session start: %+v", sessStart)
|
|
|
|
if err := e.services.Producer.Produce(e.cfg.TopicRawIOS, tokenData.ID, sessStart.Encode()); err != nil {
|
|
log.Printf("failed to produce mobile session start message: %v", err)
|
|
}
|
|
}
|
|
|
|
ResponseWithJSON(w, &StartIOSSessionResponse{
|
|
Token: e.services.Tokenizer.Compose(*tokenData),
|
|
UserUUID: userUUID,
|
|
SessionID: strconv.FormatUint(tokenData.ID, 10),
|
|
BeaconSizeLimit: e.cfg.BeaconSizeLimit,
|
|
ImageQuality: "standard", // Pull from project settings (low, standard, high)
|
|
FrameRate: 3, // Pull from project settings
|
|
}, startTime, r.URL.Path, 0)
|
|
}
|
|
|
|
func (e *Router) pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) {
|
|
startTime := time.Now()
|
|
sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r)
|
|
if err != nil {
|
|
ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
e.pushMessages(w, r, sessionData.ID, e.cfg.TopicRawIOS)
|
|
}
|
|
|
|
func (e *Router) pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) {
|
|
startTime := time.Now()
|
|
sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r)
|
|
if err != nil && err != token.EXPIRED {
|
|
ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
// Check timestamps here?
|
|
e.pushMessages(w, r, sessionData.ID, e.cfg.TopicRawIOS)
|
|
}
|
|
|
|
func (e *Router) imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) {
|
|
startTime := time.Now()
|
|
log.Printf("recieved imagerequest")
|
|
|
|
sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r)
|
|
if err != nil { // Should accept expired token?
|
|
ResponseWithError(w, http.StatusUnauthorized, err, startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
|
|
if r.Body == nil {
|
|
ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty"), startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
r.Body = http.MaxBytesReader(w, r.Body, e.cfg.FileSizeLimit)
|
|
defer r.Body.Close()
|
|
|
|
err = r.ParseMultipartForm(5 * 1e6) // ~5Mb
|
|
if err == http.ErrNotMultipart || err == http.ErrMissingBoundary {
|
|
ResponseWithError(w, http.StatusUnsupportedMediaType, err, startTime, r.URL.Path, 0)
|
|
return
|
|
// } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB
|
|
} else if err != nil {
|
|
ResponseWithError(w, http.StatusInternalServerError, err, startTime, r.URL.Path, 0) // TODO: send error here only on staging
|
|
return
|
|
}
|
|
|
|
if r.MultipartForm == nil {
|
|
ResponseWithError(w, http.StatusInternalServerError, errors.New("Multipart not parsed"), startTime, r.URL.Path, 0)
|
|
return
|
|
}
|
|
|
|
if len(r.MultipartForm.Value["projectKey"]) == 0 {
|
|
ResponseWithError(w, http.StatusBadRequest, errors.New("projectKey parameter missing"), startTime, r.URL.Path, 0) // status for missing/wrong parameter?
|
|
return
|
|
}
|
|
|
|
//prefix := r.MultipartForm.Value["projectKey"][0] + "/" + strconv.FormatUint(sessionData.ID, 10) + "/"
|
|
|
|
for _, fileHeaderList := range r.MultipartForm.File {
|
|
for _, fileHeader := range fileHeaderList {
|
|
file, err := fileHeader.Open()
|
|
if err != nil {
|
|
continue // TODO: send server error or accumulate successful files
|
|
}
|
|
//key := prefix + fileHeader.Filename
|
|
|
|
data, err := ioutil.ReadAll(file)
|
|
if err != nil {
|
|
log.Fatalf("failed reading data: %s", err)
|
|
}
|
|
|
|
log.Printf("Uploading image... %v, len: %d", util.SafeString(fileHeader.Filename), len(data))
|
|
|
|
if err := e.services.Producer.Produce(e.cfg.TopicRawImages, sessionData.ID, data); err != nil {
|
|
log.Printf("failed to produce mobile session start message: %v", err)
|
|
}
|
|
log.Printf("Image uploaded")
|
|
//go func() { //TODO: mime type from header
|
|
// log.Printf("Uploading image... %v", file)
|
|
// //if err := e.services.Storage.Upload(file, key, "image/jpeg", false); err != nil {
|
|
// // log.Printf("Upload ios screen error. %v", err)
|
|
// //}
|
|
//}()
|
|
}
|
|
}
|
|
|
|
w.WriteHeader(http.StatusOK)
|
|
}
|