From dc69131499bd4187d9a8a31dbadcc2d5733cf24e Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 29 Apr 2022 11:22:00 +0200 Subject: [PATCH 01/18] Deleted commented (unused) code --- .../pkg/messages/legacy-message-transform.go | 1 - backend/services/http/handlers-depricated.go | 1 - backend/services/http/handlers-ios.go | 26 ++------------ backend/services/http/handlers-web.go | 36 ------------------- 4 files changed, 3 insertions(+), 61 deletions(-) delete mode 100644 backend/services/http/handlers-depricated.go diff --git a/backend/pkg/messages/legacy-message-transform.go b/backend/pkg/messages/legacy-message-transform.go index 031c4444a..c22b73378 100644 --- a/backend/pkg/messages/legacy-message-transform.go +++ b/backend/pkg/messages/legacy-message-transform.go @@ -10,7 +10,6 @@ func transformDepricated(msg Message) Message { ID: m.ID, HesitationTime: m.HesitationTime, Label: m.Label, - // Selector: '', } // case *FetchDepricated: // return &Fetch { diff --git a/backend/services/http/handlers-depricated.go b/backend/services/http/handlers-depricated.go deleted file mode 100644 index 06ab7d0f9..000000000 --- a/backend/services/http/handlers-depricated.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/backend/services/http/handlers-ios.go b/backend/services/http/handlers-ios.go index 8116980e1..a7b6f984a 100644 --- a/backend/services/http/handlers-ios.go +++ b/backend/services/http/handlers-ios.go @@ -23,22 +23,9 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { TrackerVersion string `json:"trackerVersion"` RevID string `json:"revID"` UserUUID *string `json:"userUUID"` - //UserOS string `json"userOS"` //hardcoded 'MacOS' - UserOSVersion string `json:"userOSVersion"` - UserDevice string `json:"userDevice"` - Timestamp uint64 `json:"timestamp"` - // UserDeviceType uint 0:phone 1:pad 2:tv 3:carPlay 5:mac - // “performances”:{ - // “activeProcessorCount”:8, - // “isLowPowerModeEnabled”:0, - // “orientation”:0, - // “systemUptime”:585430, - // “batteryState”:0, - // “thermalState”:0, - // “batteryLevel”:0, - // “processorCount”:8, - // “physicalMemory”:17179869184 - // }, + UserOSVersion string `json:"userOSVersion"` + UserDevice string `json:"userDevice"` + Timestamp uint64 `json:"timestamp"` } type response struct { Token string `json:"token"` @@ -111,14 +98,7 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { })) } - // imagesHashList, err := s3.GetFrequentlyUsedKeys(*(req.EncodedProjectID)) // TODO: reuse index: ~ frequency * size - // if err != nil { - // responseWithError(w, http.StatusInternalServerError, err) - // return - // } - responseWithJSON(w, &response{ - // ImagesHashList: imagesHashList, Token: tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), diff --git a/backend/services/http/handlers-web.go b/backend/services/http/handlers-web.go index 7aab5bfbc..e5cf57a60 100644 --- a/backend/services/http/handlers-web.go +++ b/backend/services/http/handlers-web.go @@ -105,10 +105,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { })) } - //delayDuration := time.Now().Sub(startTime) responseWithJSON(w, &response{ - //Timestamp: startTime.UnixNano() / 1e6, - //Delay: delayDuration.Nanoseconds() / 1e6, Token: tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), @@ -154,35 +151,6 @@ func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { } } - // switch msg.(type) { - // case *BatchMeta, // TODO: watchout! Meta().Index'es are changed here (though it is still unique for the topic-session pair) - // *SetPageLocation, - // *PageLoadTiming, - // *PageRenderTiming, - // *PerformanceTrack, - // *SetInputTarget, - // *SetInputValue, - // *MouseClick, - // *RawErrorEvent, - // *JSException, - // *ResourceTiming, - // *RawCustomEvent, - // *CustomIssue, - // *Fetch, - // *StateAction, - // *GraphQL, - // *CreateElementNode, - // *CreateTextNode, - // *RemoveNode, - // *CreateDocument, - // *RemoveNodeAttribute, - // *MoveNode, - // *SetCSSData, - // *CSSInsertRule, - // *CSSDeleteRule: - // analyticsMessages = append(analyticsMessages, msg) - //} - return msg }) if err != nil { @@ -190,9 +158,6 @@ func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { return } producer.Produce(TOPIC_RAW_WEB, sessionData.ID, rewritenBuf) - //producer.Produce(TOPIC_ANALYTICS, sessionData.ID, WriteBatch(analyticsMessages)) - //duration := time.Now().Sub(startTime) - //log.Printf("Sended batch within %v nsec; %v nsek/byte", duration.Nanoseconds(), duration.Nanoseconds()/int64(len(buf))) w.WriteHeader(http.StatusOK) } @@ -201,7 +166,6 @@ func notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { ProjectKey *string `json:"projectKey"` TrackerVersion string `json:"trackerVersion"` DoNotTrack bool `json:"DoNotTrack"` - // RevID string `json:"revID"` } req := &request{} body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) From 10edeb6e2dd1f2bd2898323d372af7c65ee62336 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 29 Apr 2022 16:53:28 +0200 Subject: [PATCH 02/18] Refactoring of http handlers --- backend/pkg/messages/batch.go | 39 +--------- .../pkg/messages/legacy-message-transform.go | 13 +--- backend/pkg/queue/messages.go | 3 +- backend/services/http/handlers-web.go | 78 ++++++++++--------- backend/services/http/main.go | 24 ++++-- backend/services/http/model.go | 30 +++++++ 6 files changed, 94 insertions(+), 93 deletions(-) create mode 100644 backend/services/http/model.go diff --git a/backend/pkg/messages/batch.go b/backend/pkg/messages/batch.go index 9241672a3..850f22de9 100644 --- a/backend/pkg/messages/batch.go +++ b/backend/pkg/messages/batch.go @@ -1,17 +1,12 @@ package messages import ( - "bytes" "io" "github.com/pkg/errors" ) -func ReadBatch(b []byte, callback func(Message)) error { - return ReadBatchReader(bytes.NewReader(b), callback) -} - -func ReadBatchReader(reader io.Reader, callback func(Message)) error { +func ReadBatchReader(reader io.Reader, messageHandler func(Message)) error { var index uint64 var timestamp int64 for { @@ -21,7 +16,7 @@ func ReadBatchReader(reader io.Reader, callback func(Message)) error { } else if err != nil { return errors.Wrapf(err, "Batch Message decoding error on message with index %v", index) } - msg = transformDepricated(msg) + msg = transformDeprecated(msg) isBatchMeta := false switch m := msg.(type) { @@ -48,37 +43,11 @@ func ReadBatchReader(reader io.Reader, callback func(Message)) error { } msg.Meta().Index = index msg.Meta().Timestamp = timestamp - callback(msg) + + messageHandler(msg) if !isBatchMeta { // Without that indexes will be unique anyway, though shifted by 1 because BatchMeta is not counted in tracker index++ } } return errors.New("Error of the codeflow. (Should return on EOF)") } - -const AVG_MESSAGE_SIZE = 40 // TODO: calculate OR calculate dynamically -func WriteBatch(mList []Message) []byte { - batch := make([]byte, AVG_MESSAGE_SIZE*len(mList)) - p := 0 - for _, msg := range mList { - msgBytes := msg.Encode() - if len(batch) < p+len(msgBytes) { - newBatch := make([]byte, 2*len(batch)+len(msgBytes)) - copy(newBatch, batch) - batch = newBatch - } - copy(batch[p:], msgBytes) - p += len(msgBytes) - } - return batch[:p] -} - -func RewriteBatch(reader io.Reader, rewrite func(Message) Message) ([]byte, error) { - mList := make([]Message, 0, 10) // 10? - if err := ReadBatchReader(reader, func(m Message) { - mList = append(mList, rewrite(m)) - }); err != nil { - return nil, err - } - return WriteBatch(mList), nil -} diff --git a/backend/pkg/messages/legacy-message-transform.go b/backend/pkg/messages/legacy-message-transform.go index c22b73378..6774d95b1 100644 --- a/backend/pkg/messages/legacy-message-transform.go +++ b/backend/pkg/messages/legacy-message-transform.go @@ -1,6 +1,6 @@ package messages -func transformDepricated(msg Message) Message { +func transformDeprecated(msg Message) Message { switch m := msg.(type) { case *MouseClickDepricated: meta := m.Meta() @@ -11,17 +11,6 @@ func transformDepricated(msg Message) Message { HesitationTime: m.HesitationTime, Label: m.Label, } - // case *FetchDepricated: - // return &Fetch { - // Method: m.Method, - // URL: m.URL, - // Request: m.Request, - // Response: m.Response, - // Status: m.Status, - // Timestamp: m.Timestamp, - // Duration: m.Duration, - // // Headers: '' - // } default: return msg } diff --git a/backend/pkg/queue/messages.go b/backend/pkg/queue/messages.go index 0ab184ee6..9b9ff43a5 100644 --- a/backend/pkg/queue/messages.go +++ b/backend/pkg/queue/messages.go @@ -1,6 +1,7 @@ package queue import ( + "bytes" "log" "openreplay/backend/pkg/messages" @@ -9,7 +10,7 @@ import ( func NewMessageConsumer(group string, topics []string, handler types.DecodedMessageHandler, autoCommit bool) types.Consumer { return NewConsumer(group, topics, func(sessionID uint64, value []byte, meta *types.Meta) { - if err := messages.ReadBatch(value, func(msg messages.Message) { + if err := messages.ReadBatchReader(bytes.NewReader(value), func(msg messages.Message) { handler(sessionID, msg, meta) }); err != nil { log.Printf("Decode error: %v\n", err) diff --git a/backend/services/http/handlers-web.go b/backend/services/http/handlers-web.go index e5cf57a60..0c2fa386c 100644 --- a/backend/services/http/handlers-web.go +++ b/backend/services/http/handlers-web.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/json" "errors" "log" @@ -15,37 +16,23 @@ import ( ) func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { - type request struct { - Token string `json:"token"` - UserUUID *string `json:"userUUID"` - RevID string `json:"revID"` - Timestamp uint64 `json:"timestamp"` - TrackerVersion string `json:"trackerVersion"` - IsSnippet bool `json:"isSnippet"` - DeviceMemory uint64 `json:"deviceMemory"` - JsHeapSizeLimit uint64 `json:"jsHeapSizeLimit"` - ProjectKey *string `json:"projectKey"` - Reset bool `json:"reset"` - UserID string `json:"userID"` - } - type response struct { - Timestamp int64 `json:"timestamp"` - Delay int64 `json:"delay"` - Token string `json:"token"` - UserUUID string `json:"userUUID"` - SessionID string `json:"sessionID"` - BeaconSizeLimit int64 `json:"beaconSizeLimit"` - } - startTime := time.Now() - req := &request{} - body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) // what if Body == nil?? // use r.ContentLength to return specific error? + + // Check request body + if r.Body == nil { + responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + } + body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) defer body.Close() + + // Parse request body + req := &startSessionRequest{} if err := json.NewDecoder(body).Decode(req); err != nil { responseWithError(w, http.StatusBadRequest, err) return } + // Handler's logic if req.ProjectKey == nil { responseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) return @@ -82,9 +69,8 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { } // TODO: if EXPIRED => send message for two sessions association expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) - tokenData = &token.TokenData{sessionID, expTime.UnixNano() / 1e6} + tokenData = &token.TokenData{ID: sessionID, ExpTime: expTime.UnixNano() / 1e6} - country := geoIP.ExtractISOCodeFromHTTPRequest(r) producer.Produce(TOPIC_RAW_WEB, tokenData.ID, Encode(&SessionStart{ Timestamp: req.Timestamp, ProjectID: uint64(p.ProjectID), @@ -98,14 +84,14 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { UserBrowserVersion: ua.BrowserVersion, UserDevice: ua.Device, UserDeviceType: ua.DeviceType, - UserCountry: country, + UserCountry: geoIP.ExtractISOCodeFromHTTPRequest(r), UserDeviceMemorySize: req.DeviceMemory, UserDeviceHeapSize: req.JsHeapSizeLimit, UserID: req.UserID, })) } - responseWithJSON(w, &response{ + responseWithJSON(w, &startSessionResponse{ Token: tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), @@ -114,15 +100,24 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { } func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { + // Check authorization sessionData, err := tokenizer.ParseFromHTTPRequest(r) if err != nil { responseWithError(w, http.StatusUnauthorized, err) return } + + // Check request body + if r.Body == nil { + responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + } body := http.MaxBytesReader(w, r.Body, BEACON_SIZE_LIMIT) defer body.Close() - rewritenBuf, err := RewriteBatch(body, func(msg Message) Message { + var handledMessages bytes.Buffer + + // Process each message in request data + err = ReadBatchReader(body, func(msg Message) { switch m := msg.(type) { case *SetNodeAttributeURLBased: if m.Name == "src" || m.Name == "href" { @@ -150,30 +145,38 @@ func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { Rule: handleCSS(sessionData.ID, m.BaseURL, m.Rule), } } - - return msg + handledMessages.Write(msg.Encode()) }) if err != nil { responseWithError(w, http.StatusForbidden, err) return } - producer.Produce(TOPIC_RAW_WEB, sessionData.ID, rewritenBuf) + + // Send processed messages to queue as array of bytes + err = producer.Produce(TOPIC_RAW_WEB, sessionData.ID, handledMessages.Bytes()) + if err != nil { + log.Printf("can't send processed messages to queue: %s", err) + } + w.WriteHeader(http.StatusOK) } func notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { - type request struct { - ProjectKey *string `json:"projectKey"` - TrackerVersion string `json:"trackerVersion"` - DoNotTrack bool `json:"DoNotTrack"` + // Check request body + if r.Body == nil { + responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } - req := &request{} body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) defer body.Close() + + // Parse request body + req := ¬StartedRequest{} if err := json.NewDecoder(body).Decode(req); err != nil { responseWithError(w, http.StatusBadRequest, err) return } + + // Handler's logic if req.ProjectKey == nil { responseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) return @@ -201,5 +204,6 @@ func notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { if err != nil { log.Printf("Unable to insert Unstarted Session: %v\n", err) } + w.WriteHeader(http.StatusOK) } diff --git a/backend/services/http/main.go b/backend/services/http/main.go index 1f3bc93b3..43dd6865e 100644 --- a/backend/services/http/main.go +++ b/backend/services/http/main.go @@ -4,6 +4,7 @@ import ( "context" "log" "net/http" + "openreplay/backend/pkg/db/postgres" "os" "os/signal" "syscall" @@ -11,7 +12,6 @@ import ( "golang.org/x/net/http2" "openreplay/backend/pkg/db/cache" - "openreplay/backend/pkg/db/postgres" "openreplay/backend/pkg/env" "openreplay/backend/pkg/flakeid" "openreplay/backend/pkg/queue" @@ -47,26 +47,32 @@ func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) pprof.StartProfilingServer() + // Queue producer = queue.NewProducer() defer producer.Close(15000) + + // Database + pgconn = cache.NewPGCache(postgres.NewConn(env.String("POSTGRES_STRING")), 1000*60*20) + defer pgconn.Close() + + // Envs TOPIC_RAW_WEB = env.String("TOPIC_RAW_WEB") TOPIC_RAW_IOS = env.String("TOPIC_RAW_IOS") TOPIC_CACHE = env.String("TOPIC_CACHE") TOPIC_TRIGGER = env.String("TOPIC_TRIGGER") - //TOPIC_ANALYTICS = env.String("TOPIC_ANALYTICS") + CACHE_ASSESTS = env.Bool("CACHE_ASSETS") + BEACON_SIZE_LIMIT = int64(env.Uint64("BEACON_SIZE_LIMIT")) + HTTP_PORT := env.String("HTTP_PORT") + + // Modules rewriter = assets.NewRewriter(env.String("ASSETS_ORIGIN")) - pgconn = cache.NewPGCache(postgres.NewConn(env.String("POSTGRES_STRING")), 1000*60*20) - defer pgconn.Close() s3 = storage.NewS3(env.String("AWS_REGION"), env.String("S3_BUCKET_IOS_IMAGES")) tokenizer = token.NewTokenizer(env.String("TOKEN_SECRET")) uaParser = uaparser.NewUAParser(env.String("UAPARSER_FILE")) geoIP = geoip.NewGeoIP(env.String("MAXMINDDB_FILE")) flaker = flakeid.NewFlaker(env.WorkerID()) - CACHE_ASSESTS = env.Bool("CACHE_ASSETS") - BEACON_SIZE_LIMIT = int64(env.Uint64("BEACON_SIZE_LIMIT")) - - HTTP_PORT := env.String("HTTP_PORT") + // Server server := &http.Server{ Addr: ":" + HTTP_PORT, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -140,6 +146,7 @@ func main() { } }), } + http2.ConfigureServer(server, nil) go func() { if err := server.ListenAndServe(); err != nil { @@ -148,6 +155,7 @@ func main() { } }() log.Printf("Server successfully started on port %v\n", HTTP_PORT) + sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) <-sigchan diff --git a/backend/services/http/model.go b/backend/services/http/model.go new file mode 100644 index 000000000..5a7dd28bc --- /dev/null +++ b/backend/services/http/model.go @@ -0,0 +1,30 @@ +package main + +type startSessionRequest struct { + Token string `json:"token"` + UserUUID *string `json:"userUUID"` + RevID string `json:"revID"` + Timestamp uint64 `json:"timestamp"` + TrackerVersion string `json:"trackerVersion"` + IsSnippet bool `json:"isSnippet"` + DeviceMemory uint64 `json:"deviceMemory"` + JsHeapSizeLimit uint64 `json:"jsHeapSizeLimit"` + ProjectKey *string `json:"projectKey"` + Reset bool `json:"reset"` + UserID string `json:"userID"` +} + +type startSessionResponse struct { + Timestamp int64 `json:"timestamp"` + Delay int64 `json:"delay"` + Token string `json:"token"` + UserUUID string `json:"userUUID"` + SessionID string `json:"sessionID"` + BeaconSizeLimit int64 `json:"beaconSizeLimit"` +} + +type notStartedRequest struct { + ProjectKey *string `json:"projectKey"` + TrackerVersion string `json:"trackerVersion"` + DoNotTrack bool `json:"DoNotTrack"` +} From b0bb5bd922f5acdc8da70c44963c2d2895eea0a1 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 29 Apr 2022 17:23:20 +0200 Subject: [PATCH 03/18] Moved configuration to the separate file --- backend/services/http/assets.go | 6 ++-- backend/services/http/config.go | 39 +++++++++++++++++++++++++ backend/services/http/handlers-ios.go | 10 +++---- backend/services/http/handlers-web.go | 12 ++++---- backend/services/http/handlers.go | 4 +-- backend/services/http/main.go | 41 +++++++++------------------ 6 files changed, 68 insertions(+), 44 deletions(-) create mode 100644 backend/services/http/config.go diff --git a/backend/services/http/assets.go b/backend/services/http/assets.go index b6ac61186..bced76ef0 100644 --- a/backend/services/http/assets.go +++ b/backend/services/http/assets.go @@ -7,7 +7,7 @@ import ( func sendAssetForCache(sessionID uint64, baseURL string, relativeURL string) { if fullURL, cacheable := assets.GetFullCachableURL(baseURL, relativeURL); cacheable { - producer.Produce(TOPIC_CACHE, sessionID, messages.Encode(&messages.AssetCache{ + producer.Produce(cfg.TopicCache, sessionID, messages.Encode(&messages.AssetCache{ URL: fullURL, })) } @@ -20,7 +20,7 @@ func sendAssetsForCacheFromCSS(sessionID uint64, baseURL string, css string) { } func handleURL(sessionID uint64, baseURL string, url string) string { - if CACHE_ASSESTS { + if cfg.CacheAssets { sendAssetForCache(sessionID, baseURL, url) return rewriter.RewriteURL(sessionID, baseURL, url) } @@ -28,7 +28,7 @@ func handleURL(sessionID uint64, baseURL string, url string) string { } func handleCSS(sessionID uint64, baseURL string, css string) string { - if CACHE_ASSESTS { + if cfg.CacheAssets { sendAssetsForCacheFromCSS(sessionID, baseURL, css) return rewriter.RewriteCSS(sessionID, baseURL, css) } diff --git a/backend/services/http/config.go b/backend/services/http/config.go new file mode 100644 index 000000000..f03ef8cc4 --- /dev/null +++ b/backend/services/http/config.go @@ -0,0 +1,39 @@ +package main + +import "openreplay/backend/pkg/env" + +type config struct { + HTTPPort string + TopicRawWeb string + TopicRawIOS string + TopicCache string + CacheAssets bool + BeaconSizeLimit int64 + JsonSizeLimit int64 + AssetsOrigin string + AWSRegion string + S3BucketIOSImages string + TokenSecret string + UAParserFile string + MaxMinDBFile string + WorkerID uint16 +} + +func NewConfig() *config { + return &config{ + HTTPPort: env.String("HTTP_PORT"), + TopicRawWeb: env.String("TOPIC_RAW_WEB"), + TopicRawIOS: env.String("TOPIC_RAW_IOS"), + TopicCache: env.String("TOPIC_CACHE"), + CacheAssets: env.Bool("CACHE_ASSETS"), + BeaconSizeLimit: int64(env.Uint64("BEACON_SIZE_LIMIT")), + JsonSizeLimit: 1e3, // 1Kb + AssetsOrigin: env.String("ASSETS_ORIGIN"), + AWSRegion: env.String("AWS_REGION"), + S3BucketIOSImages: env.String("S3_BUCKET_IOS_IMAGES"), + TokenSecret: env.String("TOKEN_SECRET"), + UAParserFile: env.String("UAPARSER_FILE"), + MaxMinDBFile: env.String("MAXMINDDB_FILE"), + WorkerID: env.WorkerID(), + } +} diff --git a/backend/services/http/handlers-ios.go b/backend/services/http/handlers-ios.go index a7b6f984a..6dae4e70c 100644 --- a/backend/services/http/handlers-ios.go +++ b/backend/services/http/handlers-ios.go @@ -36,7 +36,7 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { } startTime := time.Now() req := &request{} - body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) + body := http.MaxBytesReader(w, r.Body, cfg.JsonSizeLimit) defer body.Close() if err := json.NewDecoder(body).Decode(req); err != nil { responseWithError(w, http.StatusBadRequest, err) @@ -84,7 +84,7 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { country := geoIP.ExtractISOCodeFromHTTPRequest(r) // The difference with web is mostly here: - producer.Produce(TOPIC_RAW_IOS, tokenData.ID, Encode(&IOSSessionStart{ + producer.Produce(cfg.TopicRawIOS, tokenData.ID, Encode(&IOSSessionStart{ Timestamp: req.Timestamp, ProjectID: uint64(p.ProjectID), TrackerVersion: req.TrackerVersion, @@ -102,7 +102,7 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { Token: tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), - BeaconSizeLimit: BEACON_SIZE_LIMIT, + BeaconSizeLimit: cfg.BeaconSizeLimit, }) } @@ -112,7 +112,7 @@ func pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { responseWithError(w, http.StatusUnauthorized, err) return } - pushMessages(w, r, sessionData.ID, TOPIC_RAW_IOS) + pushMessages(w, r, sessionData.ID, cfg.TopicRawIOS) } func pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { @@ -122,7 +122,7 @@ func pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { return } // Check timestamps here? - pushMessages(w, r, sessionData.ID, TOPIC_RAW_IOS) + pushMessages(w, r, sessionData.ID, cfg.TopicRawIOS) } func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { diff --git a/backend/services/http/handlers-web.go b/backend/services/http/handlers-web.go index 0c2fa386c..1b212e2ba 100644 --- a/backend/services/http/handlers-web.go +++ b/backend/services/http/handlers-web.go @@ -22,7 +22,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { if r.Body == nil { responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } - body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) + body := http.MaxBytesReader(w, r.Body, cfg.JsonSizeLimit) defer body.Close() // Parse request body @@ -71,7 +71,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) tokenData = &token.TokenData{ID: sessionID, ExpTime: expTime.UnixNano() / 1e6} - producer.Produce(TOPIC_RAW_WEB, tokenData.ID, Encode(&SessionStart{ + producer.Produce(cfg.TopicRawWeb, tokenData.ID, Encode(&SessionStart{ Timestamp: req.Timestamp, ProjectID: uint64(p.ProjectID), TrackerVersion: req.TrackerVersion, @@ -95,7 +95,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { Token: tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), - BeaconSizeLimit: BEACON_SIZE_LIMIT, + BeaconSizeLimit: cfg.BeaconSizeLimit, }) } @@ -111,7 +111,7 @@ func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { if r.Body == nil { responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } - body := http.MaxBytesReader(w, r.Body, BEACON_SIZE_LIMIT) + body := http.MaxBytesReader(w, r.Body, cfg.BeaconSizeLimit) defer body.Close() var handledMessages bytes.Buffer @@ -153,7 +153,7 @@ func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { } // Send processed messages to queue as array of bytes - err = producer.Produce(TOPIC_RAW_WEB, sessionData.ID, handledMessages.Bytes()) + err = producer.Produce(cfg.TopicRawWeb, sessionData.ID, handledMessages.Bytes()) if err != nil { log.Printf("can't send processed messages to queue: %s", err) } @@ -166,7 +166,7 @@ func notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { if r.Body == nil { responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } - body := http.MaxBytesReader(w, r.Body, JSON_SIZE_LIMIT) + body := http.MaxBytesReader(w, r.Body, cfg.JsonSizeLimit) defer body.Close() // Parse request body diff --git a/backend/services/http/handlers.go b/backend/services/http/handlers.go index dd73925af..d2d780f30 100644 --- a/backend/services/http/handlers.go +++ b/backend/services/http/handlers.go @@ -9,10 +9,8 @@ import ( gzip "github.com/klauspost/pgzip" ) -const JSON_SIZE_LIMIT int64 = 1e3 // 1Kb - func pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64, topicName string) { - body := http.MaxBytesReader(w, r.Body, BEACON_SIZE_LIMIT) + body := http.MaxBytesReader(w, r.Body, cfg.BeaconSizeLimit) defer body.Close() var reader io.ReadCloser var err error diff --git a/backend/services/http/main.go b/backend/services/http/main.go index 43dd6865e..d053c9c2f 100644 --- a/backend/services/http/main.go +++ b/backend/services/http/main.go @@ -25,6 +25,8 @@ import ( "openreplay/backend/pkg/pprof" ) +// Global variables +var cfg *config var rewriter *assets.Rewriter var producer types.Producer var pgconn *cache.PGCache @@ -34,19 +36,13 @@ var geoIP *geoip.GeoIP var tokenizer *token.Tokenizer var s3 *storage.S3 -var TOPIC_RAW_WEB string -var TOPIC_RAW_IOS string -var TOPIC_CACHE string -var TOPIC_TRIGGER string - -//var TOPIC_ANALYTICS string -var CACHE_ASSESTS bool -var BEACON_SIZE_LIMIT int64 - func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) pprof.StartProfilingServer() + // Configs + cfg = NewConfig() + // Queue producer = queue.NewProducer() defer producer.Close(15000) @@ -55,26 +51,17 @@ func main() { pgconn = cache.NewPGCache(postgres.NewConn(env.String("POSTGRES_STRING")), 1000*60*20) defer pgconn.Close() - // Envs - TOPIC_RAW_WEB = env.String("TOPIC_RAW_WEB") - TOPIC_RAW_IOS = env.String("TOPIC_RAW_IOS") - TOPIC_CACHE = env.String("TOPIC_CACHE") - TOPIC_TRIGGER = env.String("TOPIC_TRIGGER") - CACHE_ASSESTS = env.Bool("CACHE_ASSETS") - BEACON_SIZE_LIMIT = int64(env.Uint64("BEACON_SIZE_LIMIT")) - HTTP_PORT := env.String("HTTP_PORT") - - // Modules - rewriter = assets.NewRewriter(env.String("ASSETS_ORIGIN")) - s3 = storage.NewS3(env.String("AWS_REGION"), env.String("S3_BUCKET_IOS_IMAGES")) - tokenizer = token.NewTokenizer(env.String("TOKEN_SECRET")) - uaParser = uaparser.NewUAParser(env.String("UAPARSER_FILE")) - geoIP = geoip.NewGeoIP(env.String("MAXMINDDB_FILE")) - flaker = flakeid.NewFlaker(env.WorkerID()) + // Init modules + rewriter = assets.NewRewriter(cfg.AssetsOrigin) + s3 = storage.NewS3(cfg.AWSRegion, cfg.S3BucketIOSImages) + tokenizer = token.NewTokenizer(cfg.TokenSecret) + uaParser = uaparser.NewUAParser(cfg.UAParserFile) + geoIP = geoip.NewGeoIP(cfg.MaxMinDBFile) + flaker = flakeid.NewFlaker(cfg.WorkerID) // Server server := &http.Server{ - Addr: ":" + HTTP_PORT, + Addr: ":" + cfg.HTTPPort, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // TODO: agree with specification @@ -154,7 +141,7 @@ func main() { log.Fatal("Server error") } }() - log.Printf("Server successfully started on port %v\n", HTTP_PORT) + log.Printf("Server successfully started on port %v\n", cfg.HTTPPort) sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) From 66e190221d4f4bbf04d84b19b26f76bb3487d456 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 2 May 2022 14:36:02 +0200 Subject: [PATCH 04/18] Removed global objects (moved service initialization into serviceBuilder) --- backend/services/http/assets.go | 24 ++--- backend/services/http/config.go | 11 +- backend/services/http/handlers-ios.go | 38 +++---- backend/services/http/handlers-web.go | 46 ++++---- backend/services/http/handlers.go | 6 +- backend/services/http/main.go | 144 +++++--------------------- backend/services/http/router.go | 68 ++++++++++++ backend/services/http/server.go | 46 ++++++++ backend/services/http/service.go | 36 +++++++ 9 files changed, 243 insertions(+), 176 deletions(-) create mode 100644 backend/services/http/router.go create mode 100644 backend/services/http/server.go create mode 100644 backend/services/http/service.go diff --git a/backend/services/http/assets.go b/backend/services/http/assets.go index bced76ef0..0948984b0 100644 --- a/backend/services/http/assets.go +++ b/backend/services/http/assets.go @@ -5,32 +5,32 @@ import ( "openreplay/backend/pkg/url/assets" ) -func sendAssetForCache(sessionID uint64, baseURL string, relativeURL string) { +func (e *Router) sendAssetForCache(sessionID uint64, baseURL string, relativeURL string) { if fullURL, cacheable := assets.GetFullCachableURL(baseURL, relativeURL); cacheable { - producer.Produce(cfg.TopicCache, sessionID, messages.Encode(&messages.AssetCache{ + e.services.producer.Produce(e.cfg.TopicCache, sessionID, messages.Encode(&messages.AssetCache{ URL: fullURL, })) } } -func sendAssetsForCacheFromCSS(sessionID uint64, baseURL string, css string) { +func (e *Router) sendAssetsForCacheFromCSS(sessionID uint64, baseURL string, css string) { for _, u := range assets.ExtractURLsFromCSS(css) { // TODO: in one shot with rewriting - sendAssetForCache(sessionID, baseURL, u) + e.sendAssetForCache(sessionID, baseURL, u) } } -func handleURL(sessionID uint64, baseURL string, url string) string { - if cfg.CacheAssets { - sendAssetForCache(sessionID, baseURL, url) - return rewriter.RewriteURL(sessionID, baseURL, url) +func (e *Router) handleURL(sessionID uint64, baseURL string, url string) string { + if e.cfg.CacheAssets { + e.sendAssetForCache(sessionID, baseURL, url) + return e.services.rewriter.RewriteURL(sessionID, baseURL, url) } return assets.ResolveURL(baseURL, url) } -func handleCSS(sessionID uint64, baseURL string, css string) string { - if cfg.CacheAssets { - sendAssetsForCacheFromCSS(sessionID, baseURL, css) - return rewriter.RewriteCSS(sessionID, baseURL, css) +func (e *Router) handleCSS(sessionID uint64, baseURL string, css string) string { + if e.cfg.CacheAssets { + e.sendAssetsForCacheFromCSS(sessionID, baseURL, css) + return e.services.rewriter.RewriteCSS(sessionID, baseURL, css) } return assets.ResolveCSS(baseURL, css) } diff --git a/backend/services/http/config.go b/backend/services/http/config.go index f03ef8cc4..4b29afb9d 100644 --- a/backend/services/http/config.go +++ b/backend/services/http/config.go @@ -1,9 +1,14 @@ package main -import "openreplay/backend/pkg/env" +import ( + "openreplay/backend/pkg/env" + "time" +) type config struct { + HTTPHost string HTTPPort string + HTTPTimeout time.Duration TopicRawWeb string TopicRawIOS string TopicCache string @@ -13,6 +18,7 @@ type config struct { AssetsOrigin string AWSRegion string S3BucketIOSImages string + Postgres string TokenSecret string UAParserFile string MaxMinDBFile string @@ -21,7 +27,9 @@ type config struct { func NewConfig() *config { return &config{ + HTTPHost: "", HTTPPort: env.String("HTTP_PORT"), + HTTPTimeout: time.Second * 60, TopicRawWeb: env.String("TOPIC_RAW_WEB"), TopicRawIOS: env.String("TOPIC_RAW_IOS"), TopicCache: env.String("TOPIC_CACHE"), @@ -31,6 +39,7 @@ func NewConfig() *config { AssetsOrigin: env.String("ASSETS_ORIGIN"), AWSRegion: env.String("AWS_REGION"), S3BucketIOSImages: env.String("S3_BUCKET_IOS_IMAGES"), + Postgres: env.String("POSTGRES_STRING"), TokenSecret: env.String("TOKEN_SECRET"), UAParserFile: env.String("UAPARSER_FILE"), MaxMinDBFile: env.String("MAXMINDDB_FILE"), diff --git a/backend/services/http/handlers-ios.go b/backend/services/http/handlers-ios.go index 6dae4e70c..4591a7d72 100644 --- a/backend/services/http/handlers-ios.go +++ b/backend/services/http/handlers-ios.go @@ -16,7 +16,7 @@ import ( const FILES_SIZE_LIMIT int64 = 1e7 // 10Mb -func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { +func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { type request struct { Token string `json:"token"` ProjectKey *string `json:"projectKey"` @@ -36,7 +36,7 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { } startTime := time.Now() req := &request{} - body := http.MaxBytesReader(w, r.Body, cfg.JsonSizeLimit) + body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() if err := json.NewDecoder(body).Decode(req); err != nil { responseWithError(w, http.StatusBadRequest, err) @@ -48,7 +48,7 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { return } - p, err := pgconn.GetProjectByKey(*req.ProjectKey) + p, err := e.services.pgconn.GetProjectByKey(*req.ProjectKey) if err != nil { if postgres.IsNoRowsErr(err) { responseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or is not active")) @@ -58,7 +58,7 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { return } userUUID := getUUID(req.UserUUID) - tokenData, err := tokenizer.Parse(req.Token) + tokenData, err := e.services.tokenizer.Parse(req.Token) if err != nil { // Starting the new one dice := byte(rand.Intn(100)) // [0, 100) @@ -67,12 +67,12 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { return } - ua := uaParser.ParseFromHTTPRequest(r) + ua := e.services.uaParser.ParseFromHTTPRequest(r) if ua == nil { responseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } - sessionID, err := flaker.Compose(uint64(startTime.UnixMilli())) + sessionID, err := e.services.flaker.Compose(uint64(startTime.UnixMilli())) if err != nil { responseWithError(w, http.StatusInternalServerError, err) return @@ -81,10 +81,10 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) tokenData = &token.TokenData{sessionID, expTime.UnixMilli()} - country := geoIP.ExtractISOCodeFromHTTPRequest(r) + country := e.services.geoIP.ExtractISOCodeFromHTTPRequest(r) // The difference with web is mostly here: - producer.Produce(cfg.TopicRawIOS, tokenData.ID, Encode(&IOSSessionStart{ + e.services.producer.Produce(e.cfg.TopicRawIOS, tokenData.ID, Encode(&IOSSessionStart{ Timestamp: req.Timestamp, ProjectID: uint64(p.ProjectID), TrackerVersion: req.TrackerVersion, @@ -99,36 +99,36 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { } responseWithJSON(w, &response{ - Token: tokenizer.Compose(*tokenData), + Token: e.services.tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), - BeaconSizeLimit: cfg.BeaconSizeLimit, + BeaconSizeLimit: e.cfg.BeaconSizeLimit, }) } -func pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { - sessionData, err := tokenizer.ParseFromHTTPRequest(r) +func (e *Router) pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { + sessionData, err := e.services.tokenizer.ParseFromHTTPRequest(r) if err != nil { responseWithError(w, http.StatusUnauthorized, err) return } - pushMessages(w, r, sessionData.ID, cfg.TopicRawIOS) + e.pushMessages(w, r, sessionData.ID, e.cfg.TopicRawIOS) } -func pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { - sessionData, err := tokenizer.ParseFromHTTPRequest(r) +func (e *Router) pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { + sessionData, err := e.services.tokenizer.ParseFromHTTPRequest(r) if err != nil && err != token.EXPIRED { responseWithError(w, http.StatusUnauthorized, err) return } // Check timestamps here? - pushMessages(w, r, sessionData.ID, cfg.TopicRawIOS) + e.pushMessages(w, r, sessionData.ID, e.cfg.TopicRawIOS) } -func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { +func (e *Router) imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { log.Printf("recieved imagerequest") - sessionData, err := tokenizer.ParseFromHTTPRequest(r) + sessionData, err := e.services.tokenizer.ParseFromHTTPRequest(r) if err != nil { // Should accept expired token? responseWithError(w, http.StatusUnauthorized, err) return @@ -164,7 +164,7 @@ func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { key := prefix + fileHeader.Filename log.Printf("Uploading image... %v", key) go func() { //TODO: mime type from header - if err := s3.Upload(file, key, "image/jpeg", false); err != nil { + if err := e.services.s3.Upload(file, key, "image/jpeg", false); err != nil { log.Printf("Upload ios screen error. %v", err) } }() diff --git a/backend/services/http/handlers-web.go b/backend/services/http/handlers-web.go index 1b212e2ba..e9fc135b0 100644 --- a/backend/services/http/handlers-web.go +++ b/backend/services/http/handlers-web.go @@ -15,14 +15,14 @@ import ( "openreplay/backend/pkg/token" ) -func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { +func (e *Router) startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { startTime := time.Now() // Check request body if r.Body == nil { responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } - body := http.MaxBytesReader(w, r.Body, cfg.JsonSizeLimit) + body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() // Parse request body @@ -38,7 +38,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { return } - p, err := pgconn.GetProjectByKey(*req.ProjectKey) + p, err := e.services.pgconn.GetProjectByKey(*req.ProjectKey) if err != nil { if postgres.IsNoRowsErr(err) { responseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or capture limit has been reached")) @@ -49,7 +49,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { } userUUID := getUUID(req.UserUUID) - tokenData, err := tokenizer.Parse(req.Token) + tokenData, err := e.services.tokenizer.Parse(req.Token) if err != nil || req.Reset { // Starting the new one dice := byte(rand.Intn(100)) // [0, 100) if dice >= p.SampleRate { @@ -57,12 +57,12 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { return } - ua := uaParser.ParseFromHTTPRequest(r) + ua := e.services.uaParser.ParseFromHTTPRequest(r) if ua == nil { responseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } - sessionID, err := flaker.Compose(uint64(startTime.UnixNano() / 1e6)) + sessionID, err := e.services.flaker.Compose(uint64(startTime.UnixNano() / 1e6)) if err != nil { responseWithError(w, http.StatusInternalServerError, err) return @@ -71,7 +71,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) tokenData = &token.TokenData{ID: sessionID, ExpTime: expTime.UnixNano() / 1e6} - producer.Produce(cfg.TopicRawWeb, tokenData.ID, Encode(&SessionStart{ + e.services.producer.Produce(e.cfg.TopicRawWeb, tokenData.ID, Encode(&SessionStart{ Timestamp: req.Timestamp, ProjectID: uint64(p.ProjectID), TrackerVersion: req.TrackerVersion, @@ -84,7 +84,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { UserBrowserVersion: ua.BrowserVersion, UserDevice: ua.Device, UserDeviceType: ua.DeviceType, - UserCountry: geoIP.ExtractISOCodeFromHTTPRequest(r), + UserCountry: e.services.geoIP.ExtractISOCodeFromHTTPRequest(r), UserDeviceMemorySize: req.DeviceMemory, UserDeviceHeapSize: req.JsHeapSizeLimit, UserID: req.UserID, @@ -92,16 +92,16 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { } responseWithJSON(w, &startSessionResponse{ - Token: tokenizer.Compose(*tokenData), + Token: e.services.tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), - BeaconSizeLimit: cfg.BeaconSizeLimit, + BeaconSizeLimit: e.cfg.BeaconSizeLimit, }) } -func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { +func (e *Router) pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { // Check authorization - sessionData, err := tokenizer.ParseFromHTTPRequest(r) + sessionData, err := e.services.tokenizer.ParseFromHTTPRequest(r) if err != nil { responseWithError(w, http.StatusUnauthorized, err) return @@ -111,7 +111,7 @@ func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { if r.Body == nil { responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } - body := http.MaxBytesReader(w, r.Body, cfg.BeaconSizeLimit) + body := http.MaxBytesReader(w, r.Body, e.cfg.BeaconSizeLimit) defer body.Close() var handledMessages bytes.Buffer @@ -124,25 +124,25 @@ func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { msg = &SetNodeAttribute{ ID: m.ID, Name: m.Name, - Value: handleURL(sessionData.ID, m.BaseURL, m.Value), + Value: e.handleURL(sessionData.ID, m.BaseURL, m.Value), } } else if m.Name == "style" { msg = &SetNodeAttribute{ ID: m.ID, Name: m.Name, - Value: handleCSS(sessionData.ID, m.BaseURL, m.Value), + Value: e.handleCSS(sessionData.ID, m.BaseURL, m.Value), } } case *SetCSSDataURLBased: msg = &SetCSSData{ ID: m.ID, - Data: handleCSS(sessionData.ID, m.BaseURL, m.Data), + Data: e.handleCSS(sessionData.ID, m.BaseURL, m.Data), } case *CSSInsertRuleURLBased: msg = &CSSInsertRule{ ID: m.ID, Index: m.Index, - Rule: handleCSS(sessionData.ID, m.BaseURL, m.Rule), + Rule: e.handleCSS(sessionData.ID, m.BaseURL, m.Rule), } } handledMessages.Write(msg.Encode()) @@ -153,7 +153,7 @@ func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { } // Send processed messages to queue as array of bytes - err = producer.Produce(cfg.TopicRawWeb, sessionData.ID, handledMessages.Bytes()) + err = e.services.producer.Produce(e.cfg.TopicRawWeb, sessionData.ID, handledMessages.Bytes()) if err != nil { log.Printf("can't send processed messages to queue: %s", err) } @@ -161,12 +161,12 @@ func pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -func notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { +func (e *Router) notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { // Check request body if r.Body == nil { responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } - body := http.MaxBytesReader(w, r.Body, cfg.JsonSizeLimit) + body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() // Parse request body @@ -181,13 +181,13 @@ func notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { responseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) return } - ua := uaParser.ParseFromHTTPRequest(r) // TODO?: insert anyway + ua := e.services.uaParser.ParseFromHTTPRequest(r) // TODO?: insert anyway if ua == nil { responseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } - country := geoIP.ExtractISOCodeFromHTTPRequest(r) - err := pgconn.InsertUnstartedSession(postgres.UnstartedSession{ + country := e.services.geoIP.ExtractISOCodeFromHTTPRequest(r) + err := e.services.pgconn.InsertUnstartedSession(postgres.UnstartedSession{ ProjectKey: *req.ProjectKey, TrackerVersion: req.TrackerVersion, DoNotTrack: req.DoNotTrack, diff --git a/backend/services/http/handlers.go b/backend/services/http/handlers.go index d2d780f30..8f8979a4c 100644 --- a/backend/services/http/handlers.go +++ b/backend/services/http/handlers.go @@ -9,8 +9,8 @@ import ( gzip "github.com/klauspost/pgzip" ) -func pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64, topicName string) { - body := http.MaxBytesReader(w, r.Body, cfg.BeaconSizeLimit) +func (e *Router) pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64, topicName string) { + body := http.MaxBytesReader(w, r.Body, e.cfg.BeaconSizeLimit) defer body.Close() var reader io.ReadCloser var err error @@ -34,6 +34,6 @@ func pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64, topi responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging return } - producer.Produce(topicName, sessionID, buf) // What if not able to send? + e.services.producer.Produce(topicName, sessionID, buf) // What if not able to send? w.WriteHeader(http.StatusOK) } diff --git a/backend/services/http/main.go b/backend/services/http/main.go index d053c9c2f..d0e97d137 100644 --- a/backend/services/http/main.go +++ b/backend/services/http/main.go @@ -1,151 +1,59 @@ package main import ( - "context" "log" - "net/http" + "openreplay/backend/pkg/db/cache" "openreplay/backend/pkg/db/postgres" + "openreplay/backend/pkg/pprof" + "openreplay/backend/pkg/queue" "os" "os/signal" "syscall" - - "golang.org/x/net/http2" - - "openreplay/backend/pkg/db/cache" - "openreplay/backend/pkg/env" - "openreplay/backend/pkg/flakeid" - "openreplay/backend/pkg/queue" - "openreplay/backend/pkg/queue/types" - "openreplay/backend/pkg/storage" - "openreplay/backend/pkg/token" - "openreplay/backend/pkg/url/assets" - "openreplay/backend/services/http/geoip" - "openreplay/backend/services/http/uaparser" - - "openreplay/backend/pkg/pprof" ) -// Global variables -var cfg *config -var rewriter *assets.Rewriter -var producer types.Producer -var pgconn *cache.PGCache -var flaker *flakeid.Flaker -var uaParser *uaparser.UAParser -var geoIP *geoip.GeoIP -var tokenizer *token.Tokenizer -var s3 *storage.S3 - func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) pprof.StartProfilingServer() - // Configs - cfg = NewConfig() + // Load configuration + cfg := NewConfig() - // Queue - producer = queue.NewProducer() + // Connect to queue + producer := queue.NewProducer() defer producer.Close(15000) - // Database - pgconn = cache.NewPGCache(postgres.NewConn(env.String("POSTGRES_STRING")), 1000*60*20) - defer pgconn.Close() + // Connect to database + dbConn := cache.NewPGCache(postgres.NewConn(cfg.Postgres), 1000*60*20) + defer dbConn.Close() - // Init modules - rewriter = assets.NewRewriter(cfg.AssetsOrigin) - s3 = storage.NewS3(cfg.AWSRegion, cfg.S3BucketIOSImages) - tokenizer = token.NewTokenizer(cfg.TokenSecret) - uaParser = uaparser.NewUAParser(cfg.UAParserFile) - geoIP = geoip.NewGeoIP(cfg.MaxMinDBFile) - flaker = flakeid.NewFlaker(cfg.WorkerID) + // Build all services + services := NewServiceBuilder(cfg, producer, dbConn) - // Server - server := &http.Server{ - Addr: ":" + cfg.HTTPPort, - Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - - // TODO: agree with specification - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "POST") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type,Authorization") - if r.Method == http.MethodOptions { - w.Header().Set("Cache-Control", "max-age=86400") - w.WriteHeader(http.StatusOK) - return - } - - log.Printf("Request: %v - %v ", r.Method, r.URL.Path) - - switch r.URL.Path { - case "/": - w.WriteHeader(http.StatusOK) - case "/v1/web/not-started": - switch r.Method { - case http.MethodPost: - notStartedHandlerWeb(w, r) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } - case "/v1/web/start": - switch r.Method { - case http.MethodPost: - startSessionHandlerWeb(w, r) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } - case "/v1/web/i": - switch r.Method { - case http.MethodPost: - pushMessagesHandlerWeb(w, r) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } - case "/v1/ios/start": - switch r.Method { - case http.MethodPost: - startSessionHandlerIOS(w, r) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } - case "/v1/ios/i": - switch r.Method { - case http.MethodPost: - pushMessagesHandlerIOS(w, r) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } - case "/v1/ios/late": - switch r.Method { - case http.MethodPost: - pushLateMessagesHandlerIOS(w, r) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } - case "/v1/ios/images": - switch r.Method { - case http.MethodPost: - imagesUploadHandlerIOS(w, r) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } - default: - w.WriteHeader(http.StatusNotFound) - } - }), + // Init server's routes + router, err := NewRouter(cfg, services) + if err != nil { + log.Fatalf("failed while creating engine: %s", err) } - http2.ConfigureServer(server, nil) + // Init server + server, err := NewServer(router.GetHandler(), cfg.HTTPHost, cfg.HTTPPort, cfg.HTTPTimeout) + if err != nil { + log.Fatalf("failed while creating server: %s", err) + } + + // Run server go func() { - if err := server.ListenAndServe(); err != nil { + if err := server.Start(); err != nil { log.Printf("Server error: %v\n", err) log.Fatal("Server error") } }() log.Printf("Server successfully started on port %v\n", cfg.HTTPPort) + // Wait stop signal to shut down server gracefully sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) <-sigchan log.Printf("Shutting down the server\n") - server.Shutdown(context.Background()) + server.Stop() } diff --git a/backend/services/http/router.go b/backend/services/http/router.go new file mode 100644 index 000000000..af624e32c --- /dev/null +++ b/backend/services/http/router.go @@ -0,0 +1,68 @@ +package main + +import ( + "github.com/gorilla/mux" + "log" + "net/http" +) + +type Router struct { + router *mux.Router + cfg *config + services *ServiceBuilder +} + +func NewRouter(cfg *config, services *ServiceBuilder) (*Router, error) { + e := &Router{ + cfg: cfg, + services: services, + } + e.init() + return e, nil +} + +func (e *Router) init() { + e.router = mux.NewRouter() + // Root path + e.router.HandleFunc("/", e.root).Methods("POST") + + // Web handlers + e.router.HandleFunc("/v1/web/not-started", e.notStartedHandlerWeb).Methods("POST") + e.router.HandleFunc("/v1/web/start", e.startSessionHandlerWeb).Methods("POST") + e.router.HandleFunc("/v1/web/i", e.pushMessagesHandlerWeb).Methods("POST") + + // iOS handlers + e.router.HandleFunc("/v1/ios/start", e.startSessionHandlerIOS).Methods("POST") + e.router.HandleFunc("/v1/ios/i", e.pushMessagesHandlerIOS).Methods("POST") + e.router.HandleFunc("/v1/ios/late", e.pushLateMessagesHandlerIOS).Methods("POST") + e.router.HandleFunc("/v1/ios/images", e.imagesUploadHandlerIOS).Methods("POST") + + // CORS middleware + e.router.Use(e.corsMiddleware) +} + +func (e *Router) root(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + +func (e *Router) corsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Prepare headers for preflight requests + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "POST") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type,Authorization") + if r.Method == http.MethodOptions { + w.Header().Set("Cache-Control", "max-age=86400") + w.WriteHeader(http.StatusOK) + return + } + log.Printf("Request: %v - %v ", r.Method, r.URL.Path) + + // Serve request + next.ServeHTTP(w, r) + }) +} + +func (e *Router) GetHandler() http.Handler { + return e.router +} diff --git a/backend/services/http/server.go b/backend/services/http/server.go new file mode 100644 index 000000000..2b6056c20 --- /dev/null +++ b/backend/services/http/server.go @@ -0,0 +1,46 @@ +package main + +import ( + "context" + "errors" + "fmt" + "golang.org/x/net/http2" + "log" + "net/http" + "time" +) + +type Server struct { + server *http.Server +} + +func NewServer(handler http.Handler, host, port string, timeout time.Duration) (*Server, error) { + switch { + case port == "": + return nil, errors.New("empty server port") + case handler == nil: + return nil, errors.New("empty handler") + case timeout < 1: + return nil, fmt.Errorf("invalid timeout %d", timeout) + } + server := &http.Server{ + Addr: fmt.Sprintf("%s:%s", host, port), + Handler: handler, + ReadTimeout: timeout, + WriteTimeout: timeout, + } + if err := http2.ConfigureServer(server, nil); err != nil { + log.Printf("can't configure http2 server: %s", err) + } + return &Server{ + server: server, + }, nil +} + +func (s *Server) Start() error { + return s.server.ListenAndServe() +} + +func (s *Server) Stop() { + s.server.Shutdown(context.Background()) +} diff --git a/backend/services/http/service.go b/backend/services/http/service.go new file mode 100644 index 000000000..d7eda7720 --- /dev/null +++ b/backend/services/http/service.go @@ -0,0 +1,36 @@ +package main + +import ( + "openreplay/backend/pkg/db/cache" + "openreplay/backend/pkg/flakeid" + "openreplay/backend/pkg/queue/types" + "openreplay/backend/pkg/storage" + "openreplay/backend/pkg/token" + "openreplay/backend/pkg/url/assets" + "openreplay/backend/services/http/geoip" + "openreplay/backend/services/http/uaparser" +) + +type ServiceBuilder struct { + pgconn *cache.PGCache + producer types.Producer + rewriter *assets.Rewriter + flaker *flakeid.Flaker + uaParser *uaparser.UAParser + geoIP *geoip.GeoIP + tokenizer *token.Tokenizer + s3 *storage.S3 +} + +func NewServiceBuilder(cfg *config, producer types.Producer, pgconn *cache.PGCache) *ServiceBuilder { + return &ServiceBuilder{ + pgconn: pgconn, + producer: producer, + rewriter: assets.NewRewriter(cfg.AssetsOrigin), + s3: storage.NewS3(cfg.AWSRegion, cfg.S3BucketIOSImages), + tokenizer: token.NewTokenizer(cfg.TokenSecret), + uaParser: uaparser.NewUAParser(cfg.UAParserFile), + geoIP: geoip.NewGeoIP(cfg.MaxMinDBFile), + flaker: flakeid.NewFlaker(cfg.WorkerID), + } +} From f01ef3ea03d0c72cc1ff5398f9703d2d7cab568b Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 2 May 2022 14:47:13 +0200 Subject: [PATCH 05/18] Made a correct project structure for http service --- backend/{services => cmd}/http/README.md | 0 backend/{services => cmd}/http/main.go | 9 +++++---- backend/{services => internal}/http/assets.go | 2 +- backend/{services => internal}/http/config.go | 2 +- backend/{services => internal}/http/geoip/geoip.go | 0 backend/{services => internal}/http/geoip/http.go | 0 backend/{services => internal}/http/handlers-ios.go | 2 +- backend/{services => internal}/http/handlers-web.go | 2 +- backend/{services => internal}/http/handlers.go | 2 +- backend/{services => internal}/http/ios-device.go | 2 +- backend/{services => internal}/http/model.go | 2 +- backend/{services => internal}/http/response.go | 2 +- backend/{services => internal}/http/router.go | 2 +- backend/{services => internal}/http/server.go | 2 +- backend/{services => internal}/http/service.go | 6 +++--- backend/{services => internal}/http/uaparser/http.go | 0 backend/{services => internal}/http/uaparser/uaparser.go | 0 backend/{services => internal}/http/uuid.go | 2 +- 18 files changed, 19 insertions(+), 18 deletions(-) rename backend/{services => cmd}/http/README.md (100%) rename backend/{services => cmd}/http/main.go (81%) rename backend/{services => internal}/http/assets.go (98%) rename backend/{services => internal}/http/config.go (99%) rename backend/{services => internal}/http/geoip/geoip.go (100%) rename backend/{services => internal}/http/geoip/http.go (100%) rename backend/{services => internal}/http/handlers-ios.go (99%) rename backend/{services => internal}/http/handlers-web.go (99%) rename backend/{services => internal}/http/handlers.go (98%) rename backend/{services => internal}/http/ios-device.go (99%) rename backend/{services => internal}/http/model.go (98%) rename backend/{services => internal}/http/response.go (97%) rename backend/{services => internal}/http/router.go (99%) rename backend/{services => internal}/http/server.go (98%) rename backend/{services => internal}/http/service.go (90%) rename backend/{services => internal}/http/uaparser/http.go (100%) rename backend/{services => internal}/http/uaparser/uaparser.go (100%) rename backend/{services => internal}/http/uuid.go (93%) diff --git a/backend/services/http/README.md b/backend/cmd/http/README.md similarity index 100% rename from backend/services/http/README.md rename to backend/cmd/http/README.md diff --git a/backend/services/http/main.go b/backend/cmd/http/main.go similarity index 81% rename from backend/services/http/main.go rename to backend/cmd/http/main.go index d0e97d137..a7c1a4498 100644 --- a/backend/services/http/main.go +++ b/backend/cmd/http/main.go @@ -2,6 +2,7 @@ package main import ( "log" + "openreplay/backend/internal/http" "openreplay/backend/pkg/db/cache" "openreplay/backend/pkg/db/postgres" "openreplay/backend/pkg/pprof" @@ -16,7 +17,7 @@ func main() { pprof.StartProfilingServer() // Load configuration - cfg := NewConfig() + cfg := http.NewConfig() // Connect to queue producer := queue.NewProducer() @@ -27,16 +28,16 @@ func main() { defer dbConn.Close() // Build all services - services := NewServiceBuilder(cfg, producer, dbConn) + services := http.NewServiceBuilder(cfg, producer, dbConn) // Init server's routes - router, err := NewRouter(cfg, services) + router, err := http.NewRouter(cfg, services) if err != nil { log.Fatalf("failed while creating engine: %s", err) } // Init server - server, err := NewServer(router.GetHandler(), cfg.HTTPHost, cfg.HTTPPort, cfg.HTTPTimeout) + server, err := http.NewServer(router.GetHandler(), cfg.HTTPHost, cfg.HTTPPort, cfg.HTTPTimeout) if err != nil { log.Fatalf("failed while creating server: %s", err) } diff --git a/backend/services/http/assets.go b/backend/internal/http/assets.go similarity index 98% rename from backend/services/http/assets.go rename to backend/internal/http/assets.go index 0948984b0..f77c911f0 100644 --- a/backend/services/http/assets.go +++ b/backend/internal/http/assets.go @@ -1,4 +1,4 @@ -package main +package http import ( "openreplay/backend/pkg/messages" diff --git a/backend/services/http/config.go b/backend/internal/http/config.go similarity index 99% rename from backend/services/http/config.go rename to backend/internal/http/config.go index 4b29afb9d..56ca62067 100644 --- a/backend/services/http/config.go +++ b/backend/internal/http/config.go @@ -1,4 +1,4 @@ -package main +package http import ( "openreplay/backend/pkg/env" diff --git a/backend/services/http/geoip/geoip.go b/backend/internal/http/geoip/geoip.go similarity index 100% rename from backend/services/http/geoip/geoip.go rename to backend/internal/http/geoip/geoip.go diff --git a/backend/services/http/geoip/http.go b/backend/internal/http/geoip/http.go similarity index 100% rename from backend/services/http/geoip/http.go rename to backend/internal/http/geoip/http.go diff --git a/backend/services/http/handlers-ios.go b/backend/internal/http/handlers-ios.go similarity index 99% rename from backend/services/http/handlers-ios.go rename to backend/internal/http/handlers-ios.go index 4591a7d72..e92973f5f 100644 --- a/backend/services/http/handlers-ios.go +++ b/backend/internal/http/handlers-ios.go @@ -1,4 +1,4 @@ -package main +package http import ( "encoding/json" diff --git a/backend/services/http/handlers-web.go b/backend/internal/http/handlers-web.go similarity index 99% rename from backend/services/http/handlers-web.go rename to backend/internal/http/handlers-web.go index e9fc135b0..38ca89fb2 100644 --- a/backend/services/http/handlers-web.go +++ b/backend/internal/http/handlers-web.go @@ -1,4 +1,4 @@ -package main +package http import ( "bytes" diff --git a/backend/services/http/handlers.go b/backend/internal/http/handlers.go similarity index 98% rename from backend/services/http/handlers.go rename to backend/internal/http/handlers.go index 8f8979a4c..1854786fc 100644 --- a/backend/services/http/handlers.go +++ b/backend/internal/http/handlers.go @@ -1,4 +1,4 @@ -package main +package http import ( "io" diff --git a/backend/services/http/ios-device.go b/backend/internal/http/ios-device.go similarity index 99% rename from backend/services/http/ios-device.go rename to backend/internal/http/ios-device.go index 6a09e5e07..b5156d6dc 100644 --- a/backend/services/http/ios-device.go +++ b/backend/internal/http/ios-device.go @@ -1,4 +1,4 @@ -package main +package http import ( "strings" diff --git a/backend/services/http/model.go b/backend/internal/http/model.go similarity index 98% rename from backend/services/http/model.go rename to backend/internal/http/model.go index 5a7dd28bc..969bafe9c 100644 --- a/backend/services/http/model.go +++ b/backend/internal/http/model.go @@ -1,4 +1,4 @@ -package main +package http type startSessionRequest struct { Token string `json:"token"` diff --git a/backend/services/http/response.go b/backend/internal/http/response.go similarity index 97% rename from backend/services/http/response.go rename to backend/internal/http/response.go index 11d9b328d..2a3f14bea 100644 --- a/backend/services/http/response.go +++ b/backend/internal/http/response.go @@ -1,4 +1,4 @@ -package main +package http import ( "encoding/json" diff --git a/backend/services/http/router.go b/backend/internal/http/router.go similarity index 99% rename from backend/services/http/router.go rename to backend/internal/http/router.go index af624e32c..2fe1433a8 100644 --- a/backend/services/http/router.go +++ b/backend/internal/http/router.go @@ -1,4 +1,4 @@ -package main +package http import ( "github.com/gorilla/mux" diff --git a/backend/services/http/server.go b/backend/internal/http/server.go similarity index 98% rename from backend/services/http/server.go rename to backend/internal/http/server.go index 2b6056c20..ea7b08841 100644 --- a/backend/services/http/server.go +++ b/backend/internal/http/server.go @@ -1,4 +1,4 @@ -package main +package http import ( "context" diff --git a/backend/services/http/service.go b/backend/internal/http/service.go similarity index 90% rename from backend/services/http/service.go rename to backend/internal/http/service.go index d7eda7720..2bf46315b 100644 --- a/backend/services/http/service.go +++ b/backend/internal/http/service.go @@ -1,14 +1,14 @@ -package main +package http import ( + "openreplay/backend/internal/http/geoip" + "openreplay/backend/internal/http/uaparser" "openreplay/backend/pkg/db/cache" "openreplay/backend/pkg/flakeid" "openreplay/backend/pkg/queue/types" "openreplay/backend/pkg/storage" "openreplay/backend/pkg/token" "openreplay/backend/pkg/url/assets" - "openreplay/backend/services/http/geoip" - "openreplay/backend/services/http/uaparser" ) type ServiceBuilder struct { diff --git a/backend/services/http/uaparser/http.go b/backend/internal/http/uaparser/http.go similarity index 100% rename from backend/services/http/uaparser/http.go rename to backend/internal/http/uaparser/http.go diff --git a/backend/services/http/uaparser/uaparser.go b/backend/internal/http/uaparser/uaparser.go similarity index 100% rename from backend/services/http/uaparser/uaparser.go rename to backend/internal/http/uaparser/uaparser.go diff --git a/backend/services/http/uuid.go b/backend/internal/http/uuid.go similarity index 93% rename from backend/services/http/uuid.go rename to backend/internal/http/uuid.go index 87704d740..6ce1611a9 100644 --- a/backend/services/http/uuid.go +++ b/backend/internal/http/uuid.go @@ -1,4 +1,4 @@ -package main +package http import ( "github.com/google/uuid" From c347198fc170b1098030243d77ce5d31007c245e Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 2 May 2022 15:05:45 +0200 Subject: [PATCH 06/18] Moved http handlers to a separate dir --- backend/cmd/http/main.go | 6 +- backend/internal/{http => config}/config.go | 10 +-- backend/internal/http/model.go | 6 +- backend/internal/http/response.go | 6 +- backend/internal/http/service.go | 35 +++++----- backend/internal/{http => router}/assets.go | 8 +-- .../internal/{http => router}/handlers-ios.go | 62 ++++++++--------- .../internal/{http => router}/handlers-web.go | 66 ++++++++++--------- backend/internal/{http => router}/handlers.go | 12 ++-- backend/internal/{http => router}/router.go | 10 +-- backend/internal/{http => uuid}/uuid.go | 4 +- 11 files changed, 117 insertions(+), 108 deletions(-) rename backend/internal/{http => config}/config.go (91%) rename backend/internal/{http => router}/assets.go (82%) rename backend/internal/{http => router}/handlers-ios.go (63%) rename backend/internal/{http => router}/handlers-web.go (65%) rename backend/internal/{http => router}/handlers.go (68%) rename backend/internal/{http => router}/router.go (87%) rename backend/internal/{http => uuid}/uuid.go (76%) diff --git a/backend/cmd/http/main.go b/backend/cmd/http/main.go index a7c1a4498..fd6dbd453 100644 --- a/backend/cmd/http/main.go +++ b/backend/cmd/http/main.go @@ -2,7 +2,9 @@ package main import ( "log" + "openreplay/backend/internal/config" "openreplay/backend/internal/http" + "openreplay/backend/internal/router" "openreplay/backend/pkg/db/cache" "openreplay/backend/pkg/db/postgres" "openreplay/backend/pkg/pprof" @@ -17,7 +19,7 @@ func main() { pprof.StartProfilingServer() // Load configuration - cfg := http.NewConfig() + cfg := config.New() // Connect to queue producer := queue.NewProducer() @@ -31,7 +33,7 @@ func main() { services := http.NewServiceBuilder(cfg, producer, dbConn) // Init server's routes - router, err := http.NewRouter(cfg, services) + router, err := router.NewRouter(cfg, services) if err != nil { log.Fatalf("failed while creating engine: %s", err) } diff --git a/backend/internal/http/config.go b/backend/internal/config/config.go similarity index 91% rename from backend/internal/http/config.go rename to backend/internal/config/config.go index 56ca62067..45f957d1c 100644 --- a/backend/internal/http/config.go +++ b/backend/internal/config/config.go @@ -1,11 +1,11 @@ -package http +package config import ( "openreplay/backend/pkg/env" "time" ) -type config struct { +type Config struct { HTTPHost string HTTPPort string HTTPTimeout time.Duration @@ -25,9 +25,9 @@ type config struct { WorkerID uint16 } -func NewConfig() *config { - return &config{ - HTTPHost: "", +func New() *Config { + return &Config{ + HTTPHost: "", // empty by default HTTPPort: env.String("HTTP_PORT"), HTTPTimeout: time.Second * 60, TopicRawWeb: env.String("TOPIC_RAW_WEB"), diff --git a/backend/internal/http/model.go b/backend/internal/http/model.go index 969bafe9c..3fe4abd3d 100644 --- a/backend/internal/http/model.go +++ b/backend/internal/http/model.go @@ -1,6 +1,6 @@ package http -type startSessionRequest struct { +type StartSessionRequest struct { Token string `json:"token"` UserUUID *string `json:"userUUID"` RevID string `json:"revID"` @@ -14,7 +14,7 @@ type startSessionRequest struct { UserID string `json:"userID"` } -type startSessionResponse struct { +type StartSessionResponse struct { Timestamp int64 `json:"timestamp"` Delay int64 `json:"delay"` Token string `json:"token"` @@ -23,7 +23,7 @@ type startSessionResponse struct { BeaconSizeLimit int64 `json:"beaconSizeLimit"` } -type notStartedRequest struct { +type NotStartedRequest struct { ProjectKey *string `json:"projectKey"` TrackerVersion string `json:"trackerVersion"` DoNotTrack bool `json:"DoNotTrack"` diff --git a/backend/internal/http/response.go b/backend/internal/http/response.go index 2a3f14bea..1b87c33b9 100644 --- a/backend/internal/http/response.go +++ b/backend/internal/http/response.go @@ -6,7 +6,7 @@ import ( "net/http" ) -func responseWithJSON(w http.ResponseWriter, res interface{}) { +func ResponseWithJSON(w http.ResponseWriter, res interface{}) { body, err := json.Marshal(res) if err != nil { log.Println(err) @@ -15,10 +15,10 @@ func responseWithJSON(w http.ResponseWriter, res interface{}) { w.Write(body) } -func responseWithError(w http.ResponseWriter, code int, err error) { +func ResponseWithError(w http.ResponseWriter, code int, err error) { type response struct { Error string `json:"error"` } w.WriteHeader(code) - responseWithJSON(w, &response{err.Error()}) + ResponseWithJSON(w, &response{err.Error()}) } diff --git a/backend/internal/http/service.go b/backend/internal/http/service.go index 2bf46315b..8f553db32 100644 --- a/backend/internal/http/service.go +++ b/backend/internal/http/service.go @@ -1,6 +1,7 @@ package http import ( + "openreplay/backend/internal/config" "openreplay/backend/internal/http/geoip" "openreplay/backend/internal/http/uaparser" "openreplay/backend/pkg/db/cache" @@ -12,25 +13,25 @@ import ( ) type ServiceBuilder struct { - pgconn *cache.PGCache - producer types.Producer - rewriter *assets.Rewriter - flaker *flakeid.Flaker - uaParser *uaparser.UAParser - geoIP *geoip.GeoIP - tokenizer *token.Tokenizer - s3 *storage.S3 + Pgconn *cache.PGCache + Producer types.Producer + Rewriter *assets.Rewriter + Flaker *flakeid.Flaker + UaParser *uaparser.UAParser + GeoIP *geoip.GeoIP + Tokenizer *token.Tokenizer + S3 *storage.S3 } -func NewServiceBuilder(cfg *config, producer types.Producer, pgconn *cache.PGCache) *ServiceBuilder { +func NewServiceBuilder(cfg *config.Config, producer types.Producer, pgconn *cache.PGCache) *ServiceBuilder { return &ServiceBuilder{ - pgconn: pgconn, - producer: producer, - rewriter: assets.NewRewriter(cfg.AssetsOrigin), - s3: storage.NewS3(cfg.AWSRegion, cfg.S3BucketIOSImages), - tokenizer: token.NewTokenizer(cfg.TokenSecret), - uaParser: uaparser.NewUAParser(cfg.UAParserFile), - geoIP: geoip.NewGeoIP(cfg.MaxMinDBFile), - flaker: flakeid.NewFlaker(cfg.WorkerID), + Pgconn: pgconn, + Producer: producer, + Rewriter: assets.NewRewriter(cfg.AssetsOrigin), + S3: storage.NewS3(cfg.AWSRegion, cfg.S3BucketIOSImages), + Tokenizer: token.NewTokenizer(cfg.TokenSecret), + UaParser: uaparser.NewUAParser(cfg.UAParserFile), + GeoIP: geoip.NewGeoIP(cfg.MaxMinDBFile), + Flaker: flakeid.NewFlaker(cfg.WorkerID), } } diff --git a/backend/internal/http/assets.go b/backend/internal/router/assets.go similarity index 82% rename from backend/internal/http/assets.go rename to backend/internal/router/assets.go index f77c911f0..42347f224 100644 --- a/backend/internal/http/assets.go +++ b/backend/internal/router/assets.go @@ -1,4 +1,4 @@ -package http +package router import ( "openreplay/backend/pkg/messages" @@ -7,7 +7,7 @@ import ( func (e *Router) sendAssetForCache(sessionID uint64, baseURL string, relativeURL string) { if fullURL, cacheable := assets.GetFullCachableURL(baseURL, relativeURL); cacheable { - e.services.producer.Produce(e.cfg.TopicCache, sessionID, messages.Encode(&messages.AssetCache{ + e.services.Producer.Produce(e.cfg.TopicCache, sessionID, messages.Encode(&messages.AssetCache{ URL: fullURL, })) } @@ -22,7 +22,7 @@ func (e *Router) sendAssetsForCacheFromCSS(sessionID uint64, baseURL string, css func (e *Router) handleURL(sessionID uint64, baseURL string, url string) string { if e.cfg.CacheAssets { e.sendAssetForCache(sessionID, baseURL, url) - return e.services.rewriter.RewriteURL(sessionID, baseURL, url) + return e.services.Rewriter.RewriteURL(sessionID, baseURL, url) } return assets.ResolveURL(baseURL, url) } @@ -30,7 +30,7 @@ func (e *Router) handleURL(sessionID uint64, baseURL string, url string) string func (e *Router) handleCSS(sessionID uint64, baseURL string, css string) string { if e.cfg.CacheAssets { e.sendAssetsForCacheFromCSS(sessionID, baseURL, css) - return e.services.rewriter.RewriteCSS(sessionID, baseURL, css) + return e.services.Rewriter.RewriteCSS(sessionID, baseURL, css) } return assets.ResolveCSS(baseURL, css) } diff --git a/backend/internal/http/handlers-ios.go b/backend/internal/router/handlers-ios.go similarity index 63% rename from backend/internal/http/handlers-ios.go rename to backend/internal/router/handlers-ios.go index e92973f5f..e1853f555 100644 --- a/backend/internal/http/handlers-ios.go +++ b/backend/internal/router/handlers-ios.go @@ -1,4 +1,4 @@ -package http +package router import ( "encoding/json" @@ -6,6 +6,8 @@ import ( "log" "math/rand" "net/http" + http2 "openreplay/backend/internal/http" + "openreplay/backend/internal/uuid" "strconv" "time" @@ -39,52 +41,52 @@ func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() if err := json.NewDecoder(body).Decode(req); err != nil { - responseWithError(w, http.StatusBadRequest, err) + http2.ResponseWithError(w, http.StatusBadRequest, err) return } if req.ProjectKey == nil { - responseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) + http2.ResponseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) return } - p, err := e.services.pgconn.GetProjectByKey(*req.ProjectKey) + p, err := e.services.Pgconn.GetProjectByKey(*req.ProjectKey) if err != nil { if postgres.IsNoRowsErr(err) { - responseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or is not active")) + http2.ResponseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or is not active")) } else { - responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging + http2.ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging } return } - userUUID := getUUID(req.UserUUID) - tokenData, err := e.services.tokenizer.Parse(req.Token) + userUUID := uuid.GetUUID(req.UserUUID) + tokenData, err := e.services.Tokenizer.Parse(req.Token) if err != nil { // Starting the new one dice := byte(rand.Intn(100)) // [0, 100) if dice >= p.SampleRate { - responseWithError(w, http.StatusForbidden, errors.New("cancel")) + http2.ResponseWithError(w, http.StatusForbidden, errors.New("cancel")) return } - ua := e.services.uaParser.ParseFromHTTPRequest(r) + ua := e.services.UaParser.ParseFromHTTPRequest(r) if ua == nil { - responseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) + http2.ResponseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } - sessionID, err := e.services.flaker.Compose(uint64(startTime.UnixMilli())) + sessionID, err := e.services.Flaker.Compose(uint64(startTime.UnixMilli())) if err != nil { - responseWithError(w, http.StatusInternalServerError, err) + http2.ResponseWithError(w, http.StatusInternalServerError, err) return } // TODO: if EXPIRED => send message for two sessions association expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) tokenData = &token.TokenData{sessionID, expTime.UnixMilli()} - country := e.services.geoIP.ExtractISOCodeFromHTTPRequest(r) + country := e.services.GeoIP.ExtractISOCodeFromHTTPRequest(r) // The difference with web is mostly here: - e.services.producer.Produce(e.cfg.TopicRawIOS, tokenData.ID, Encode(&IOSSessionStart{ + e.services.Producer.Produce(e.cfg.TopicRawIOS, tokenData.ID, Encode(&IOSSessionStart{ Timestamp: req.Timestamp, ProjectID: uint64(p.ProjectID), TrackerVersion: req.TrackerVersion, @@ -92,14 +94,14 @@ func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) UserUUID: userUUID, UserOS: "IOS", UserOSVersion: req.UserOSVersion, - UserDevice: MapIOSDevice(req.UserDevice), - UserDeviceType: GetIOSDeviceType(req.UserDevice), + UserDevice: http2.MapIOSDevice(req.UserDevice), + UserDeviceType: http2.GetIOSDeviceType(req.UserDevice), UserCountry: country, })) } - responseWithJSON(w, &response{ - Token: e.services.tokenizer.Compose(*tokenData), + http2.ResponseWithJSON(w, &response{ + Token: e.services.Tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), BeaconSizeLimit: e.cfg.BeaconSizeLimit, @@ -107,18 +109,18 @@ func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) } func (e *Router) pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { - sessionData, err := e.services.tokenizer.ParseFromHTTPRequest(r) + sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r) if err != nil { - responseWithError(w, http.StatusUnauthorized, err) + http2.ResponseWithError(w, http.StatusUnauthorized, err) return } e.pushMessages(w, r, sessionData.ID, e.cfg.TopicRawIOS) } func (e *Router) pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { - sessionData, err := e.services.tokenizer.ParseFromHTTPRequest(r) + sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r) if err != nil && err != token.EXPIRED { - responseWithError(w, http.StatusUnauthorized, err) + http2.ResponseWithError(w, http.StatusUnauthorized, err) return } // Check timestamps here? @@ -128,9 +130,9 @@ func (e *Router) pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Reque func (e *Router) imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { log.Printf("recieved imagerequest") - sessionData, err := e.services.tokenizer.ParseFromHTTPRequest(r) + sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r) if err != nil { // Should accept expired token? - responseWithError(w, http.StatusUnauthorized, err) + http2.ResponseWithError(w, http.StatusUnauthorized, err) return } @@ -138,18 +140,18 @@ func (e *Router) imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) defer r.Body.Close() err = r.ParseMultipartForm(1e6) // ~1Mb if err == http.ErrNotMultipart || err == http.ErrMissingBoundary { - responseWithError(w, http.StatusUnsupportedMediaType, err) + http2.ResponseWithError(w, http.StatusUnsupportedMediaType, err) // } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB } else if err != nil { - responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging + http2.ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging } if r.MultipartForm == nil { - responseWithError(w, http.StatusInternalServerError, errors.New("Multipart not parsed")) + http2.ResponseWithError(w, http.StatusInternalServerError, errors.New("Multipart not parsed")) } if len(r.MultipartForm.Value["projectKey"]) == 0 { - responseWithError(w, http.StatusBadRequest, errors.New("projectKey parameter missing")) // status for missing/wrong parameter? + http2.ResponseWithError(w, http.StatusBadRequest, errors.New("projectKey parameter missing")) // status for missing/wrong parameter? return } @@ -164,7 +166,7 @@ func (e *Router) imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) key := prefix + fileHeader.Filename log.Printf("Uploading image... %v", key) go func() { //TODO: mime type from header - if err := e.services.s3.Upload(file, key, "image/jpeg", false); err != nil { + if err := e.services.S3.Upload(file, key, "image/jpeg", false); err != nil { log.Printf("Upload ios screen error. %v", err) } }() diff --git a/backend/internal/http/handlers-web.go b/backend/internal/router/handlers-web.go similarity index 65% rename from backend/internal/http/handlers-web.go rename to backend/internal/router/handlers-web.go index 38ca89fb2..7bf3b6e7f 100644 --- a/backend/internal/http/handlers-web.go +++ b/backend/internal/router/handlers-web.go @@ -1,4 +1,4 @@ -package http +package router import ( "bytes" @@ -7,6 +7,8 @@ import ( "log" "math/rand" "net/http" + http2 "openreplay/backend/internal/http" + "openreplay/backend/internal/uuid" "strconv" "time" @@ -20,58 +22,58 @@ func (e *Router) startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) // Check request body if r.Body == nil { - responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + http2.ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() // Parse request body - req := &startSessionRequest{} + req := &http2.StartSessionRequest{} if err := json.NewDecoder(body).Decode(req); err != nil { - responseWithError(w, http.StatusBadRequest, err) + http2.ResponseWithError(w, http.StatusBadRequest, err) return } // Handler's logic if req.ProjectKey == nil { - responseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) + http2.ResponseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) return } - p, err := e.services.pgconn.GetProjectByKey(*req.ProjectKey) + p, err := e.services.Pgconn.GetProjectByKey(*req.ProjectKey) if err != nil { if postgres.IsNoRowsErr(err) { - responseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or capture limit has been reached")) + http2.ResponseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or capture limit has been reached")) } else { - responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging + http2.ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging } return } - userUUID := getUUID(req.UserUUID) - tokenData, err := e.services.tokenizer.Parse(req.Token) + userUUID := uuid.GetUUID(req.UserUUID) + tokenData, err := e.services.Tokenizer.Parse(req.Token) if err != nil || req.Reset { // Starting the new one dice := byte(rand.Intn(100)) // [0, 100) if dice >= p.SampleRate { - responseWithError(w, http.StatusForbidden, errors.New("cancel")) + http2.ResponseWithError(w, http.StatusForbidden, errors.New("cancel")) return } - ua := e.services.uaParser.ParseFromHTTPRequest(r) + ua := e.services.UaParser.ParseFromHTTPRequest(r) if ua == nil { - responseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) + http2.ResponseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } - sessionID, err := e.services.flaker.Compose(uint64(startTime.UnixNano() / 1e6)) + sessionID, err := e.services.Flaker.Compose(uint64(startTime.UnixNano() / 1e6)) if err != nil { - responseWithError(w, http.StatusInternalServerError, err) + http2.ResponseWithError(w, http.StatusInternalServerError, err) return } // TODO: if EXPIRED => send message for two sessions association expTime := startTime.Add(time.Duration(p.MaxSessionDuration) * time.Millisecond) tokenData = &token.TokenData{ID: sessionID, ExpTime: expTime.UnixNano() / 1e6} - e.services.producer.Produce(e.cfg.TopicRawWeb, tokenData.ID, Encode(&SessionStart{ + e.services.Producer.Produce(e.cfg.TopicRawWeb, tokenData.ID, Encode(&SessionStart{ Timestamp: req.Timestamp, ProjectID: uint64(p.ProjectID), TrackerVersion: req.TrackerVersion, @@ -84,15 +86,15 @@ func (e *Router) startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) UserBrowserVersion: ua.BrowserVersion, UserDevice: ua.Device, UserDeviceType: ua.DeviceType, - UserCountry: e.services.geoIP.ExtractISOCodeFromHTTPRequest(r), + UserCountry: e.services.GeoIP.ExtractISOCodeFromHTTPRequest(r), UserDeviceMemorySize: req.DeviceMemory, UserDeviceHeapSize: req.JsHeapSizeLimit, UserID: req.UserID, })) } - responseWithJSON(w, &startSessionResponse{ - Token: e.services.tokenizer.Compose(*tokenData), + http2.ResponseWithJSON(w, &http2.StartSessionResponse{ + Token: e.services.Tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), BeaconSizeLimit: e.cfg.BeaconSizeLimit, @@ -101,15 +103,15 @@ func (e *Router) startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) func (e *Router) pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) { // Check authorization - sessionData, err := e.services.tokenizer.ParseFromHTTPRequest(r) + sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r) if err != nil { - responseWithError(w, http.StatusUnauthorized, err) + http2.ResponseWithError(w, http.StatusUnauthorized, err) return } // Check request body if r.Body == nil { - responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + http2.ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } body := http.MaxBytesReader(w, r.Body, e.cfg.BeaconSizeLimit) defer body.Close() @@ -148,12 +150,12 @@ func (e *Router) pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) handledMessages.Write(msg.Encode()) }) if err != nil { - responseWithError(w, http.StatusForbidden, err) + http2.ResponseWithError(w, http.StatusForbidden, err) return } // Send processed messages to queue as array of bytes - err = e.services.producer.Produce(e.cfg.TopicRawWeb, sessionData.ID, handledMessages.Bytes()) + err = e.services.Producer.Produce(e.cfg.TopicRawWeb, sessionData.ID, handledMessages.Bytes()) if err != nil { log.Printf("can't send processed messages to queue: %s", err) } @@ -164,30 +166,30 @@ func (e *Router) pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) func (e *Router) notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { // Check request body if r.Body == nil { - responseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + http2.ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() // Parse request body - req := ¬StartedRequest{} + req := &http2.NotStartedRequest{} if err := json.NewDecoder(body).Decode(req); err != nil { - responseWithError(w, http.StatusBadRequest, err) + http2.ResponseWithError(w, http.StatusBadRequest, err) return } // Handler's logic if req.ProjectKey == nil { - responseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) + http2.ResponseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) return } - ua := e.services.uaParser.ParseFromHTTPRequest(r) // TODO?: insert anyway + ua := e.services.UaParser.ParseFromHTTPRequest(r) // TODO?: insert anyway if ua == nil { - responseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) + http2.ResponseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } - country := e.services.geoIP.ExtractISOCodeFromHTTPRequest(r) - err := e.services.pgconn.InsertUnstartedSession(postgres.UnstartedSession{ + country := e.services.GeoIP.ExtractISOCodeFromHTTPRequest(r) + err := e.services.Pgconn.InsertUnstartedSession(postgres.UnstartedSession{ ProjectKey: *req.ProjectKey, TrackerVersion: req.TrackerVersion, DoNotTrack: req.DoNotTrack, diff --git a/backend/internal/http/handlers.go b/backend/internal/router/handlers.go similarity index 68% rename from backend/internal/http/handlers.go rename to backend/internal/router/handlers.go index 1854786fc..81a055145 100644 --- a/backend/internal/http/handlers.go +++ b/backend/internal/router/handlers.go @@ -1,12 +1,12 @@ -package http +package router import ( + gzip "github.com/klauspost/pgzip" "io" "io/ioutil" "log" "net/http" - - gzip "github.com/klauspost/pgzip" + http2 "openreplay/backend/internal/http" ) func (e *Router) pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64, topicName string) { @@ -20,7 +20,7 @@ func (e *Router) pushMessages(w http.ResponseWriter, r *http.Request, sessionID reader, err = gzip.NewReader(body) if err != nil { - responseWithError(w, http.StatusInternalServerError, err) // TODO: stage-dependent responce + http2.ResponseWithError(w, http.StatusInternalServerError, err) // TODO: stage-dependent responce return } log.Println("Gzip reader init", reader) @@ -31,9 +31,9 @@ func (e *Router) pushMessages(w http.ResponseWriter, r *http.Request, sessionID log.Println("Reader after switch:", reader) buf, err := ioutil.ReadAll(reader) if err != nil { - responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging + http2.ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging return } - e.services.producer.Produce(topicName, sessionID, buf) // What if not able to send? + e.services.Producer.Produce(topicName, sessionID, buf) // What if not able to send? w.WriteHeader(http.StatusOK) } diff --git a/backend/internal/http/router.go b/backend/internal/router/router.go similarity index 87% rename from backend/internal/http/router.go rename to backend/internal/router/router.go index 2fe1433a8..c49bbf161 100644 --- a/backend/internal/http/router.go +++ b/backend/internal/router/router.go @@ -1,18 +1,20 @@ -package http +package router import ( "github.com/gorilla/mux" "log" "net/http" + "openreplay/backend/internal/config" + http2 "openreplay/backend/internal/http" ) type Router struct { router *mux.Router - cfg *config - services *ServiceBuilder + cfg *config.Config + services *http2.ServiceBuilder } -func NewRouter(cfg *config, services *ServiceBuilder) (*Router, error) { +func NewRouter(cfg *config.Config, services *http2.ServiceBuilder) (*Router, error) { e := &Router{ cfg: cfg, services: services, diff --git a/backend/internal/http/uuid.go b/backend/internal/uuid/uuid.go similarity index 76% rename from backend/internal/http/uuid.go rename to backend/internal/uuid/uuid.go index 6ce1611a9..44dd76827 100644 --- a/backend/internal/http/uuid.go +++ b/backend/internal/uuid/uuid.go @@ -1,10 +1,10 @@ -package http +package uuid import ( "github.com/google/uuid" ) -func getUUID(u *string) string { +func GetUUID(u *string) string { if u != nil { _, err := uuid.Parse(*u) if err == nil { From df722761e55b76e69ca63fa4a224a097e973755c Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 2 May 2022 15:20:10 +0200 Subject: [PATCH 07/18] Moved server to a separate dir --- backend/cmd/http/main.go | 3 ++- backend/internal/{http => server}/server.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) rename backend/internal/{http => server}/server.go (87%) diff --git a/backend/cmd/http/main.go b/backend/cmd/http/main.go index fd6dbd453..49507727c 100644 --- a/backend/cmd/http/main.go +++ b/backend/cmd/http/main.go @@ -5,6 +5,7 @@ import ( "openreplay/backend/internal/config" "openreplay/backend/internal/http" "openreplay/backend/internal/router" + "openreplay/backend/internal/server" "openreplay/backend/pkg/db/cache" "openreplay/backend/pkg/db/postgres" "openreplay/backend/pkg/pprof" @@ -39,7 +40,7 @@ func main() { } // Init server - server, err := http.NewServer(router.GetHandler(), cfg.HTTPHost, cfg.HTTPPort, cfg.HTTPTimeout) + server, err := server.New(router.GetHandler(), cfg.HTTPHost, cfg.HTTPPort, cfg.HTTPTimeout) if err != nil { log.Fatalf("failed while creating server: %s", err) } diff --git a/backend/internal/http/server.go b/backend/internal/server/server.go similarity index 87% rename from backend/internal/http/server.go rename to backend/internal/server/server.go index ea7b08841..2670ba537 100644 --- a/backend/internal/http/server.go +++ b/backend/internal/server/server.go @@ -1,4 +1,4 @@ -package http +package server import ( "context" @@ -14,7 +14,7 @@ type Server struct { server *http.Server } -func NewServer(handler http.Handler, host, port string, timeout time.Duration) (*Server, error) { +func New(handler http.Handler, host, port string, timeout time.Duration) (*Server, error) { switch { case port == "": return nil, errors.New("empty server port") From 69cabaecfe593dc3ab3373f9fb461a1f9d160914 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 2 May 2022 15:28:51 +0200 Subject: [PATCH 08/18] Moved the rest of the code to separate dirs --- backend/cmd/http/main.go | 4 +- backend/internal/{http => }/geoip/geoip.go | 0 backend/internal/{http => }/geoip/http.go | 0 backend/internal/{http => ios}/ios-device.go | 2 +- backend/internal/router/handlers-ios.go | 36 +++++++++--------- backend/internal/router/handlers-web.go | 37 +++++++++---------- backend/internal/router/handlers.go | 5 +-- backend/internal/{http => router}/model.go | 2 +- backend/internal/{http => router}/response.go | 2 +- backend/internal/router/router.go | 6 +-- .../{http/service.go => services/services.go} | 12 +++--- backend/internal/{http => }/uaparser/http.go | 0 .../internal/{http => }/uaparser/uaparser.go | 0 13 files changed, 52 insertions(+), 54 deletions(-) rename backend/internal/{http => }/geoip/geoip.go (100%) rename backend/internal/{http => }/geoip/http.go (100%) rename backend/internal/{http => ios}/ios-device.go (99%) rename backend/internal/{http => router}/model.go (98%) rename backend/internal/{http => router}/response.go (96%) rename backend/internal/{http/service.go => services/services.go} (76%) rename backend/internal/{http => }/uaparser/http.go (100%) rename backend/internal/{http => }/uaparser/uaparser.go (100%) diff --git a/backend/cmd/http/main.go b/backend/cmd/http/main.go index 49507727c..41f739277 100644 --- a/backend/cmd/http/main.go +++ b/backend/cmd/http/main.go @@ -3,9 +3,9 @@ package main import ( "log" "openreplay/backend/internal/config" - "openreplay/backend/internal/http" "openreplay/backend/internal/router" "openreplay/backend/internal/server" + "openreplay/backend/internal/services" "openreplay/backend/pkg/db/cache" "openreplay/backend/pkg/db/postgres" "openreplay/backend/pkg/pprof" @@ -31,7 +31,7 @@ func main() { defer dbConn.Close() // Build all services - services := http.NewServiceBuilder(cfg, producer, dbConn) + services := services.New(cfg, producer, dbConn) // Init server's routes router, err := router.NewRouter(cfg, services) diff --git a/backend/internal/http/geoip/geoip.go b/backend/internal/geoip/geoip.go similarity index 100% rename from backend/internal/http/geoip/geoip.go rename to backend/internal/geoip/geoip.go diff --git a/backend/internal/http/geoip/http.go b/backend/internal/geoip/http.go similarity index 100% rename from backend/internal/http/geoip/http.go rename to backend/internal/geoip/http.go diff --git a/backend/internal/http/ios-device.go b/backend/internal/ios/ios-device.go similarity index 99% rename from backend/internal/http/ios-device.go rename to backend/internal/ios/ios-device.go index b5156d6dc..8df33035b 100644 --- a/backend/internal/http/ios-device.go +++ b/backend/internal/ios/ios-device.go @@ -1,4 +1,4 @@ -package http +package ios import ( "strings" diff --git a/backend/internal/router/handlers-ios.go b/backend/internal/router/handlers-ios.go index e1853f555..f0b0d977f 100644 --- a/backend/internal/router/handlers-ios.go +++ b/backend/internal/router/handlers-ios.go @@ -6,7 +6,7 @@ import ( "log" "math/rand" "net/http" - http2 "openreplay/backend/internal/http" + "openreplay/backend/internal/ios" "openreplay/backend/internal/uuid" "strconv" "time" @@ -41,21 +41,21 @@ func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() if err := json.NewDecoder(body).Decode(req); err != nil { - http2.ResponseWithError(w, http.StatusBadRequest, err) + ResponseWithError(w, http.StatusBadRequest, err) return } if req.ProjectKey == nil { - http2.ResponseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) + ResponseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) return } p, err := e.services.Pgconn.GetProjectByKey(*req.ProjectKey) if err != nil { if postgres.IsNoRowsErr(err) { - http2.ResponseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or is not active")) + ResponseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or is not active")) } else { - http2.ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging + ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging } return } @@ -65,18 +65,18 @@ func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) if err != nil { // Starting the new one dice := byte(rand.Intn(100)) // [0, 100) if dice >= p.SampleRate { - http2.ResponseWithError(w, http.StatusForbidden, errors.New("cancel")) + ResponseWithError(w, http.StatusForbidden, errors.New("cancel")) return } ua := e.services.UaParser.ParseFromHTTPRequest(r) if ua == nil { - http2.ResponseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) + ResponseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } sessionID, err := e.services.Flaker.Compose(uint64(startTime.UnixMilli())) if err != nil { - http2.ResponseWithError(w, http.StatusInternalServerError, err) + ResponseWithError(w, http.StatusInternalServerError, err) return } // TODO: if EXPIRED => send message for two sessions association @@ -94,13 +94,13 @@ func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) UserUUID: userUUID, UserOS: "IOS", UserOSVersion: req.UserOSVersion, - UserDevice: http2.MapIOSDevice(req.UserDevice), - UserDeviceType: http2.GetIOSDeviceType(req.UserDevice), + UserDevice: ios.MapIOSDevice(req.UserDevice), + UserDeviceType: ios.GetIOSDeviceType(req.UserDevice), UserCountry: country, })) } - http2.ResponseWithJSON(w, &response{ + ResponseWithJSON(w, &response{ Token: e.services.Tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), @@ -111,7 +111,7 @@ func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) func (e *Router) pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r) if err != nil { - http2.ResponseWithError(w, http.StatusUnauthorized, err) + ResponseWithError(w, http.StatusUnauthorized, err) return } e.pushMessages(w, r, sessionData.ID, e.cfg.TopicRawIOS) @@ -120,7 +120,7 @@ func (e *Router) pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) func (e *Router) pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r) if err != nil && err != token.EXPIRED { - http2.ResponseWithError(w, http.StatusUnauthorized, err) + ResponseWithError(w, http.StatusUnauthorized, err) return } // Check timestamps here? @@ -132,7 +132,7 @@ func (e *Router) imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r) if err != nil { // Should accept expired token? - http2.ResponseWithError(w, http.StatusUnauthorized, err) + ResponseWithError(w, http.StatusUnauthorized, err) return } @@ -140,18 +140,18 @@ func (e *Router) imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) defer r.Body.Close() err = r.ParseMultipartForm(1e6) // ~1Mb if err == http.ErrNotMultipart || err == http.ErrMissingBoundary { - http2.ResponseWithError(w, http.StatusUnsupportedMediaType, err) + ResponseWithError(w, http.StatusUnsupportedMediaType, err) // } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB } else if err != nil { - http2.ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging + ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging } if r.MultipartForm == nil { - http2.ResponseWithError(w, http.StatusInternalServerError, errors.New("Multipart not parsed")) + ResponseWithError(w, http.StatusInternalServerError, errors.New("Multipart not parsed")) } if len(r.MultipartForm.Value["projectKey"]) == 0 { - http2.ResponseWithError(w, http.StatusBadRequest, errors.New("projectKey parameter missing")) // status for missing/wrong parameter? + ResponseWithError(w, http.StatusBadRequest, errors.New("projectKey parameter missing")) // status for missing/wrong parameter? return } diff --git a/backend/internal/router/handlers-web.go b/backend/internal/router/handlers-web.go index 7bf3b6e7f..b0a3a9b94 100644 --- a/backend/internal/router/handlers-web.go +++ b/backend/internal/router/handlers-web.go @@ -7,7 +7,6 @@ import ( "log" "math/rand" "net/http" - http2 "openreplay/backend/internal/http" "openreplay/backend/internal/uuid" "strconv" "time" @@ -22,30 +21,30 @@ func (e *Router) startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) // Check request body if r.Body == nil { - http2.ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() // Parse request body - req := &http2.StartSessionRequest{} + req := &StartSessionRequest{} if err := json.NewDecoder(body).Decode(req); err != nil { - http2.ResponseWithError(w, http.StatusBadRequest, err) + ResponseWithError(w, http.StatusBadRequest, err) return } // Handler's logic if req.ProjectKey == nil { - http2.ResponseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) + ResponseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) return } p, err := e.services.Pgconn.GetProjectByKey(*req.ProjectKey) if err != nil { if postgres.IsNoRowsErr(err) { - http2.ResponseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or capture limit has been reached")) + ResponseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or capture limit has been reached")) } else { - http2.ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging + ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging } return } @@ -55,18 +54,18 @@ func (e *Router) startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) if err != nil || req.Reset { // Starting the new one dice := byte(rand.Intn(100)) // [0, 100) if dice >= p.SampleRate { - http2.ResponseWithError(w, http.StatusForbidden, errors.New("cancel")) + ResponseWithError(w, http.StatusForbidden, errors.New("cancel")) return } ua := e.services.UaParser.ParseFromHTTPRequest(r) if ua == nil { - http2.ResponseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) + ResponseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } sessionID, err := e.services.Flaker.Compose(uint64(startTime.UnixNano() / 1e6)) if err != nil { - http2.ResponseWithError(w, http.StatusInternalServerError, err) + ResponseWithError(w, http.StatusInternalServerError, err) return } // TODO: if EXPIRED => send message for two sessions association @@ -93,7 +92,7 @@ func (e *Router) startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) })) } - http2.ResponseWithJSON(w, &http2.StartSessionResponse{ + ResponseWithJSON(w, &StartSessionResponse{ Token: e.services.Tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), @@ -105,13 +104,13 @@ func (e *Router) pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) // Check authorization sessionData, err := e.services.Tokenizer.ParseFromHTTPRequest(r) if err != nil { - http2.ResponseWithError(w, http.StatusUnauthorized, err) + ResponseWithError(w, http.StatusUnauthorized, err) return } // Check request body if r.Body == nil { - http2.ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } body := http.MaxBytesReader(w, r.Body, e.cfg.BeaconSizeLimit) defer body.Close() @@ -150,7 +149,7 @@ func (e *Router) pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) handledMessages.Write(msg.Encode()) }) if err != nil { - http2.ResponseWithError(w, http.StatusForbidden, err) + ResponseWithError(w, http.StatusForbidden, err) return } @@ -166,26 +165,26 @@ func (e *Router) pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) func (e *Router) notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { // Check request body if r.Body == nil { - http2.ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) } body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() // Parse request body - req := &http2.NotStartedRequest{} + req := &NotStartedRequest{} if err := json.NewDecoder(body).Decode(req); err != nil { - http2.ResponseWithError(w, http.StatusBadRequest, err) + ResponseWithError(w, http.StatusBadRequest, err) return } // Handler's logic if req.ProjectKey == nil { - http2.ResponseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) + ResponseWithError(w, http.StatusForbidden, errors.New("ProjectKey value required")) return } ua := e.services.UaParser.ParseFromHTTPRequest(r) // TODO?: insert anyway if ua == nil { - http2.ResponseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) + ResponseWithError(w, http.StatusForbidden, errors.New("browser not recognized")) return } country := e.services.GeoIP.ExtractISOCodeFromHTTPRequest(r) diff --git a/backend/internal/router/handlers.go b/backend/internal/router/handlers.go index 81a055145..e6e4c1d26 100644 --- a/backend/internal/router/handlers.go +++ b/backend/internal/router/handlers.go @@ -6,7 +6,6 @@ import ( "io/ioutil" "log" "net/http" - http2 "openreplay/backend/internal/http" ) func (e *Router) pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64, topicName string) { @@ -20,7 +19,7 @@ func (e *Router) pushMessages(w http.ResponseWriter, r *http.Request, sessionID reader, err = gzip.NewReader(body) if err != nil { - http2.ResponseWithError(w, http.StatusInternalServerError, err) // TODO: stage-dependent responce + ResponseWithError(w, http.StatusInternalServerError, err) // TODO: stage-dependent responce return } log.Println("Gzip reader init", reader) @@ -31,7 +30,7 @@ func (e *Router) pushMessages(w http.ResponseWriter, r *http.Request, sessionID log.Println("Reader after switch:", reader) buf, err := ioutil.ReadAll(reader) if err != nil { - http2.ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging + ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging return } e.services.Producer.Produce(topicName, sessionID, buf) // What if not able to send? diff --git a/backend/internal/http/model.go b/backend/internal/router/model.go similarity index 98% rename from backend/internal/http/model.go rename to backend/internal/router/model.go index 3fe4abd3d..339593d98 100644 --- a/backend/internal/http/model.go +++ b/backend/internal/router/model.go @@ -1,4 +1,4 @@ -package http +package router type StartSessionRequest struct { Token string `json:"token"` diff --git a/backend/internal/http/response.go b/backend/internal/router/response.go similarity index 96% rename from backend/internal/http/response.go rename to backend/internal/router/response.go index 1b87c33b9..0b4725419 100644 --- a/backend/internal/http/response.go +++ b/backend/internal/router/response.go @@ -1,4 +1,4 @@ -package http +package router import ( "encoding/json" diff --git a/backend/internal/router/router.go b/backend/internal/router/router.go index c49bbf161..145630351 100644 --- a/backend/internal/router/router.go +++ b/backend/internal/router/router.go @@ -5,16 +5,16 @@ import ( "log" "net/http" "openreplay/backend/internal/config" - http2 "openreplay/backend/internal/http" + http2 "openreplay/backend/internal/services" ) type Router struct { router *mux.Router cfg *config.Config - services *http2.ServiceBuilder + services *http2.ServicesBuilder } -func NewRouter(cfg *config.Config, services *http2.ServiceBuilder) (*Router, error) { +func NewRouter(cfg *config.Config, services *http2.ServicesBuilder) (*Router, error) { e := &Router{ cfg: cfg, services: services, diff --git a/backend/internal/http/service.go b/backend/internal/services/services.go similarity index 76% rename from backend/internal/http/service.go rename to backend/internal/services/services.go index 8f553db32..62569630e 100644 --- a/backend/internal/http/service.go +++ b/backend/internal/services/services.go @@ -1,9 +1,9 @@ -package http +package services import ( "openreplay/backend/internal/config" - "openreplay/backend/internal/http/geoip" - "openreplay/backend/internal/http/uaparser" + "openreplay/backend/internal/geoip" + "openreplay/backend/internal/uaparser" "openreplay/backend/pkg/db/cache" "openreplay/backend/pkg/flakeid" "openreplay/backend/pkg/queue/types" @@ -12,7 +12,7 @@ import ( "openreplay/backend/pkg/url/assets" ) -type ServiceBuilder struct { +type ServicesBuilder struct { Pgconn *cache.PGCache Producer types.Producer Rewriter *assets.Rewriter @@ -23,8 +23,8 @@ type ServiceBuilder struct { S3 *storage.S3 } -func NewServiceBuilder(cfg *config.Config, producer types.Producer, pgconn *cache.PGCache) *ServiceBuilder { - return &ServiceBuilder{ +func New(cfg *config.Config, producer types.Producer, pgconn *cache.PGCache) *ServicesBuilder { + return &ServicesBuilder{ Pgconn: pgconn, Producer: producer, Rewriter: assets.NewRewriter(cfg.AssetsOrigin), diff --git a/backend/internal/http/uaparser/http.go b/backend/internal/uaparser/http.go similarity index 100% rename from backend/internal/http/uaparser/http.go rename to backend/internal/uaparser/http.go diff --git a/backend/internal/http/uaparser/uaparser.go b/backend/internal/uaparser/uaparser.go similarity index 100% rename from backend/internal/http/uaparser/uaparser.go rename to backend/internal/uaparser/uaparser.go From 5ec46ad7530bee862d939d628d3661cd34b1eb16 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 2 May 2022 17:36:33 +0200 Subject: [PATCH 09/18] Moved assets cache logic --- backend/internal/assetscache/assets.go | 83 +++++++++++++++++++++++++ backend/internal/config/config.go | 2 + backend/internal/router/assets.go | 36 ----------- backend/internal/router/handlers-ios.go | 32 +++------- backend/internal/router/handlers-web.go | 32 +--------- backend/internal/router/handlers.go | 4 +- backend/internal/router/model.go | 19 ++++++ backend/internal/services/services.go | 14 +++-- 8 files changed, 126 insertions(+), 96 deletions(-) create mode 100644 backend/internal/assetscache/assets.go delete mode 100644 backend/internal/router/assets.go diff --git a/backend/internal/assetscache/assets.go b/backend/internal/assetscache/assets.go new file mode 100644 index 000000000..1ef70b56c --- /dev/null +++ b/backend/internal/assetscache/assets.go @@ -0,0 +1,83 @@ +package assetscache + +import ( + "openreplay/backend/internal/config" + "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/queue/types" + "openreplay/backend/pkg/url/assets" +) + +type AssetsCache struct { + cfg *config.Config + rewriter *assets.Rewriter + producer types.Producer +} + +func New(cfg *config.Config, rewriter *assets.Rewriter, producer types.Producer) *AssetsCache { + return &AssetsCache{ + cfg: cfg, + rewriter: rewriter, + producer: producer, + } +} + +func (e *AssetsCache) ParseAssets(sessID uint64, msg messages.Message) messages.Message { + switch m := msg.(type) { + case *messages.SetNodeAttributeURLBased: + if m.Name == "src" || m.Name == "href" { + return &messages.SetNodeAttribute{ + ID: m.ID, + Name: m.Name, + Value: e.handleURL(sessID, m.BaseURL, m.Value), + } + } else if m.Name == "style" { + return &messages.SetNodeAttribute{ + ID: m.ID, + Name: m.Name, + Value: e.handleCSS(sessID, m.BaseURL, m.Value), + } + } + case *messages.SetCSSDataURLBased: + return &messages.SetCSSData{ + ID: m.ID, + Data: e.handleCSS(sessID, m.BaseURL, m.Data), + } + case *messages.CSSInsertRuleURLBased: + return &messages.CSSInsertRule{ + ID: m.ID, + Index: m.Index, + Rule: e.handleCSS(sessID, m.BaseURL, m.Rule), + } + } + return msg +} + +func (e *AssetsCache) sendAssetForCache(sessionID uint64, baseURL string, relativeURL string) { + if fullURL, cacheable := assets.GetFullCachableURL(baseURL, relativeURL); cacheable { + e.producer.Produce(e.cfg.TopicCache, sessionID, messages.Encode(&messages.AssetCache{ + URL: fullURL, + })) + } +} + +func (e *AssetsCache) sendAssetsForCacheFromCSS(sessionID uint64, baseURL string, css string) { + for _, u := range assets.ExtractURLsFromCSS(css) { // TODO: in one shot with rewriting + e.sendAssetForCache(sessionID, baseURL, u) + } +} + +func (e *AssetsCache) handleURL(sessionID uint64, baseURL string, url string) string { + if e.cfg.CacheAssets { + e.sendAssetForCache(sessionID, baseURL, url) + return e.rewriter.RewriteURL(sessionID, baseURL, url) + } + return assets.ResolveURL(baseURL, url) +} + +func (e *AssetsCache) handleCSS(sessionID uint64, baseURL string, css string) string { + if e.cfg.CacheAssets { + e.sendAssetsForCacheFromCSS(sessionID, baseURL, css) + return e.rewriter.RewriteCSS(sessionID, baseURL, css) + } + return assets.ResolveCSS(baseURL, css) +} diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 45f957d1c..5b55ba346 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -15,6 +15,7 @@ type Config struct { CacheAssets bool BeaconSizeLimit int64 JsonSizeLimit int64 + FileSizeLimit int64 AssetsOrigin string AWSRegion string S3BucketIOSImages string @@ -36,6 +37,7 @@ func New() *Config { CacheAssets: env.Bool("CACHE_ASSETS"), BeaconSizeLimit: int64(env.Uint64("BEACON_SIZE_LIMIT")), JsonSizeLimit: 1e3, // 1Kb + FileSizeLimit: 1e7, // 10Mb AssetsOrigin: env.String("ASSETS_ORIGIN"), AWSRegion: env.String("AWS_REGION"), S3BucketIOSImages: env.String("S3_BUCKET_IOS_IMAGES"), diff --git a/backend/internal/router/assets.go b/backend/internal/router/assets.go deleted file mode 100644 index 42347f224..000000000 --- a/backend/internal/router/assets.go +++ /dev/null @@ -1,36 +0,0 @@ -package router - -import ( - "openreplay/backend/pkg/messages" - "openreplay/backend/pkg/url/assets" -) - -func (e *Router) sendAssetForCache(sessionID uint64, baseURL string, relativeURL string) { - if fullURL, cacheable := assets.GetFullCachableURL(baseURL, relativeURL); cacheable { - e.services.Producer.Produce(e.cfg.TopicCache, sessionID, messages.Encode(&messages.AssetCache{ - URL: fullURL, - })) - } -} - -func (e *Router) sendAssetsForCacheFromCSS(sessionID uint64, baseURL string, css string) { - for _, u := range assets.ExtractURLsFromCSS(css) { // TODO: in one shot with rewriting - e.sendAssetForCache(sessionID, baseURL, u) - } -} - -func (e *Router) handleURL(sessionID uint64, baseURL string, url string) string { - if e.cfg.CacheAssets { - e.sendAssetForCache(sessionID, baseURL, url) - return e.services.Rewriter.RewriteURL(sessionID, baseURL, url) - } - return assets.ResolveURL(baseURL, url) -} - -func (e *Router) handleCSS(sessionID uint64, baseURL string, css string) string { - if e.cfg.CacheAssets { - e.sendAssetsForCacheFromCSS(sessionID, baseURL, css) - return e.services.Rewriter.RewriteCSS(sessionID, baseURL, css) - } - return assets.ResolveCSS(baseURL, css) -} diff --git a/backend/internal/router/handlers-ios.go b/backend/internal/router/handlers-ios.go index f0b0d977f..426de7c05 100644 --- a/backend/internal/router/handlers-ios.go +++ b/backend/internal/router/handlers-ios.go @@ -16,30 +16,13 @@ import ( "openreplay/backend/pkg/token" ) -const FILES_SIZE_LIMIT int64 = 1e7 // 10Mb - func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { - type request struct { - Token string `json:"token"` - ProjectKey *string `json:"projectKey"` - TrackerVersion string `json:"trackerVersion"` - RevID string `json:"revID"` - UserUUID *string `json:"userUUID"` - UserOSVersion string `json:"userOSVersion"` - UserDevice string `json:"userDevice"` - Timestamp uint64 `json:"timestamp"` - } - type response struct { - Token string `json:"token"` - ImagesHashList []string `json:"imagesHashList"` - UserUUID string `json:"userUUID"` - BeaconSizeLimit int64 `json:"beaconSizeLimit"` - SessionID string `json:"sessionID"` - } startTime := time.Now() - req := &request{} + req := &StartIOSSessionRequest{} + body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() + if err := json.NewDecoder(body).Decode(req); err != nil { ResponseWithError(w, http.StatusBadRequest, err) return @@ -50,7 +33,7 @@ func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) return } - p, err := e.services.Pgconn.GetProjectByKey(*req.ProjectKey) + p, err := e.services.Database.GetProjectByKey(*req.ProjectKey) if err != nil { if postgres.IsNoRowsErr(err) { ResponseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or is not active")) @@ -100,7 +83,7 @@ func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) })) } - ResponseWithJSON(w, &response{ + ResponseWithJSON(w, &StartIOSSessionResponse{ Token: e.services.Tokenizer.Compose(*tokenData), UserUUID: userUUID, SessionID: strconv.FormatUint(tokenData.ID, 10), @@ -136,8 +119,9 @@ func (e *Router) imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) return } - r.Body = http.MaxBytesReader(w, r.Body, FILES_SIZE_LIMIT) + r.Body = http.MaxBytesReader(w, r.Body, e.cfg.FileSizeLimit) defer r.Body.Close() + err = r.ParseMultipartForm(1e6) // ~1Mb if err == http.ErrNotMultipart || err == http.ErrMissingBoundary { ResponseWithError(w, http.StatusUnsupportedMediaType, err) @@ -166,7 +150,7 @@ func (e *Router) imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) key := prefix + fileHeader.Filename log.Printf("Uploading image... %v", key) go func() { //TODO: mime type from header - if err := e.services.S3.Upload(file, key, "image/jpeg", false); err != nil { + if err := e.services.Storage.Upload(file, key, "image/jpeg", false); err != nil { log.Printf("Upload ios screen error. %v", err) } }() diff --git a/backend/internal/router/handlers-web.go b/backend/internal/router/handlers-web.go index b0a3a9b94..d11a9b8ea 100644 --- a/backend/internal/router/handlers-web.go +++ b/backend/internal/router/handlers-web.go @@ -39,7 +39,7 @@ func (e *Router) startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) return } - p, err := e.services.Pgconn.GetProjectByKey(*req.ProjectKey) + p, err := e.services.Database.GetProjectByKey(*req.ProjectKey) if err != nil { if postgres.IsNoRowsErr(err) { ResponseWithError(w, http.StatusNotFound, errors.New("Project doesn't exist or capture limit has been reached")) @@ -119,33 +119,7 @@ func (e *Router) pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) // Process each message in request data err = ReadBatchReader(body, func(msg Message) { - switch m := msg.(type) { - case *SetNodeAttributeURLBased: - if m.Name == "src" || m.Name == "href" { - msg = &SetNodeAttribute{ - ID: m.ID, - Name: m.Name, - Value: e.handleURL(sessionData.ID, m.BaseURL, m.Value), - } - } else if m.Name == "style" { - msg = &SetNodeAttribute{ - ID: m.ID, - Name: m.Name, - Value: e.handleCSS(sessionData.ID, m.BaseURL, m.Value), - } - } - case *SetCSSDataURLBased: - msg = &SetCSSData{ - ID: m.ID, - Data: e.handleCSS(sessionData.ID, m.BaseURL, m.Data), - } - case *CSSInsertRuleURLBased: - msg = &CSSInsertRule{ - ID: m.ID, - Index: m.Index, - Rule: e.handleCSS(sessionData.ID, m.BaseURL, m.Rule), - } - } + msg = e.services.Assets.ParseAssets(sessionData.ID, msg) handledMessages.Write(msg.Encode()) }) if err != nil { @@ -188,7 +162,7 @@ func (e *Router) notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { return } country := e.services.GeoIP.ExtractISOCodeFromHTTPRequest(r) - err := e.services.Pgconn.InsertUnstartedSession(postgres.UnstartedSession{ + err := e.services.Database.InsertUnstartedSession(postgres.UnstartedSession{ ProjectKey: *req.ProjectKey, TrackerVersion: req.TrackerVersion, DoNotTrack: req.DoNotTrack, diff --git a/backend/internal/router/handlers.go b/backend/internal/router/handlers.go index e6e4c1d26..34a7a990d 100644 --- a/backend/internal/router/handlers.go +++ b/backend/internal/router/handlers.go @@ -11,15 +11,17 @@ import ( func (e *Router) pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64, topicName string) { body := http.MaxBytesReader(w, r.Body, e.cfg.BeaconSizeLimit) defer body.Close() + var reader io.ReadCloser var err error + switch r.Header.Get("Content-Encoding") { case "gzip": log.Println("Gzip", reader) reader, err = gzip.NewReader(body) if err != nil { - ResponseWithError(w, http.StatusInternalServerError, err) // TODO: stage-dependent responce + ResponseWithError(w, http.StatusInternalServerError, err) // TODO: stage-dependent response return } log.Println("Gzip reader init", reader) diff --git a/backend/internal/router/model.go b/backend/internal/router/model.go index 339593d98..b39c49688 100644 --- a/backend/internal/router/model.go +++ b/backend/internal/router/model.go @@ -28,3 +28,22 @@ type NotStartedRequest struct { TrackerVersion string `json:"trackerVersion"` DoNotTrack bool `json:"DoNotTrack"` } + +type StartIOSSessionRequest struct { + Token string `json:"token"` + ProjectKey *string `json:"projectKey"` + TrackerVersion string `json:"trackerVersion"` + RevID string `json:"revID"` + UserUUID *string `json:"userUUID"` + UserOSVersion string `json:"userOSVersion"` + UserDevice string `json:"userDevice"` + Timestamp uint64 `json:"timestamp"` +} + +type StartIOSSessionResponse struct { + Token string `json:"token"` + ImagesHashList []string `json:"imagesHashList"` + UserUUID string `json:"userUUID"` + BeaconSizeLimit int64 `json:"beaconSizeLimit"` + SessionID string `json:"sessionID"` +} diff --git a/backend/internal/services/services.go b/backend/internal/services/services.go index 62569630e..5b84e1dfb 100644 --- a/backend/internal/services/services.go +++ b/backend/internal/services/services.go @@ -1,6 +1,7 @@ package services import ( + "openreplay/backend/internal/assetscache" "openreplay/backend/internal/config" "openreplay/backend/internal/geoip" "openreplay/backend/internal/uaparser" @@ -13,22 +14,23 @@ import ( ) type ServicesBuilder struct { - Pgconn *cache.PGCache + Database *cache.PGCache Producer types.Producer - Rewriter *assets.Rewriter + Assets *assetscache.AssetsCache Flaker *flakeid.Flaker UaParser *uaparser.UAParser GeoIP *geoip.GeoIP Tokenizer *token.Tokenizer - S3 *storage.S3 + Storage *storage.S3 } func New(cfg *config.Config, producer types.Producer, pgconn *cache.PGCache) *ServicesBuilder { + rewriter := assets.NewRewriter(cfg.AssetsOrigin) return &ServicesBuilder{ - Pgconn: pgconn, + Database: pgconn, Producer: producer, - Rewriter: assets.NewRewriter(cfg.AssetsOrigin), - S3: storage.NewS3(cfg.AWSRegion, cfg.S3BucketIOSImages), + Assets: assetscache.New(cfg, rewriter, producer), + Storage: storage.NewS3(cfg.AWSRegion, cfg.S3BucketIOSImages), Tokenizer: token.NewTokenizer(cfg.TokenSecret), UaParser: uaparser.NewUAParser(cfg.UAParserFile), GeoIP: geoip.NewGeoIP(cfg.MaxMinDBFile), From d02ecba354ce294f230e1f35c8ff1d32b01e91c7 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 2 May 2022 17:38:53 +0200 Subject: [PATCH 10/18] Added missed return statements --- backend/internal/router/handlers-ios.go | 11 +++++++++++ backend/internal/router/handlers-web.go | 3 +++ 2 files changed, 14 insertions(+) diff --git a/backend/internal/router/handlers-ios.go b/backend/internal/router/handlers-ios.go index 426de7c05..50f92a6ad 100644 --- a/backend/internal/router/handlers-ios.go +++ b/backend/internal/router/handlers-ios.go @@ -20,6 +20,10 @@ func (e *Router) startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) startTime := time.Now() req := &StartIOSSessionRequest{} + if r.Body == nil { + ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + return + } body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() @@ -119,19 +123,26 @@ func (e *Router) imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) return } + if r.Body == nil { + ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + return + } r.Body = http.MaxBytesReader(w, r.Body, e.cfg.FileSizeLimit) defer r.Body.Close() err = r.ParseMultipartForm(1e6) // ~1Mb if err == http.ErrNotMultipart || err == http.ErrMissingBoundary { ResponseWithError(w, http.StatusUnsupportedMediaType, err) + return // } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB } else if err != nil { ResponseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging + return } if r.MultipartForm == nil { ResponseWithError(w, http.StatusInternalServerError, errors.New("Multipart not parsed")) + return } if len(r.MultipartForm.Value["projectKey"]) == 0 { diff --git a/backend/internal/router/handlers-web.go b/backend/internal/router/handlers-web.go index d11a9b8ea..fc7c6421d 100644 --- a/backend/internal/router/handlers-web.go +++ b/backend/internal/router/handlers-web.go @@ -22,6 +22,7 @@ func (e *Router) startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) // Check request body if r.Body == nil { ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + return } body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() @@ -111,6 +112,7 @@ func (e *Router) pushMessagesHandlerWeb(w http.ResponseWriter, r *http.Request) // Check request body if r.Body == nil { ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + return } body := http.MaxBytesReader(w, r.Body, e.cfg.BeaconSizeLimit) defer body.Close() @@ -140,6 +142,7 @@ func (e *Router) notStartedHandlerWeb(w http.ResponseWriter, r *http.Request) { // Check request body if r.Body == nil { ResponseWithError(w, http.StatusBadRequest, errors.New("request body is empty")) + return } body := http.MaxBytesReader(w, r.Body, e.cfg.JsonSizeLimit) defer body.Close() From 18d18164b3fc6f3a26e6b767796d3a3fcfe07bd3 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 3 May 2022 10:42:24 +0200 Subject: [PATCH 11/18] Added temporary hack for http service building --- backend/build.sh | 23 +++++++++--- backend/cmd/Dockerfile | 60 ++++++++++++++++++++++++++++++++ backend/services/http/build_hack | 0 3 files changed, 79 insertions(+), 4 deletions(-) mode change 100644 => 100755 backend/build.sh create mode 100644 backend/cmd/Dockerfile create mode 100644 backend/services/http/build_hack diff --git a/backend/build.sh b/backend/build.sh old mode 100644 new mode 100755 index c3c40dd33..8c1714d55 --- a/backend/build.sh +++ b/backend/build.sh @@ -33,12 +33,27 @@ function build_api(){ echo "build completed for http" return } + cwd=$PWD for image in $(ls services); do - docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image . - [[ $PUSH_IMAGE -eq 1 ]] && { - docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1} - } + case "$image" in + http) + echo build http + echo "$PWD" + cd cmd + docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image -f ../backend . + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1} + } + cd ../ + ;; + *) + docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image . + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1} + } + ;; + esac echo "::set-output name=image::${DOCKER_REPO:-'local'}/$image:${git_sha1}" done echo "backend build completed" diff --git a/backend/cmd/Dockerfile b/backend/cmd/Dockerfile new file mode 100644 index 000000000..e2f611afa --- /dev/null +++ b/backend/cmd/Dockerfile @@ -0,0 +1,60 @@ +FROM golang:1.18-alpine3.15 AS prepare + +RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash + +WORKDIR /root + +COPY go.mod . +COPY go.sum . +RUN go mod download + + +FROM prepare AS build +COPY pkg pkg +COPY services services +COPY internal internal +COPY cmd cmd + +ARG SERVICE_NAME +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o service -tags musl openreplay/backend/cmd/$SERVICE_NAME + +FROM alpine +RUN apk add --no-cache ca-certificates + +ENV TZ=UTC \ + FS_ULIMIT=1000 \ + FS_DIR=/mnt/efs \ + MAXMINDDB_FILE=/root/geoip.mmdb \ + UAPARSER_FILE=/root/regexes.yaml \ + HTTP_PORT=80 \ + BEACON_SIZE_LIMIT=7000000 \ + KAFKA_USE_SSL=true \ + KAFKA_MAX_POLL_INTERVAL_MS=400000 \ + REDIS_STREAMS_MAX_LEN=3000 \ + TOPIC_RAW_WEB=raw \ + TOPIC_RAW_IOS=raw-ios \ + TOPIC_CACHE=cache \ + TOPIC_ANALYTICS=analytics \ + TOPIC_TRIGGER=trigger \ + GROUP_SINK=sink \ + GROUP_STORAGE=storage \ + GROUP_DB=db \ + GROUP_ENDER=ender \ + GROUP_CACHE=cache \ + AWS_REGION_WEB=eu-central-1 \ + AWS_REGION_IOS=eu-west-1 \ + AWS_REGION_ASSETS=eu-central-1 \ + CACHE_ASSETS=true \ + ASSETS_SIZE_LIMIT=6291456 \ + FS_CLEAN_HRS=72 \ + LOG_QUEUE_STATS_INTERVAL_SEC=60 + + +ARG SERVICE_NAME +RUN if [ "$SERVICE_NAME" = "http" ]; then \ + wget https://raw.githubusercontent.com/ua-parser/uap-core/master/regexes.yaml -O "$UAPARSER_FILE" &&\ + wget https://static.openreplay.com/geoip/GeoLite2-Country.mmdb -O "$MAXMINDDB_FILE"; fi + + +COPY --from=build /root/service /root/service +ENTRYPOINT /root/service diff --git a/backend/services/http/build_hack b/backend/services/http/build_hack new file mode 100644 index 000000000..e69de29bb From b2456e9ac69d10bca4f3b920b115466839df3513 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 3 May 2022 12:33:43 +0200 Subject: [PATCH 12/18] Removed debug lines from build.sh --- backend/build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/build.sh b/backend/build.sh index 8c1714d55..fa9d6ce8e 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -33,13 +33,11 @@ function build_api(){ echo "build completed for http" return } - cwd=$PWD for image in $(ls services); do case "$image" in http) echo build http - echo "$PWD" cd cmd docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image -f ../backend . [[ $PUSH_IMAGE -eq 1 ]] && { From 414fbee962be31eeb3895e25a2be285fed8f1ebb Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 3 May 2022 13:55:56 +0200 Subject: [PATCH 13/18] Fixed build.sh file --- backend/build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/build.sh b/backend/build.sh index fa9d6ce8e..5a52b1063 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -38,12 +38,10 @@ function build_api(){ case "$image" in http) echo build http - cd cmd - docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image -f ../backend . + docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image -f ./cmd/Dockerfile . [[ $PUSH_IMAGE -eq 1 ]] && { docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1} } - cd ../ ;; *) docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image . From 42e6a63e44ac9aeff3728830b0f9657f624111b0 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 3 May 2022 21:43:16 +0200 Subject: [PATCH 14/18] docs(vagrant): create user account comment Signed-off-by: rjshrjndrn --- scripts/vagrant/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/vagrant/README.md b/scripts/vagrant/README.md index ffe132c73..5262930ba 100644 --- a/scripts/vagrant/README.md +++ b/scripts/vagrant/README.md @@ -38,8 +38,8 @@ Use the following instructions if you’re running Windows 10 or Windows 8: openreplay.local Select File > Save to save your changes. -**Open browser** -http://openreplay.local +**Open the below URL and create an account** +http://openreplay.local/signup ``` ### To start developing From 82084c9717be5028b52c56fcaae3bdc07d18a23a Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 4 May 2022 11:23:38 +0200 Subject: [PATCH 15/18] fix (backend): build.sh build_service incapsulate --- backend/build.sh | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/backend/build.sh b/backend/build.sh index 5a52b1063..4b55b787e 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -18,6 +18,27 @@ check_prereq() { return } + +function build_service() { + image="$1" + echo "BUILDING $image" + case "$image" in + http) + echo build http + docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image -f ./cmd/Dockerfile . + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1} + } + ;; + *) + docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image . + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1} + } + ;; + esac +} + function build_api(){ # Copy enterprise code [[ $1 == "ee" ]] && { @@ -25,31 +46,12 @@ function build_api(){ ee="true" } [[ $2 != "" ]] && { - image="$2" - docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image . - [[ $PUSH_IMAGE -eq 1 ]] && { - docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1} - } - echo "build completed for http" + build_service $2 return } for image in $(ls services); do - case "$image" in - http) - echo build http - docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image -f ./cmd/Dockerfile . - [[ $PUSH_IMAGE -eq 1 ]] && { - docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1} - } - ;; - *) - docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image . - [[ $PUSH_IMAGE -eq 1 ]] && { - docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1} - } - ;; - esac + build_service $image echo "::set-output name=image::${DOCKER_REPO:-'local'}/$image:${git_sha1}" done echo "backend build completed" From b72a332cd097ce567aa3776a0b139c3ff9833480 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 4 May 2022 11:38:23 +0200 Subject: [PATCH 16/18] chore(build): returning from function Signed-off-by: rjshrjndrn --- backend/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/build.sh b/backend/build.sh index 4b55b787e..8e3bea86a 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -37,6 +37,7 @@ function build_service() { } ;; esac + return } function build_api(){ From 767fa31026499e24384a96cf72ad79ec57edb9dc Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 4 May 2022 12:09:46 +0200 Subject: [PATCH 17/18] chore(actions): include cmd dir for build Signed-off-by: rjshrjndrn --- .github/workflows/workers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workers.yaml b/.github/workflows/workers.yaml index 49fd0948b..d412fd756 100644 --- a/.github/workflows/workers.yaml +++ b/.github/workflows/workers.yaml @@ -49,7 +49,7 @@ jobs: # { - git diff --name-only HEAD HEAD~1 | grep backend/services | grep -vE ^ee/ | cut -d '/' -f3 + git diff --name-only HEAD HEAD~1 | grep backend/cmd | grep backend/services | grep -vE ^ee/ | cut -d '/' -f3 git diff --name-only HEAD HEAD~1 | grep backend/pkg | grep -vE ^ee/ | cut -d '/' -f3 | uniq | while read -r pkg_name ; do grep -rl "pkg/$pkg_name" backend/services | cut -d '/' -f3 From a4278aec231714ca82c1f70d88e950a2b353918e Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Wed, 4 May 2022 12:14:26 +0200 Subject: [PATCH 18/18] [http] removed extra log in main.go Signed-off-by: rjshrjndrn --- .github/workflows/workers.yaml | 2 +- backend/cmd/http/main.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/workers.yaml b/.github/workflows/workers.yaml index d412fd756..7ce78ad6f 100644 --- a/.github/workflows/workers.yaml +++ b/.github/workflows/workers.yaml @@ -49,7 +49,7 @@ jobs: # { - git diff --name-only HEAD HEAD~1 | grep backend/cmd | grep backend/services | grep -vE ^ee/ | cut -d '/' -f3 + git diff --name-only HEAD HEAD~1 | grep -E "backend/cmd|backend/services" | grep -vE ^ee/ | cut -d '/' -f3 git diff --name-only HEAD HEAD~1 | grep backend/pkg | grep -vE ^ee/ | cut -d '/' -f3 | uniq | while read -r pkg_name ; do grep -rl "pkg/$pkg_name" backend/services | cut -d '/' -f3 diff --git a/backend/cmd/http/main.go b/backend/cmd/http/main.go index 41f739277..541baab62 100644 --- a/backend/cmd/http/main.go +++ b/backend/cmd/http/main.go @@ -48,8 +48,7 @@ func main() { // Run server go func() { if err := server.Start(); err != nil { - log.Printf("Server error: %v\n", err) - log.Fatal("Server error") + log.Fatalf("Server error: %v\n", err) } }() log.Printf("Server successfully started on port %v\n", cfg.HTTPPort)