From 09adccf4bef07fe80f38ec44f4e2fb3b0d887f0e Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 13 Dec 2022 17:48:00 +0100 Subject: [PATCH] feat(backend): added env variable for enabling pprof (#880) --- backend/cmd/assets/main.go | 5 ++++- backend/cmd/db/main.go | 5 ++++- backend/cmd/ender/main.go | 7 ++++++- backend/cmd/heuristics/main.go | 5 ++++- backend/cmd/http/main.go | 5 ++++- backend/cmd/integrations/main.go | 5 ++++- backend/cmd/sink/main.go | 5 ++++- backend/cmd/storage/main.go | 5 ++++- backend/internal/config/assets/config.go | 1 + backend/internal/config/db/config.go | 1 + backend/internal/config/ender/config.go | 1 + backend/internal/config/heuristics/config.go | 1 + backend/internal/config/http/config.go | 1 + backend/internal/config/integrations/config.go | 1 + backend/internal/config/sink/config.go | 1 + backend/internal/config/storage/config.go | 1 + 16 files changed, 42 insertions(+), 8 deletions(-) diff --git a/backend/cmd/assets/main.go b/backend/cmd/assets/main.go index 5fdc85107..b41dedd87 100644 --- a/backend/cmd/assets/main.go +++ b/backend/cmd/assets/main.go @@ -3,6 +3,7 @@ package main import ( "context" "log" + "openreplay/backend/pkg/pprof" "os" "os/signal" "syscall" @@ -18,10 +19,12 @@ import ( func main() { metrics := monitoring.New("assets") - log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := config.New() + if cfg.UseProfiler { + pprof.StartProfilingServer() + } cacher := cacher.NewCacher(cfg, metrics) diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index 51fa99122..c8dd60796 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -4,6 +4,7 @@ import ( "errors" "log" types2 "openreplay/backend/pkg/db/types" + "openreplay/backend/pkg/pprof" "openreplay/backend/pkg/queue/types" "os" "os/signal" @@ -25,10 +26,12 @@ import ( func main() { metrics := monitoring.New("db") - log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := db.New() + if cfg.UseProfiler { + pprof.StartProfilingServer() + } // Init database pg := cache.NewPGCache( diff --git a/backend/cmd/ender/main.go b/backend/cmd/ender/main.go index 89354fe64..3190b07d4 100644 --- a/backend/cmd/ender/main.go +++ b/backend/cmd/ender/main.go @@ -3,6 +3,7 @@ package main import ( "log" "openreplay/backend/internal/storage" + "openreplay/backend/pkg/pprof" "os" "os/signal" "strings" @@ -21,9 +22,13 @@ import ( ) func main() { - log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) metrics := monitoring.New("ender") + log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) + cfg := ender.New() + if cfg.UseProfiler { + pprof.StartProfilingServer() + } pg := cache.NewPGCache(postgres.NewConn(cfg.Postgres, 0, 0, metrics), cfg.ProjectExpirationTimeoutMs) defer pg.Close() diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index 0a5c77ea5..fdbb6552e 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -2,6 +2,7 @@ package main import ( "log" + "openreplay/backend/pkg/pprof" "os" "os/signal" "syscall" @@ -20,8 +21,10 @@ import ( func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - // Load service configuration cfg := heuristics.New() + if cfg.UseProfiler { + pprof.StartProfilingServer() + } // HandlersFabric returns the list of message handlers we want to be applied to each incoming message. handlersFabric := func() []handlers.MessageProcessor { diff --git a/backend/cmd/http/main.go b/backend/cmd/http/main.go index 7012917e4..72dfab5ed 100644 --- a/backend/cmd/http/main.go +++ b/backend/cmd/http/main.go @@ -7,6 +7,7 @@ import ( "openreplay/backend/internal/http/server" "openreplay/backend/internal/http/services" "openreplay/backend/pkg/monitoring" + "openreplay/backend/pkg/pprof" "os" "os/signal" "syscall" @@ -18,10 +19,12 @@ import ( func main() { metrics := monitoring.New("http") - log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := http.New() + if cfg.UseProfiler { + pprof.StartProfilingServer() + } // Connect to queue producer := queue.NewProducer(cfg.MessageSizeLimit, true) diff --git a/backend/cmd/integrations/main.go b/backend/cmd/integrations/main.go index 4f5a30dcf..59d12b0c7 100644 --- a/backend/cmd/integrations/main.go +++ b/backend/cmd/integrations/main.go @@ -5,6 +5,7 @@ import ( config "openreplay/backend/internal/config/integrations" "openreplay/backend/internal/integrations/clientManager" "openreplay/backend/pkg/monitoring" + "openreplay/backend/pkg/pprof" "time" "os" @@ -19,10 +20,12 @@ import ( func main() { metrics := monitoring.New("integrations") - log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := config.New() + if cfg.UseProfiler { + pprof.StartProfilingServer() + } pg := postgres.NewConn(cfg.PostgresURI, 0, 0, metrics) defer pg.Close() diff --git a/backend/cmd/sink/main.go b/backend/cmd/sink/main.go index 03f11b200..0999e3936 100644 --- a/backend/cmd/sink/main.go +++ b/backend/cmd/sink/main.go @@ -3,6 +3,7 @@ package main import ( "context" "log" + "openreplay/backend/pkg/pprof" "os" "os/signal" "syscall" @@ -20,10 +21,12 @@ import ( func main() { metrics := monitoring.New("sink") - log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := sink.New() + if cfg.UseProfiler { + pprof.StartProfilingServer() + } if _, err := os.Stat(cfg.FsDir); os.IsNotExist(err) { log.Fatalf("%v doesn't exist. %v", cfg.FsDir, err) diff --git a/backend/cmd/storage/main.go b/backend/cmd/storage/main.go index 76c75ea5e..dcb1b53ed 100644 --- a/backend/cmd/storage/main.go +++ b/backend/cmd/storage/main.go @@ -2,6 +2,7 @@ package main import ( "log" + "openreplay/backend/pkg/pprof" "os" "os/signal" "syscall" @@ -18,10 +19,12 @@ import ( func main() { metrics := monitoring.New("storage") - log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := config.New() + if cfg.UseProfiler { + pprof.StartProfilingServer() + } s3 := s3storage.NewS3(cfg.S3Region, cfg.S3Bucket) srv, err := storage.New(cfg, s3, metrics) diff --git a/backend/internal/config/assets/config.go b/backend/internal/config/assets/config.go index 1dfc8a4a8..399ee84f4 100644 --- a/backend/internal/config/assets/config.go +++ b/backend/internal/config/assets/config.go @@ -14,6 +14,7 @@ type Config struct { AssetsOrigin string `env:"ASSETS_ORIGIN,required"` AssetsSizeLimit int `env:"ASSETS_SIZE_LIMIT,required"` AssetsRequestHeaders map[string]string `env:"ASSETS_REQUEST_HEADERS"` + UseProfiler bool `env:"PROFILER_ENABLED,default=false"` } func New() *Config { diff --git a/backend/internal/config/db/config.go b/backend/internal/config/db/config.go index 715d9ff8e..6ec25ab01 100644 --- a/backend/internal/config/db/config.go +++ b/backend/internal/config/db/config.go @@ -18,6 +18,7 @@ type Config struct { BatchQueueLimit int `env:"DB_BATCH_QUEUE_LIMIT,required"` BatchSizeLimit int `env:"DB_BATCH_SIZE_LIMIT,required"` UseQuickwit bool `env:"QUICKWIT_ENABLED,default=false"` + UseProfiler bool `env:"PROFILER_ENABLED,default=false"` } func New() *Config { diff --git a/backend/internal/config/ender/config.go b/backend/internal/config/ender/config.go index fb315acbe..cc963285f 100644 --- a/backend/internal/config/ender/config.go +++ b/backend/internal/config/ender/config.go @@ -15,6 +15,7 @@ type Config struct { ProducerTimeout int `env:"PRODUCER_TIMEOUT,default=2000"` PartitionsNumber int `env:"PARTITIONS_NUMBER,required"` UseEncryption bool `env:"USE_ENCRYPTION,default=false"` + UseProfiler bool `env:"PROFILER_ENABLED,default=false"` } func New() *Config { diff --git a/backend/internal/config/heuristics/config.go b/backend/internal/config/heuristics/config.go index fbe0eab81..6552944a3 100644 --- a/backend/internal/config/heuristics/config.go +++ b/backend/internal/config/heuristics/config.go @@ -13,6 +13,7 @@ type Config struct { TopicRawWeb string `env:"TOPIC_RAW_WEB,required"` TopicRawIOS string `env:"TOPIC_RAW_IOS,required"` ProducerTimeout int `env:"PRODUCER_TIMEOUT,default=2000"` + UseProfiler bool `env:"PROFILER_ENABLED,default=false"` } func New() *Config { diff --git a/backend/internal/config/http/config.go b/backend/internal/config/http/config.go index 3c30d3980..b8bd6cfbb 100644 --- a/backend/internal/config/http/config.go +++ b/backend/internal/config/http/config.go @@ -23,6 +23,7 @@ type Config struct { TokenSecret string `env:"TOKEN_SECRET,required"` UAParserFile string `env:"UAPARSER_FILE,required"` MaxMinDBFile string `env:"MAXMINDDB_FILE,required"` + UseProfiler bool `env:"PROFILER_ENABLED,default=false"` WorkerID uint16 } diff --git a/backend/internal/config/integrations/config.go b/backend/internal/config/integrations/config.go index c61377b8c..4713fd6b0 100644 --- a/backend/internal/config/integrations/config.go +++ b/backend/internal/config/integrations/config.go @@ -10,6 +10,7 @@ type Config struct { TopicAnalytics string `env:"TOPIC_ANALYTICS,required"` PostgresURI string `env:"POSTGRES_STRING,required"` TokenSecret string `env:"TOKEN_SECRET,required"` + UseProfiler bool `env:"PROFILER_ENABLED,default=false"` } func New() *Config { diff --git a/backend/internal/config/sink/config.go b/backend/internal/config/sink/config.go index 53e3517a4..c65d62605 100644 --- a/backend/internal/config/sink/config.go +++ b/backend/internal/config/sink/config.go @@ -21,6 +21,7 @@ type Config struct { ProducerCloseTimeout int `env:"PRODUCER_CLOSE_TIMEOUT,default=15000"` CacheThreshold int64 `env:"CACHE_THRESHOLD,default=5"` CacheExpiration int64 `env:"CACHE_EXPIRATION,default=120"` + UseProfiler bool `env:"PROFILER_ENABLED,default=false"` } func New() *Config { diff --git a/backend/internal/config/storage/config.go b/backend/internal/config/storage/config.go index 6083f0249..63c595f62 100644 --- a/backend/internal/config/storage/config.go +++ b/backend/internal/config/storage/config.go @@ -21,6 +21,7 @@ type Config struct { ProducerCloseTimeout int `env:"PRODUCER_CLOSE_TIMEOUT,default=15000"` UseFailover bool `env:"USE_FAILOVER,default=false"` MaxFileSize int64 `env:"MAX_FILE_SIZE,default=524288000"` + UseProfiler bool `env:"PROFILER_ENABLED,default=false"` } func New() *Config {