diff --git a/backend/cmd/assets/main.go b/backend/cmd/assets/main.go index 5627d0be8..044fb848c 100644 --- a/backend/cmd/assets/main.go +++ b/backend/cmd/assets/main.go @@ -29,6 +29,7 @@ func main() { cfg.S3BucketAssets, cfg.AssetsOrigin, cfg.AssetsSizeLimit, + cfg.AssetsRequestHeaders, metrics, ) diff --git a/backend/internal/assets/cacher/cacher.go b/backend/internal/assets/cacher/cacher.go index fe8cc7ef8..365ecbfbe 100644 --- a/backend/internal/assets/cacher/cacher.go +++ b/backend/internal/assets/cacher/cacher.go @@ -31,9 +31,17 @@ type cacher struct { Errors chan error sizeLimit int downloadedAssets syncfloat64.Counter + requestHeaders map[string]string } -func NewCacher(region string, bucket string, origin string, sizeLimit int, metrics *monitoring.Metrics) *cacher { +func NewCacher( + region string, + bucket string, + origin string, + sizeLimit int, + requestHeaders map[string]string, + metrics *monitoring.Metrics, +) *cacher { rewriter := assets.NewRewriter(origin) if metrics == nil { log.Fatalf("metrics are empty") @@ -55,6 +63,7 @@ func NewCacher(region string, bucket string, origin string, sizeLimit int, metri Errors: make(chan error), sizeLimit: sizeLimit, downloadedAssets: downloadedAssets, + requestHeaders: requestHeaders, } } @@ -74,15 +83,17 @@ func (c *cacher) cacheURL(requestURL string, sessionID uint64, depth byte, urlCo } req, _ := http.NewRequest("GET", requestURL, nil) - req.Header.Set("Cookie", "ABv=3;") // Hack for rueducommerce req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:31.0) Gecko/20100101 Firefox/31.0") + for k, v := range c.requestHeaders { + req.Header.Set(k, v) + } res, err := c.httpClient.Do(req) if err != nil { c.Errors <- errors.Wrap(err, urlContext) return } defer res.Body.Close() - if res.StatusCode != 200 { + if res.StatusCode >= 400 { // TODO: retry c.Errors <- errors.Wrap(fmt.Errorf("Status code is %v, ", res.StatusCode), urlContext) return diff --git a/backend/internal/config/assets/config.go b/backend/internal/config/assets/config.go index 13c49bf03..1dfc8a4a8 100644 --- a/backend/internal/config/assets/config.go +++ b/backend/internal/config/assets/config.go @@ -7,12 +7,13 @@ import ( type Config struct { common.Config - GroupCache string `env:"GROUP_CACHE,required"` - TopicCache string `env:"TOPIC_CACHE,required"` - AWSRegion string `env:"AWS_REGION,required"` - S3BucketAssets string `env:"S3_BUCKET_ASSETS,required"` - AssetsOrigin string `env:"ASSETS_ORIGIN,required"` - AssetsSizeLimit int `env:"ASSETS_SIZE_LIMIT,required"` + GroupCache string `env:"GROUP_CACHE,required"` + TopicCache string `env:"TOPIC_CACHE,required"` + AWSRegion string `env:"AWS_REGION,required"` + S3BucketAssets string `env:"S3_BUCKET_ASSETS,required"` + AssetsOrigin string `env:"ASSETS_ORIGIN,required"` + AssetsSizeLimit int `env:"ASSETS_SIZE_LIMIT,required"` + AssetsRequestHeaders map[string]string `env:"ASSETS_REQUEST_HEADERS"` } func New() *Config { diff --git a/backend/internal/config/configurator/configurator.go b/backend/internal/config/configurator/configurator.go index a2cab1029..cfd7732ff 100644 --- a/backend/internal/config/configurator/configurator.go +++ b/backend/internal/config/configurator/configurator.go @@ -2,6 +2,7 @@ package configurator import ( "context" + "encoding/json" "fmt" "io" "log" @@ -87,6 +88,12 @@ func parseFile(a interface{}, path string) { continue } val.Field(i).SetInt(int64(d)) + case "map[string]string": + var stringMap map[string]string + if json.Unmarshal([]byte(value), &stringMap); err != nil { + log.Printf("can't parse map[string]string value: %s", err) + } + val.Field(i).Set(reflect.ValueOf(stringMap)) default: log.Println("unknown config type: ", val.Type().Field(i).Type.String()) } diff --git a/backend/pkg/env/vars.go b/backend/pkg/env/vars.go index eb88b3c6b..1dbc346b0 100644 --- a/backend/pkg/env/vars.go +++ b/backend/pkg/env/vars.go @@ -1,6 +1,7 @@ package env import ( + "encoding/json" "log" "os" "strconv" @@ -56,3 +57,16 @@ func Bool(key string) bool { } return false } + +func StringMapOptional(key string) map[string]string { + v := StringOptional(key) + if v == "" { + return nil + } + var stringMap map[string]string + + if err := json.Unmarshal([]byte(v), &stringMap); err != nil { + log.Fatalln(key + ": wrong json format. Expected string map") + } + return stringMap +}