jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(backend/assets): increased number of workers and retries, new retry tactic within/without headers

Browse source
This commit is contained in:
Alexander Zavorotynskiy 2022-10-19 13:34:19 +02:00
parent 469d9f74e8
commit 7adfba1603
1 changed files with 11 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
backend/internal/assets/cacher/cacher.go
View file

@ -65,7 +65,7 @@ func NewCacher(cfg *config.Config, metrics *monitoring.Metrics) *cacher {
downloadedAssets: downloadedAssets,
requestHeaders: cfg.AssetsRequestHeaders,
}
c.workers = NewPool(32, c.CacheFile)
c.workers = NewPool(64, c.CacheFile)
return c
}
@ -75,9 +75,10 @@ func (c *cacher) CacheFile(task *Task) {
func (c *cacher) cacheURL(t *Task) {
t.retries--
//requestURL sessionID uint64, depth byte, urlContext, cachePath string) {
req, _ := http.NewRequest("GET", t.requestURL, nil)
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:31.0) Gecko/20100101 Firefox/31.0")
if t.retries%2 == 0 {
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:31.0) Gecko/20100101 Firefox/31.0")
}
for k, v := range c.requestHeaders {
req.Header.Set(k, v)
}
@ -138,7 +139,7 @@ func (c *cacher) cacheURL(t *Task) {
depth: t.depth - 1,
urlContext: t.urlContext + "\n -> " + fullURL,
isJS: false,
retries: 5,
retries: setRetries(),
})
}
}
@ -182,7 +183,7 @@ func (c *cacher) CacheJSFile(sourceURL string) {
depth: 0,
urlContext: sourceURL,
isJS: true,
retries: 5,
retries: setRetries(),
})
}
@ -193,7 +194,7 @@ func (c *cacher) CacheURL(sessionID uint64, fullURL string) {
depth: MAX_CACHE_DEPTH,
urlContext: fullURL,
isJS: false,
retries: 5,
retries: setRetries(),
})
}
@ -204,3 +205,7 @@ func (c *cacher) UpdateTimeouts() {
func (c *cacher) Stop() {
c.workers.Stop()
}
func setRetries() int {
return 10
}