jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(assest): use original content-encoding for uploaded to s3 assets

Browse source
This commit is contained in:
Alexander 2024-01-24 12:00:00 +01:00
parent 5e116856c9
commit 3ac2e5d6ef
4 changed files with 14 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
backend/internal/assets/cacher/cacher.go
View file

@ -142,6 +142,7 @@ func (c *cacher) cacheURL(t *Task) {
if contentType == "" {
contentType = mime.TypeByExtension(filepath.Ext(res.Request.URL.Path))
}
contentEncoding := res.Header.Get("Content-Encoding")
// Skip html file (usually it's a CDN mock for 404 error)
if strings.HasPrefix(contentType, "text/html") {
@ -158,7 +159,7 @@ func (c *cacher) cacheURL(t *Task) {
// TODO: implement in streams
start = time.Now()
err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, objectstorage.NoCompression)
err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, contentEncoding, objectstorage.NoCompression)
if err != nil {
metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), true)
c.Errors <- errors.Wrap(err, t.urlContext)

6
backend/internal/storage/storage.go
View file

@ -387,7 +387,7 @@ func (s *Storage) uploadSession(task *Task) {
metrics.RecordSessionCompressionRatio(task.domsRawSize/float64(task.doms.Len()), DOM.String())
// Upload session to s3
start := time.Now()
if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", task.compression); err != nil {
if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", "", task.compression); err != nil {
log.Fatalf("Storage: start upload failed. %s", err)
}
uploadDoms = time.Now().Sub(start).Milliseconds()
@ -400,7 +400,7 @@ func (s *Storage) uploadSession(task *Task) {
metrics.RecordSessionCompressionRatio(task.domeRawSize/float64(task.dome.Len()), DOM.String())
// Upload session to s3
start := time.Now()
if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", task.compression); err != nil {
if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", "", task.compression); err != nil {
log.Fatalf("Storage: start upload failed. %s", err)
}
uploadDome = time.Now().Sub(start).Milliseconds()
@ -413,7 +413,7 @@ func (s *Storage) uploadSession(task *Task) {
metrics.RecordSessionCompressionRatio(task.devRawSize/float64(task.dev.Len()), DEV.String())
// Upload session to s3
start := time.Now()
if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", task.compression); err != nil {
if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", "", task.compression); err != nil {
log.Fatalf("Storage: start upload failed. %s", err)
}
uploadDev = time.Now().Sub(start).Milliseconds()

2
backend/pkg/objectstorage/objectstorage.go
View file

@ -14,7 +14,7 @@ const (
)
type ObjectStorage interface {
Upload(reader io.Reader, key string, contentType string, compression CompressionType) error
Upload(reader io.Reader, key string, contentType, contentEncoding string, compression CompressionType) error
Get(key string) (io.ReadCloser, error)
Exists(key string) bool
GetCreationTime(key string) *time.Time

13
backend/pkg/objectstorage/s3/s3.go
View file

@ -67,16 +67,19 @@ func (s *storageImpl) tagging() *string {
return &s.fileTag
}
func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, compression objectstorage.CompressionType) error {
func (s *storageImpl) Upload(reader io.Reader, key string, contentType, contentEncoding string, compression objectstorage.CompressionType) error {
cacheControl := "max-age=2628000, immutable, private"
var contentEncoding *string
var encoding *string
switch compression {
case objectstorage.Gzip:
gzipStr := "gzip"
contentEncoding = &gzipStr
encoding = &gzipStr
case objectstorage.Brotli:
gzipStr := "br"
contentEncoding = &gzipStr
encoding = &gzipStr
}
if contentEncoding != "" {
encoding = &contentEncoding
}
_, err := s.uploader.Upload(&s3manager.UploadInput{
@ -85,7 +88,7 @@ func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, c
Key: &key,
ContentType: &contentType,
CacheControl: &cacheControl,
ContentEncoding: contentEncoding,
ContentEncoding: encoding,
Tagging: s.tagging(),
})
return err