From 2401449a4ae38ddf1094068de67e84ea45be3834 Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 7 May 2024 18:13:25 +0200 Subject: [PATCH] feat(assest): use original content-encoding for uploaded to s3/azure assets (#2178) --- backend/internal/assets/cacher/cacher.go | 3 ++- backend/internal/storage/storage.go | 6 +++--- backend/internal/videostorage/service.go | 2 +- backend/pkg/objectstorage/objectstorage.go | 2 +- backend/pkg/objectstorage/s3/s3.go | 13 ++++++++----- ee/backend/pkg/objectstorage/azure/azure.go | 15 ++++++++++----- 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/backend/internal/assets/cacher/cacher.go b/backend/internal/assets/cacher/cacher.go index cd8476062..d0a940046 100644 --- a/backend/internal/assets/cacher/cacher.go +++ b/backend/internal/assets/cacher/cacher.go @@ -142,6 +142,7 @@ func (c *cacher) cacheURL(t *Task) { if contentType == "" { contentType = mime.TypeByExtension(filepath.Ext(res.Request.URL.Path)) } + contentEncoding := res.Header.Get("Content-Encoding") // Skip html file (usually it's a CDN mock for 404 error) if strings.HasPrefix(contentType, "text/html") { @@ -158,7 +159,7 @@ func (c *cacher) cacheURL(t *Task) { // TODO: implement in streams start = time.Now() - err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, objectstorage.NoCompression) + err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, contentEncoding, objectstorage.NoCompression) if err != nil { metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), true) c.Errors <- errors.Wrap(err, t.urlContext) diff --git a/backend/internal/storage/storage.go b/backend/internal/storage/storage.go index 1222bd8f9..aa26cb9aa 100644 --- a/backend/internal/storage/storage.go +++ b/backend/internal/storage/storage.go @@ -373,7 +373,7 @@ func (s *Storage) uploadSession(task *Task) { metrics.RecordSessionCompressionRatio(task.domsRawSize/float64(task.doms.Len()), DOM.String()) // Upload session to s3 start := time.Now() - if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", task.compression); err != nil { + if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", "", task.compression); err != nil { log.Fatalf("Storage: start upload failed. %s", err) } uploadDoms = time.Now().Sub(start).Milliseconds() @@ -386,7 +386,7 @@ func (s *Storage) uploadSession(task *Task) { metrics.RecordSessionCompressionRatio(task.domeRawSize/float64(task.dome.Len()), DOM.String()) // Upload session to s3 start := time.Now() - if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", task.compression); err != nil { + if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", "", task.compression); err != nil { log.Fatalf("Storage: start upload failed. %s", err) } uploadDome = time.Now().Sub(start).Milliseconds() @@ -399,7 +399,7 @@ func (s *Storage) uploadSession(task *Task) { metrics.RecordSessionCompressionRatio(task.devRawSize/float64(task.dev.Len()), DEV.String()) // Upload session to s3 start := time.Now() - if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", task.compression); err != nil { + if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", "", task.compression); err != nil { log.Fatalf("Storage: start upload failed. %s", err) } uploadDev = time.Now().Sub(start).Milliseconds() diff --git a/backend/internal/videostorage/service.go b/backend/internal/videostorage/service.go index e5e6ddd11..c83bebb9d 100644 --- a/backend/internal/videostorage/service.go +++ b/backend/internal/videostorage/service.go @@ -125,7 +125,7 @@ func (v *VideoStorage) sendToS3(task *Task) { } else { key += "/replay.mp4" } - if err := v.objStorage.Upload(bytes.NewReader(video), key, "video/mp4", objectstorage.NoCompression); err != nil { + if err := v.objStorage.Upload(bytes.NewReader(video), key, "video/mp4", "", objectstorage.NoCompression); err != nil { log.Fatalf("Storage: start upload video replay failed. %s", err) } log.Printf("Video file (size: %d) uploaded successfully in %v", len(video), time.Since(start)) diff --git a/backend/pkg/objectstorage/objectstorage.go b/backend/pkg/objectstorage/objectstorage.go index f97e6408f..49b06332b 100644 --- a/backend/pkg/objectstorage/objectstorage.go +++ b/backend/pkg/objectstorage/objectstorage.go @@ -15,7 +15,7 @@ const ( ) type ObjectStorage interface { - Upload(reader io.Reader, key string, contentType string, compression CompressionType) error + Upload(reader io.Reader, key string, contentType, contentEncoding string, compression CompressionType) error Get(key string) (io.ReadCloser, error) Exists(key string) bool GetCreationTime(key string) *time.Time diff --git a/backend/pkg/objectstorage/s3/s3.go b/backend/pkg/objectstorage/s3/s3.go index 76b5c9060..e0d86ad43 100644 --- a/backend/pkg/objectstorage/s3/s3.go +++ b/backend/pkg/objectstorage/s3/s3.go @@ -63,19 +63,22 @@ func NewS3(cfg *objConfig.ObjectsConfig) (objectstorage.ObjectStorage, error) { }, nil } -func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, compression objectstorage.CompressionType) error { +func (s *storageImpl) Upload(reader io.Reader, key string, contentType, contentEncoding string, compression objectstorage.CompressionType) error { cacheControl := "max-age=2628000, immutable, private" - var contentEncoding *string + var encoding *string switch compression { case objectstorage.Gzip: encodeStr := "gzip" - contentEncoding = &encodeStr + encoding = &encodeStr case objectstorage.Brotli: encodeStr := "br" - contentEncoding = &encodeStr + encoding = &encodeStr case objectstorage.Zstd: // Have to ignore contentEncoding for Zstd (otherwise will be an error in browser) } + if contentEncoding != "" { + encoding = &contentEncoding + } _, err := s.uploader.Upload(&s3manager.UploadInput{ Body: reader, @@ -83,7 +86,7 @@ func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, c Key: &key, ContentType: &contentType, CacheControl: &cacheControl, - ContentEncoding: contentEncoding, + ContentEncoding: encoding, Tagging: s.fileTag, }) return err diff --git a/ee/backend/pkg/objectstorage/azure/azure.go b/ee/backend/pkg/objectstorage/azure/azure.go index 642df1c97..2b24dcf51 100644 --- a/ee/backend/pkg/objectstorage/azure/azure.go +++ b/ee/backend/pkg/objectstorage/azure/azure.go @@ -50,16 +50,21 @@ func NewStorage(cfg *config.ObjectsConfig) (objectstorage.ObjectStorage, error) }, nil } -func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, compression objectstorage.CompressionType) error { +func (s *storageImpl) Upload(reader io.Reader, key string, contentType, contentEncoding string, compression objectstorage.CompressionType) error { cacheControl := "max-age=2628000, immutable, private" - var contentEncoding *string + var encoding *string switch compression { case objectstorage.Gzip: gzipStr := "gzip" - contentEncoding = &gzipStr + encoding = &gzipStr case objectstorage.Brotli: gzipStr := "br" - contentEncoding = &gzipStr + encoding = &gzipStr + case objectstorage.Zstd: + // Have to ignore contentEncoding for Zstd (otherwise will be an error in browser) + } + if contentEncoding != "" { + encoding = &contentEncoding } // Remove leading slash to avoid empty folder creation if strings.HasPrefix(key, "/") { @@ -68,7 +73,7 @@ func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, c _, err := s.client.UploadStream(context.Background(), s.container, key, reader, &azblob.UploadStreamOptions{ HTTPHeaders: &blob.HTTPHeaders{ BlobCacheControl: &cacheControl, - BlobContentEncoding: contentEncoding, + BlobContentEncoding: encoding, BlobContentType: &contentType, }, Tags: s.tags,