jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(assest): use original content-encoding for uploaded to s3/azure assets (#2178)

Browse source
This commit is contained in:
Alexander 2024-05-07 18:13:25 +02:00 • committed by GitHub
parent f68aff93d2
commit 2401449a4a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 25 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
backend/internal/assets/cacher/cacher.go
View file

@ -142,6 +142,7 @@ func (c *cacher) cacheURL(t *Task) {
if contentType == "" {
contentType = mime.TypeByExtension(filepath.Ext(res.Request.URL.Path))
}
contentEncoding := res.Header.Get("Content-Encoding")
// Skip html file (usually it's a CDN mock for 404 error)
if strings.HasPrefix(contentType, "text/html") {
@ -158,7 +159,7 @@ func (c *cacher) cacheURL(t *Task) {
// TODO: implement in streams
start = time.Now()
err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, objectstorage.NoCompression)
err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, contentEncoding, objectstorage.NoCompression)
if err != nil {
metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), true)
c.Errors <- errors.Wrap(err, t.urlContext)

6
backend/internal/storage/storage.go
View file

@ -373,7 +373,7 @@ func (s *Storage) uploadSession(task *Task) {
metrics.RecordSessionCompressionRatio(task.domsRawSize/float64(task.doms.Len()), DOM.String())
// Upload session to s3
start := time.Now()
if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", task.compression); err != nil {
if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", "", task.compression); err != nil {
log.Fatalf("Storage: start upload failed. %s", err)
}
uploadDoms = time.Now().Sub(start).Milliseconds()
@ -386,7 +386,7 @@ func (s *Storage) uploadSession(task *Task) {
metrics.RecordSessionCompressionRatio(task.domeRawSize/float64(task.dome.Len()), DOM.String())
// Upload session to s3
start := time.Now()
if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", task.compression); err != nil {
if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", "", task.compression); err != nil {
log.Fatalf("Storage: start upload failed. %s", err)
}
uploadDome = time.Now().Sub(start).Milliseconds()
@ -399,7 +399,7 @@ func (s *Storage) uploadSession(task *Task) {
metrics.RecordSessionCompressionRatio(task.devRawSize/float64(task.dev.Len()), DEV.String())
// Upload session to s3
start := time.Now()
if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", task.compression); err != nil {
if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", "", task.compression); err != nil {
log.Fatalf("Storage: start upload failed. %s", err)
}
uploadDev = time.Now().Sub(start).Milliseconds()

2
backend/internal/videostorage/service.go
View file

@ -125,7 +125,7 @@ func (v *VideoStorage) sendToS3(task *Task) {
} else {
key += "/replay.mp4"
}
if err := v.objStorage.Upload(bytes.NewReader(video), key, "video/mp4", objectstorage.NoCompression); err != nil {
if err := v.objStorage.Upload(bytes.NewReader(video), key, "video/mp4", "", objectstorage.NoCompression); err != nil {
log.Fatalf("Storage: start upload video replay failed. %s", err)
}
log.Printf("Video file (size: %d) uploaded successfully in %v", len(video), time.Since(start))

2
backend/pkg/objectstorage/objectstorage.go
View file

@ -15,7 +15,7 @@ const (
)
type ObjectStorage interface {
Upload(reader io.Reader, key string, contentType string, compression CompressionType) error
Upload(reader io.Reader, key string, contentType, contentEncoding string, compression CompressionType) error
Get(key string) (io.ReadCloser, error)
Exists(key string) bool
GetCreationTime(key string) *time.Time

13
backend/pkg/objectstorage/s3/s3.go
View file

@ -63,19 +63,22 @@ func NewS3(cfg *objConfig.ObjectsConfig) (objectstorage.ObjectStorage, error) {
}, nil
}
func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, compression objectstorage.CompressionType) error {
func (s *storageImpl) Upload(reader io.Reader, key string, contentType, contentEncoding string, compression objectstorage.CompressionType) error {
cacheControl := "max-age=2628000, immutable, private"
var contentEncoding *string
var encoding *string
switch compression {
case objectstorage.Gzip:
encodeStr := "gzip"
contentEncoding = &encodeStr
encoding = &encodeStr
case objectstorage.Brotli:
encodeStr := "br"
contentEncoding = &encodeStr
encoding = &encodeStr
case objectstorage.Zstd:
// Have to ignore contentEncoding for Zstd (otherwise will be an error in browser)
}
if contentEncoding != "" {
encoding = &contentEncoding
}
_, err := s.uploader.Upload(&s3manager.UploadInput{
Body: reader,
@ -83,7 +86,7 @@ func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, c
Key: &key,
ContentType: &contentType,
CacheControl: &cacheControl,
ContentEncoding: contentEncoding,
ContentEncoding: encoding,
Tagging: s.fileTag,
})
return err

15
ee/backend/pkg/objectstorage/azure/azure.go
View file

@ -50,16 +50,21 @@ func NewStorage(cfg *config.ObjectsConfig) (objectstorage.ObjectStorage, error)
}, nil
}
func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, compression objectstorage.CompressionType) error {
func (s *storageImpl) Upload(reader io.Reader, key string, contentType, contentEncoding string, compression objectstorage.CompressionType) error {
cacheControl := "max-age=2628000, immutable, private"
var contentEncoding *string
var encoding *string
switch compression {
case objectstorage.Gzip:
gzipStr := "gzip"
contentEncoding = &gzipStr
encoding = &gzipStr
case objectstorage.Brotli:
gzipStr := "br"
contentEncoding = &gzipStr
encoding = &gzipStr
case objectstorage.Zstd:
// Have to ignore contentEncoding for Zstd (otherwise will be an error in browser)
}
if contentEncoding != "" {
encoding = &contentEncoding
}
// Remove leading slash to avoid empty folder creation
if strings.HasPrefix(key, "/") {
@ -68,7 +73,7 @@ func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, c
_, err := s.client.UploadStream(context.Background(), s.container, key, reader, &azblob.UploadStreamOptions{
HTTPHeaders: &blob.HTTPHeaders{
BlobCacheControl: &cacheControl,
BlobContentEncoding: contentEncoding,
BlobContentEncoding: encoding,
BlobContentType: &contentType,
},
Tags: s.tags,