jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(s3): added the content encoding support

Browse source
This commit is contained in:
Alexander 2025-01-27 15:37:04 +01:00
parent 6cbe17c8e6
commit 16182031e1
9 changed files with 22 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
backend/internal/assets/cacher/cacher.go
View file

@ -143,6 +143,7 @@ func (c *cacher) cacheURL(t *Task) {
if contentType == "" { if contentType == "" {
contentType = mime.TypeByExtension(filepath.Ext(res.Request.URL.Path)) contentType = mime.TypeByExtension(filepath.Ext(res.Request.URL.Path))
} }
contentEncoding := res.Header.Get("Content-Encoding")
// Skip html file (usually it's a CDN mock for 404 error) // Skip html file (usually it's a CDN mock for 404 error)
if strings.HasPrefix(contentType, "text/html") { if strings.HasPrefix(contentType, "text/html") {
@ -159,7 +160,7 @@ func (c *cacher) cacheURL(t *Task) {
// TODO: implement in streams // TODO: implement in streams
start = time.Now() start = time.Now()
err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, objectstorage.NoCompression) err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, contentEncoding, objectstorage.NoCompression)
if err != nil { if err != nil {
metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), true) metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), true)
c.Errors <- errors.Wrap(err, t.urlContext) c.Errors <- errors.Wrap(err, t.urlContext)

2
backend/internal/canvas-handler/service.go
View file

@ -154,7 +154,7 @@ func (v *ImageStorage) sendToS3(payload interface{}) {
if err != nil { if err != nil {
v.log.Fatal(task.ctx, "failed to read canvas archive: %s", err) v.log.Fatal(task.ctx, "failed to read canvas archive: %s", err)
} }
if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.Zstd); err != nil { if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.NoContentEncoding, objectstorage.Zstd); err != nil {
v.log.Fatal(task.ctx, "failed to upload canvas to storage: %s", err) v.log.Fatal(task.ctx, "failed to upload canvas to storage: %s", err)
} }
v.log.Info(task.ctx, "replay file (size: %d) uploaded successfully in %v", len(video), time.Since(start)) v.log.Info(task.ctx, "replay file (size: %d) uploaded successfully in %v", len(video), time.Since(start))

2
backend/internal/screenshot-handler/service.go
View file

@ -159,7 +159,7 @@ func (v *ImageStorage) sendToS3(payload interface{}) {
if err != nil { if err != nil {
v.log.Fatal(task.ctx, "failed to read replay file: %s", err) v.log.Fatal(task.ctx, "failed to read replay file: %s", err)
} }
if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.Zstd); err != nil { if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.NoContentEncoding, objectstorage.Zstd); err != nil {
v.log.Fatal(task.ctx, "failed to upload replay file: %s", err) v.log.Fatal(task.ctx, "failed to upload replay file: %s", err)
} }
v.log.Info(task.ctx, "replay file (size: %d) uploaded successfully in %v", len(video), time.Since(start)) v.log.Info(task.ctx, "replay file (size: %d) uploaded successfully in %v", len(video), time.Since(start))

6
backend/internal/storage/storage.go
View file

@ -385,7 +385,7 @@ func (s *Storage) uploadSession(payload interface{}) {
metrics.RecordSessionCompressionRatio(task.domsRawSize/float64(task.doms.Len()), DOM.String()) metrics.RecordSessionCompressionRatio(task.domsRawSize/float64(task.doms.Len()), DOM.String())
// Upload session to s3 // Upload session to s3
start := time.Now() start := time.Now()
if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", task.compression); err != nil { if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil {
s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err) s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err)
} }
uploadDoms = time.Now().Sub(start).Milliseconds() uploadDoms = time.Now().Sub(start).Milliseconds()
@ -398,7 +398,7 @@ func (s *Storage) uploadSession(payload interface{}) {
metrics.RecordSessionCompressionRatio(task.domeRawSize/float64(task.dome.Len()), DOM.String()) metrics.RecordSessionCompressionRatio(task.domeRawSize/float64(task.dome.Len()), DOM.String())
// Upload session to s3 // Upload session to s3
start := time.Now() start := time.Now()
if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", task.compression); err != nil { if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil {
s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err) s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err)
} }
uploadDome = time.Now().Sub(start).Milliseconds() uploadDome = time.Now().Sub(start).Milliseconds()
@ -411,7 +411,7 @@ func (s *Storage) uploadSession(payload interface{}) {
metrics.RecordSessionCompressionRatio(task.devRawSize/float64(task.dev.Len()), DEV.String()) metrics.RecordSessionCompressionRatio(task.devRawSize/float64(task.dev.Len()), DEV.String())
// Upload session to s3 // Upload session to s3
start := time.Now() start := time.Now()
if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", task.compression); err != nil { if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil {
s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err) s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err)
} }
uploadDev = time.Now().Sub(start).Milliseconds() uploadDev = time.Now().Sub(start).Milliseconds()

2
backend/pkg/integrations/service/service.go
View file

@ -182,7 +182,7 @@ func (s *serviceImpl) fetchSessionData(provider string, credentials interface{},
func (s *serviceImpl) uploadSessionData(provider string, sessionID uint64, data interface{}) error { func (s *serviceImpl) uploadSessionData(provider string, sessionID uint64, data interface{}) error {
key := fmt.Sprintf("%d/%s.logs", sessionID, provider) key := fmt.Sprintf("%d/%s.logs", sessionID, provider)
dataBytes, _ := data.([]byte) dataBytes, _ := data.([]byte)
return s.storage.Upload(bytes.NewReader(dataBytes), key, "text/plain", objectstorage.NoCompression) return s.storage.Upload(bytes.NewReader(dataBytes), key, "text/plain", objectstorage.NoContentEncoding, objectstorage.NoCompression)
} }
func (s *serviceImpl) markSessionData(projectID uint64, provider string, sessionID uint64) error { func (s *serviceImpl) markSessionData(projectID uint64, provider string, sessionID uint64) error {

4
backend/pkg/objectstorage/objectstorage.go
View file

@ -14,8 +14,10 @@ const (
Zstd Zstd
) )
const NoContentEncoding = ""
type ObjectStorage interface { type ObjectStorage interface {
Upload(reader io.Reader, key string, contentType string, compression CompressionType) error Upload(reader io.Reader, key string, contentType, contentEncoding string, compression CompressionType) error
Get(key string) (io.ReadCloser, error) Get(key string) (io.ReadCloser, error)
Exists(key string) bool Exists(key string) bool
GetCreationTime(key string) *time.Time GetCreationTime(key string) *time.Time

13
backend/pkg/objectstorage/s3/s3.go
View file

@ -67,19 +67,22 @@ func NewS3(cfg *objConfig.ObjectsConfig) (objectstorage.ObjectStorage, error) {
}, nil }, nil
} }
func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, compression objectstorage.CompressionType) error { func (s *storageImpl) Upload(reader io.Reader, key string, contentType, contentEncoding string, compression objectstorage.CompressionType) error {
cacheControl := "max-age=2628000, immutable, private" cacheControl := "max-age=2628000, immutable, private"
var contentEncoding *string var encoding *string
switch compression { switch compression {
case objectstorage.Gzip: case objectstorage.Gzip:
encodeStr := "gzip" encodeStr := "gzip"
contentEncoding = &encodeStr encoding = &encodeStr
case objectstorage.Brotli: case objectstorage.Brotli:
encodeStr := "br" encodeStr := "br"
contentEncoding = &encodeStr encoding = &encodeStr
case objectstorage.Zstd: case objectstorage.Zstd:
// Have to ignore contentEncoding for Zstd (otherwise will be an error in browser) // Have to ignore contentEncoding for Zstd (otherwise will be an error in browser)
} }
if contentEncoding != "" {
encoding = &contentEncoding
}
_, err := s.uploader.Upload(&s3manager.UploadInput{ _, err := s.uploader.Upload(&s3manager.UploadInput{
Body: reader, Body: reader,
@ -87,7 +90,7 @@ func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, c
Key: &key, Key: &key,
ContentType: &contentType, ContentType: &contentType,
CacheControl: &cacheControl, CacheControl: &cacheControl,
ContentEncoding: contentEncoding, ContentEncoding: encoding,
Tagging: s.fileTag, Tagging: s.fileTag,
}) })
return err return err

2
backend/pkg/spot/api/handlers.go
View file

@ -99,7 +99,7 @@ func (e *handlersImpl) createSpot(w http.ResponseWriter, r *http.Request) {
} }
previewName := fmt.Sprintf("%d/preview.jpeg", newSpot.ID) previewName := fmt.Sprintf("%d/preview.jpeg", newSpot.ID)
if err = e.objStorage.Upload(bytes.NewReader(previewImage), previewName, "image/jpeg", objectstorage.NoCompression); err != nil { if err = e.objStorage.Upload(bytes.NewReader(previewImage), previewName, "image/jpeg", objectstorage.NoContentEncoding, objectstorage.NoCompression); err != nil {
e.log.Error(r.Context(), "can't upload preview image: %s", err) e.log.Error(r.Context(), "can't upload preview image: %s", err)
e.responser.ResponseWithError(e.log, r.Context(), w, http.StatusInternalServerError, errors.New("can't upload preview image"), startTime, r.URL.Path, bodySize) e.responser.ResponseWithError(e.log, r.Context(), w, http.StatusInternalServerError, errors.New("can't upload preview image"), startTime, r.URL.Path, bodySize)
return return

4
backend/pkg/spot/transcoder/transcoder.go
View file

@ -249,7 +249,7 @@ func (t *transcoderImpl) cropSpotVideo(spotID uint64, crop []int, path string) e
metrics.RecordCroppedVideoSize(float64(fileInfo.Size())) metrics.RecordCroppedVideoSize(float64(fileInfo.Size()))
} }
err = t.objStorage.Upload(video, fmt.Sprintf("%d/video.webm", spotID), "video/webm", objectstorage.NoCompression) err = t.objStorage.Upload(video, fmt.Sprintf("%d/video.webm", spotID), "video/webm", objectstorage.NoContentEncoding, objectstorage.NoCompression)
if err != nil { if err != nil {
return fmt.Errorf("failed to upload cropped video: %v", err) return fmt.Errorf("failed to upload cropped video: %v", err)
} }
@ -321,7 +321,7 @@ func (t *transcoderImpl) transcodeSpotVideo(spotID uint64, path string) (string,
defer chunkFile.Close() defer chunkFile.Close()
key := fmt.Sprintf("%d/%s", spotID, chunk) key := fmt.Sprintf("%d/%s", spotID, chunk)
err = t.objStorage.Upload(chunkFile, key, "video/mp2t", objectstorage.NoCompression) err = t.objStorage.Upload(chunkFile, key, "video/mp2t", objectstorage.NoContentEncoding, objectstorage.NoCompression)
if err != nil { if err != nil {
fmt.Println("Error uploading file:", err) fmt.Println("Error uploading file:", err)
return "", err return "", err