jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(s3): added the content encoding support

Browse source
This commit is contained in:
Alexander 2025-01-27 15:37:04 +01:00
parent 6cbe17c8e6
commit 16182031e1
9 changed files with 22 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
backend/internal/assets/cacher/cacher.go
View file

@ -143,6 +143,7 @@ func (c *cacher) cacheURL(t *Task) {
if contentType == "" {
contentType = mime.TypeByExtension(filepath.Ext(res.Request.URL.Path))
}
contentEncoding := res.Header.Get("Content-Encoding")
// Skip html file (usually it's a CDN mock for 404 error)
if strings.HasPrefix(contentType, "text/html") {
@ -159,7 +160,7 @@ func (c *cacher) cacheURL(t *Task) {
// TODO: implement in streams
start = time.Now()
err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, objectstorage.NoCompression)
err = c.objStorage.Upload(strings.NewReader(strData), t.cachePath, contentType, contentEncoding, objectstorage.NoCompression)
if err != nil {
metrics.RecordUploadDuration(float64(time.Now().Sub(start).Milliseconds()), true)
c.Errors <- errors.Wrap(err, t.urlContext)

2
backend/internal/canvas-handler/service.go
View file

@ -154,7 +154,7 @@ func (v *ImageStorage) sendToS3(payload interface{}) {
if err != nil {
v.log.Fatal(task.ctx, "failed to read canvas archive: %s", err)
}
if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.Zstd); err != nil {
if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.NoContentEncoding, objectstorage.Zstd); err != nil {
v.log.Fatal(task.ctx, "failed to upload canvas to storage: %s", err)
}
v.log.Info(task.ctx, "replay file (size: %d) uploaded successfully in %v", len(video), time.Since(start))

2
backend/internal/screenshot-handler/service.go
View file

@ -159,7 +159,7 @@ func (v *ImageStorage) sendToS3(payload interface{}) {
if err != nil {
v.log.Fatal(task.ctx, "failed to read replay file: %s", err)
}
if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.Zstd); err != nil {
if err := v.objStorage.Upload(bytes.NewReader(video), task.name, "application/octet-stream", objectstorage.NoContentEncoding, objectstorage.Zstd); err != nil {
v.log.Fatal(task.ctx, "failed to upload replay file: %s", err)
}
v.log.Info(task.ctx, "replay file (size: %d) uploaded successfully in %v", len(video), time.Since(start))

6
backend/internal/storage/storage.go
View file

@ -385,7 +385,7 @@ func (s *Storage) uploadSession(payload interface{}) {
metrics.RecordSessionCompressionRatio(task.domsRawSize/float64(task.doms.Len()), DOM.String())
// Upload session to s3
start := time.Now()
if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", task.compression); err != nil {
if err := s.objStorage.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil {
s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err)
}
uploadDoms = time.Now().Sub(start).Milliseconds()
@ -398,7 +398,7 @@ func (s *Storage) uploadSession(payload interface{}) {
metrics.RecordSessionCompressionRatio(task.domeRawSize/float64(task.dome.Len()), DOM.String())
// Upload session to s3
start := time.Now()
if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", task.compression); err != nil {
if err := s.objStorage.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil {
s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err)
}
uploadDome = time.Now().Sub(start).Milliseconds()
@ -411,7 +411,7 @@ func (s *Storage) uploadSession(payload interface{}) {
metrics.RecordSessionCompressionRatio(task.devRawSize/float64(task.dev.Len()), DEV.String())
// Upload session to s3
start := time.Now()
if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", task.compression); err != nil {
if err := s.objStorage.Upload(task.dev, task.id+string(DEV), "application/octet-stream", objectstorage.NoContentEncoding, task.compression); err != nil {
s.log.Fatal(task.ctx, "failed to upload mob file, err: %s", err)
}
uploadDev = time.Now().Sub(start).Milliseconds()

2
backend/pkg/integrations/service/service.go
View file

@ -182,7 +182,7 @@ func (s *serviceImpl) fetchSessionData(provider string, credentials interface{},
func (s *serviceImpl) uploadSessionData(provider string, sessionID uint64, data interface{}) error {
key := fmt.Sprintf("%d/%s.logs", sessionID, provider)
dataBytes, _ := data.([]byte)
return s.storage.Upload(bytes.NewReader(dataBytes), key, "text/plain", objectstorage.NoCompression)
return s.storage.Upload(bytes.NewReader(dataBytes), key, "text/plain", objectstorage.NoContentEncoding, objectstorage.NoCompression)
}
func (s *serviceImpl) markSessionData(projectID uint64, provider string, sessionID uint64) error {

4
backend/pkg/objectstorage/objectstorage.go
View file

@ -14,8 +14,10 @@ const (
Zstd
)
const NoContentEncoding = ""
type ObjectStorage interface {
Upload(reader io.Reader, key string, contentType string, compression CompressionType) error
Upload(reader io.Reader, key string, contentType, contentEncoding string, compression CompressionType) error
Get(key string) (io.ReadCloser, error)
Exists(key string) bool
GetCreationTime(key string) *time.Time

13
backend/pkg/objectstorage/s3/s3.go
View file

@ -67,19 +67,22 @@ func NewS3(cfg *objConfig.ObjectsConfig) (objectstorage.ObjectStorage, error) {
}, nil
}
func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, compression objectstorage.CompressionType) error {
func (s *storageImpl) Upload(reader io.Reader, key string, contentType, contentEncoding string, compression objectstorage.CompressionType) error {
cacheControl := "max-age=2628000, immutable, private"
var contentEncoding *string
var encoding *string
switch compression {
case objectstorage.Gzip:
encodeStr := "gzip"
contentEncoding = &encodeStr
encoding = &encodeStr
case objectstorage.Brotli:
encodeStr := "br"
contentEncoding = &encodeStr
encoding = &encodeStr
case objectstorage.Zstd:
// Have to ignore contentEncoding for Zstd (otherwise will be an error in browser)
}
if contentEncoding != "" {
encoding = &contentEncoding
}
_, err := s.uploader.Upload(&s3manager.UploadInput{
Body: reader,
@ -87,7 +90,7 @@ func (s *storageImpl) Upload(reader io.Reader, key string, contentType string, c
Key: &key,
ContentType: &contentType,
CacheControl: &cacheControl,
ContentEncoding: contentEncoding,
ContentEncoding: encoding,
Tagging: s.fileTag,
})
return err

2
backend/pkg/spot/api/handlers.go
View file

@ -99,7 +99,7 @@ func (e *handlersImpl) createSpot(w http.ResponseWriter, r *http.Request) {
}
previewName := fmt.Sprintf("%d/preview.jpeg", newSpot.ID)
if err = e.objStorage.Upload(bytes.NewReader(previewImage), previewName, "image/jpeg", objectstorage.NoCompression); err != nil {
if err = e.objStorage.Upload(bytes.NewReader(previewImage), previewName, "image/jpeg", objectstorage.NoContentEncoding, objectstorage.NoCompression); err != nil {
e.log.Error(r.Context(), "can't upload preview image: %s", err)
e.responser.ResponseWithError(e.log, r.Context(), w, http.StatusInternalServerError, errors.New("can't upload preview image"), startTime, r.URL.Path, bodySize)
return

4
backend/pkg/spot/transcoder/transcoder.go
View file

@ -249,7 +249,7 @@ func (t *transcoderImpl) cropSpotVideo(spotID uint64, crop []int, path string) e
metrics.RecordCroppedVideoSize(float64(fileInfo.Size()))
}
err = t.objStorage.Upload(video, fmt.Sprintf("%d/video.webm", spotID), "video/webm", objectstorage.NoCompression)
err = t.objStorage.Upload(video, fmt.Sprintf("%d/video.webm", spotID), "video/webm", objectstorage.NoContentEncoding, objectstorage.NoCompression)
if err != nil {
return fmt.Errorf("failed to upload cropped video: %v", err)
}
@ -321,7 +321,7 @@ func (t *transcoderImpl) transcodeSpotVideo(spotID uint64, path string) (string,
defer chunkFile.Close()
key := fmt.Sprintf("%d/%s", spotID, chunk)
err = t.objStorage.Upload(chunkFile, key, "video/mp2t", objectstorage.NoCompression)
err = t.objStorage.Upload(chunkFile, key, "video/mp2t", objectstorage.NoContentEncoding, objectstorage.NoCompression)
if err != nil {
fmt.Println("Error uploading file:", err)
return "", err