jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[Storage] different order to compress and encrypt (#1182)

Browse source 
* feat(backend): try to compress and encrypt in a new way
This commit is contained in:
Alexander 2023-04-19 11:04:35 +02:00 • committed by GitHub
parent 41f8b5ad57
commit ef9fed8bf8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 27 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
backend/internal/storage/storage.go
View file

@ -186,19 +186,20 @@ func (s *Storage) packSession(task *Task, tp FileType) {
mob := task.Mob(tp) mob := task.Mob(tp)
if tp == DEV || len(mob) <= s.cfg.FileSplitSize { if tp == DEV || len(mob) <= s.cfg.FileSplitSize {
// Encryption
start := time.Now()
data := s.encryptSession(mob, task.key)
metrics.RecordSessionEncryptionDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
// Compression // Compression
start = time.Now() start := time.Now()
result := s.compressSession(data) data := s.compressSession(mob)
metrics.RecordSessionCompressDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String()) metrics.RecordSessionCompressDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
// Encryption
start = time.Now()
result := s.encryptSession(data.Bytes(), task.key)
metrics.RecordSessionEncryptionDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
if tp == DOM { if tp == DOM {
task.doms = result task.doms = bytes.NewBuffer(result)
} else { } else {
task.dev = result task.dev = bytes.NewBuffer(result)
} }
return return
} }
@ -210,31 +211,31 @@ func (s *Storage) packSession(task *Task, tp FileType) {
// DomStart part // DomStart part
go func() { go func() {
// Encryption
start := time.Now()
data := s.encryptSession(mob[:s.cfg.FileSplitSize], task.key)
firstEncrypt = time.Since(start).Milliseconds()
// Compression // Compression
start = time.Now() start := time.Now()
task.doms = s.compressSession(data) data := s.compressSession(mob[:s.cfg.FileSplitSize])
firstPart = time.Since(start).Milliseconds() firstPart = time.Since(start).Milliseconds()
// Encryption
start = time.Now()
task.doms = bytes.NewBuffer(s.encryptSession(data.Bytes(), task.key))
firstEncrypt = time.Since(start).Milliseconds()
// Finish task // Finish task
wg.Done() wg.Done()
}() }()
// DomEnd part // DomEnd part
go func() { go func() {
// Encryption
start := time.Now()
data := s.encryptSession(mob[s.cfg.FileSplitSize:], task.key)
secondEncrypt = time.Since(start).Milliseconds()
// Compression // Compression
start = time.Now() start := time.Now()
task.dome = s.compressSession(data) data := s.compressSession(mob[s.cfg.FileSplitSize:])
secondPart = time.Since(start).Milliseconds() secondPart = time.Since(start).Milliseconds()
// Encryption
start = time.Now()
task.dome = bytes.NewBuffer(s.encryptSession(data.Bytes(), task.key))
secondEncrypt = time.Since(start).Milliseconds()
// Finish task // Finish task
wg.Done() wg.Done()
}() }()
@ -262,7 +263,7 @@ func (s *Storage) encryptSession(data []byte, encryptionKey string) []byte {
func (s *Storage) compressSession(data []byte) *bytes.Buffer { func (s *Storage) compressSession(data []byte) *bytes.Buffer {
zippedMob := new(bytes.Buffer) zippedMob := new(bytes.Buffer)
z, _ := gzip.NewWriterLevel(zippedMob, gzip.BestSpeed) z, _ := gzip.NewWriterLevel(zippedMob, gzip.DefaultCompression)
if _, err := z.Write(data); err != nil { if _, err := z.Write(data); err != nil {
log.Printf("can't write session data to compressor: %s", err) log.Printf("can't write session data to compressor: %s", err)
} }
@ -283,7 +284,7 @@ func (s *Storage) uploadSession(task *Task) {
go func() { go func() {
if task.doms != nil { if task.doms != nil {
start := time.Now() start := time.Now()
if err := s.s3.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", true); err != nil { if err := s.s3.Upload(task.doms, task.id+string(DOM)+"s", "application/octet-stream", task.key == ""); err != nil {
log.Fatalf("Storage: start upload failed. %s", err) log.Fatalf("Storage: start upload failed. %s", err)
} }
uploadDoms = time.Now().Sub(start).Milliseconds() uploadDoms = time.Now().Sub(start).Milliseconds()
@ -293,7 +294,7 @@ func (s *Storage) uploadSession(task *Task) {
go func() { go func() {
if task.dome != nil { if task.dome != nil {
start := time.Now() start := time.Now()
if err := s.s3.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", true); err != nil { if err := s.s3.Upload(task.dome, task.id+string(DOM)+"e", "application/octet-stream", task.key == ""); err != nil {
log.Fatalf("Storage: start upload failed. %s", err) log.Fatalf("Storage: start upload failed. %s", err)
} }
uploadDome = time.Now().Sub(start).Milliseconds() uploadDome = time.Now().Sub(start).Milliseconds()
@ -303,7 +304,7 @@ func (s *Storage) uploadSession(task *Task) {
go func() { go func() {
if task.dev != nil { if task.dev != nil {
start := time.Now() start := time.Now()
if err := s.s3.Upload(task.dev, task.id+string(DEV), "application/octet-stream", true); err != nil { if err := s.s3.Upload(task.dev, task.id+string(DEV), "application/octet-stream", task.key == ""); err != nil {
log.Fatalf("Storage: start upload failed. %s", err) log.Fatalf("Storage: start upload failed. %s", err)
} }
uploadDev = time.Now().Sub(start).Milliseconds() uploadDev = time.Now().Sub(start).Milliseconds()