jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Session mod files encryption (#766)

Browse source 
* feat(backend): added session mod files encryption
This commit is contained in:
Alexander 2022-10-18 12:50:36 +02:00 • committed by GitHub
parent ca77c6b531
commit a166482227
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 2820 additions and 2539 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
backend/cmd/ender/main.go
View file

@ -2,6 +2,7 @@ package main
import (
"log"
"openreplay/backend/internal/storage"
"os"
"os/signal"
"syscall"
@ -78,6 +79,15 @@ func main() {
currDuration, newDuration)
return true
}
if cfg.UseEncryption {
if key := storage.GenerateEncryptionKey(); key != nil {
if err := pg.InsertSessionEncryptionKey(sessionID, key); err != nil {
log.Printf("can't save session encryption key: %s, session will not be encrypted", err)
} else {
msg.EncryptionKey = string(key)
}
}
}
if err := producer.Produce(cfg.TopicRawWeb, sessionID, msg.Encode()); err != nil {
log.Printf("can't send sessionEnd to topic: %s; sessID: %d", err, sessionID)
return false

2
backend/cmd/storage/main.go
View file

@ -44,7 +44,7 @@ func main() {
messages.NewMessageIterator(
func(msg messages.Message) {
sesEnd := msg.(*messages.SessionEnd)
if err := srv.UploadSessionFiles(msg.SessionID()); err != nil {
if err := srv.UploadSessionFiles(sesEnd); err != nil {
log.Printf("can't find session: %d", msg.SessionID())
sessionFinder.Find(msg.SessionID(), sesEnd.Timestamp)
}

1
backend/internal/config/ender/config.go
View file

@ -14,6 +14,7 @@ type Config struct {
TopicRawWeb string `env:"TOPIC_RAW_WEB,required"`
ProducerTimeout int `env:"PRODUCER_TIMEOUT,default=2000"`
PartitionsNumber int `env:"PARTITIONS_NUMBER,required"`
UseEncryption bool `env:"USE_ENCRYPTION,default=false"`
}
func New() *Config {

17
backend/internal/storage/encryptor.go Normal file
View file

@ -0,0 +1,17 @@
package storage
import (
"errors"
)
func GenerateEncryptionKey() []byte {
return nil
}
func EncryptData(data, fullKey []byte) ([]byte, error) {
return nil, errors.New("not supported")
}
func DecryptData(data, fullKey []byte) ([]byte, error) {
return nil, errors.New("not supported")
}

104
backend/internal/storage/storage.go
View file

@ -8,6 +8,7 @@ import (
"log"
config "openreplay/backend/internal/config/storage"
"openreplay/backend/pkg/flakeid"
"openreplay/backend/pkg/messages"
"openreplay/backend/pkg/monitoring"
"openreplay/backend/pkg/storage"
"os"
@ -68,19 +69,19 @@ func New(cfg *config.Config, s3 *storage.S3, metrics *monitoring.Metrics) (*Stor
}, nil
}
func (s *Storage) UploadSessionFiles(sessID uint64) error {
sessionDir := strconv.FormatUint(sessID, 10)
if err := s.uploadKey(sessID, sessionDir+"/dom.mob", true, 5); err != nil {
func (s *Storage) UploadSessionFiles(msg *messages.SessionEnd) error {
sessionDir := strconv.FormatUint(msg.SessionID(), 10)
if err := s.uploadKey(msg.SessionID(), sessionDir+"/dom.mob", true, 5, msg.EncryptionKey); err != nil {
return err
}
if err := s.uploadKey(sessID, sessionDir+"/devtools.mob", false, 4); err != nil {
if err := s.uploadKey(msg.SessionID(), sessionDir+"/devtools.mob", false, 4, msg.EncryptionKey); err != nil {
return err
}
return nil
}
// TODO: make a bit cleaner
func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCount int) error {
func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCount int, encryptionKey string) error {
if retryCount <= 0 {
return nil
}
@ -95,6 +96,14 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo
}
defer file.Close()
var fileSize int64 = 0
fileInfo, err := file.Stat()
if err != nil {
log.Printf("can't get file info: %s", err)
} else {
fileSize = fileInfo.Size()
}
var encryptedData []byte
if shouldSplit {
nRead, err := file.Read(s.startBytes)
if err != nil {
@ -105,45 +114,104 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo
time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))),
)
time.AfterFunc(s.cfg.RetryTimeout, func() {
s.uploadKey(sessID, key, shouldSplit, retryCount-1)
s.uploadKey(sessID, key, shouldSplit, retryCount-1, encryptionKey)
})
return nil
}
s.readingTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()))
start = time.Now()
startReader := bytes.NewBuffer(s.startBytes[:nRead])
// Encrypt session file if we have encryption key
if encryptionKey != "" {
encryptedData, err = EncryptData(s.startBytes[:nRead], []byte(encryptionKey))
if err != nil {
log.Printf("can't encrypt data: %s", err)
encryptedData = s.startBytes[:nRead]
}
} else {
encryptedData = s.startBytes[:nRead]
}
// Compress and save to s3
startReader := bytes.NewBuffer(encryptedData)
if err := s.s3.Upload(s.gzipFile(startReader), key+"s", "application/octet-stream", true); err != nil {
log.Fatalf("Storage: start upload failed. %v\n", err)
}
// TODO: fix possible error (if we read less then FileSplitSize)
if nRead == s.cfg.FileSplitSize {
if err := s.s3.Upload(s.gzipFile(file), key+"e", "application/octet-stream", true); err != nil {
restPartSize := fileSize - int64(nRead)
fileData := make([]byte, restPartSize)
nRead, err = file.Read(fileData)
if err != nil {
log.Printf("File read error: %s; sessID: %s, part: %d, sessStart: %s",
err,
key,
sessID%16,
time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))),
)
return nil
}
if int64(nRead) != restPartSize {
log.Printf("can't read the rest part of file")
}
// Encrypt session file if we have encryption key
if encryptionKey != "" {
encryptedData, err = EncryptData(fileData, []byte(encryptionKey))
if err != nil {
log.Printf("can't encrypt data: %s", err)
encryptedData = fileData
}
} else {
encryptedData = fileData
}
// Compress and save to s3
endReader := bytes.NewBuffer(encryptedData)
if err := s.s3.Upload(s.gzipFile(endReader), key+"e", "application/octet-stream", true); err != nil {
log.Fatalf("Storage: end upload failed. %v\n", err)
}
}
s.archivingTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()))
} else {
start = time.Now()
if err := s.s3.Upload(s.gzipFile(file), key+"s", "application/octet-stream", true); err != nil {
fileData := make([]byte, fileSize)
nRead, err := file.Read(fileData)
if err != nil {
log.Printf("File read error: %s; sessID: %s, part: %d, sessStart: %s",
err,
key,
sessID%16,
time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))),
)
return nil
}
if int64(nRead) != fileSize {
log.Printf("can't read the rest part of file")
}
// Encrypt session file if we have encryption key
if encryptionKey != "" {
encryptedData, err = EncryptData(fileData, []byte(encryptionKey))
if err != nil {
log.Printf("can't encrypt data: %s", err)
encryptedData = fileData
}
} else {
encryptedData = fileData
}
endReader := bytes.NewBuffer(encryptedData)
if err := s.s3.Upload(s.gzipFile(endReader), key+"s", "application/octet-stream", true); err != nil {
log.Fatalf("Storage: end upload failed. %v\n", err)
}
s.archivingTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()))
}
// Save metrics
var fileSize float64 = 0
fileInfo, err := file.Stat()
if err != nil {
log.Printf("can't get file info: %s", err)
} else {
fileSize = float64(fileInfo.Size())
}
ctx, _ := context.WithTimeout(context.Background(), time.Millisecond*200)
if shouldSplit {
s.totalSessions.Add(ctx, 1)
s.sessionDOMSize.Record(ctx, fileSize)
s.sessionDOMSize.Record(ctx, float64(fileSize))
} else {
s.sessionDevtoolsSize.Record(ctx, fileSize)
s.sessionDevtoolsSize.Record(ctx, float64(fileSize))
}
return nil

4
backend/pkg/db/cache/messages-common.go vendored
View file

@ -11,6 +11,10 @@ func (c *PGCache) InsertSessionEnd(sessionID uint64, timestamp uint64) (uint64,
return c.Conn.InsertSessionEnd(sessionID, timestamp)
}
func (c *PGCache) InsertSessionEncryptionKey(sessionID uint64, key []byte) error {
return c.Conn.InsertSessionEncryptionKey(sessionID, key)
}
func (c *PGCache) HandleSessionEnd(sessionID uint64) error {
if err := c.Conn.HandleSessionEnd(sessionID); err != nil {
log.Printf("can't handle session end: %s", err)

4
backend/pkg/db/postgres/messages-common.go
View file

@ -82,6 +82,10 @@ func (conn *Conn) InsertSessionEnd(sessionID uint64, timestamp uint64) (uint64,
return dur, nil
}
func (conn *Conn) InsertSessionEncryptionKey(sessionID uint64, key []byte) error {
return conn.c.Exec(`UPDATE sessions SET file_key = $2 WHERE session_id = $1`, sessionID, string(key))
}
func (conn *Conn) HandleSessionEnd(sessionID uint64) error {
sqlRequest := `
UPDATE sessions

11
backend/pkg/messages/messages.go
View file

@ -4,6 +4,7 @@ package messages
import "encoding/binary"
const (
MsgBatchMeta = 80
MsgBatchMetadata = 81
@ -160,6 +161,8 @@ const (
MsgSessionSearch = 127
MsgExceptionWithMeta = 78
MsgIOSBatchMeta = 107
MsgIOSSessionStart = 90
@ -197,8 +200,10 @@ const (
MsgIOSPerformanceAggregated = 110
MsgIOSIssueEvent = 111
)
type BatchMeta struct {
message
PageNo uint64
@ -406,13 +411,15 @@ func (msg *SessionStart) TypeID() int {
type SessionEnd struct {
message
Timestamp uint64
EncryptionKey string
}
func (msg *SessionEnd) Encode() []byte {
buf := make([]byte, 11)
buf := make([]byte, 21+len(msg.EncryptionKey))
buf[0] = 3
p := 1
p = WriteUint(msg.Timestamp, buf, p)
p = WriteString(msg.EncryptionKey, buf, p)
return buf[:p]
}
@ -541,6 +548,7 @@ func (msg *SetViewportScroll) TypeID() int {
type CreateDocument struct {
message
}
func (msg *CreateDocument) Encode() []byte {
@ -3873,3 +3881,4 @@ func (msg *IOSIssueEvent) Decode() Message {
func (msg *IOSIssueEvent) TypeID() int {
return 111
}

98
backend/pkg/messages/read-message.go
View file

@ -6,6 +6,7 @@ import (
"io"
)
func DecodeBatchMeta(reader io.Reader) (Message, error) {
var err error = nil
msg := &BatchMeta{}
@ -21,6 +22,7 @@ func DecodeBatchMeta(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeBatchMetadata(reader io.Reader) (Message, error) {
var err error = nil
msg := &BatchMetadata{}
@ -42,6 +44,7 @@ func DecodeBatchMetadata(reader io.Reader) (Message, error) {
return msg, err
}
func DecodePartitionedMessage(reader io.Reader) (Message, error) {
var err error = nil
msg := &PartitionedMessage{}
@ -54,6 +57,7 @@ func DecodePartitionedMessage(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeTimestamp(reader io.Reader) (Message, error) {
var err error = nil
msg := &Timestamp{}
@ -63,6 +67,7 @@ func DecodeTimestamp(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSessionStart(reader io.Reader) (Message, error) {
var err error = nil
msg := &SessionStart{}
@ -117,15 +122,20 @@ func DecodeSessionStart(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSessionEnd(reader io.Reader) (Message, error) {
var err error = nil
msg := &SessionEnd{}
if msg.Timestamp, err = ReadUint(reader); err != nil {
return nil, err
}
if msg.EncryptionKey, err = ReadString(reader); err != nil {
return nil, err
}
return msg, err
}
func DecodeSetPageLocation(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetPageLocation{}
@ -141,6 +151,7 @@ func DecodeSetPageLocation(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetViewportSize(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetViewportSize{}
@ -153,6 +164,7 @@ func DecodeSetViewportSize(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetViewportScroll(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetViewportScroll{}
@ -165,6 +177,7 @@ func DecodeSetViewportScroll(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeCreateDocument(reader io.Reader) (Message, error) {
var err error = nil
msg := &CreateDocument{}
@ -172,6 +185,7 @@ func DecodeCreateDocument(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeCreateElementNode(reader io.Reader) (Message, error) {
var err error = nil
msg := &CreateElementNode{}
@ -193,6 +207,7 @@ func DecodeCreateElementNode(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeCreateTextNode(reader io.Reader) (Message, error) {
var err error = nil
msg := &CreateTextNode{}
@ -208,6 +223,7 @@ func DecodeCreateTextNode(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeMoveNode(reader io.Reader) (Message, error) {
var err error = nil
msg := &MoveNode{}
@ -223,6 +239,7 @@ func DecodeMoveNode(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeRemoveNode(reader io.Reader) (Message, error) {
var err error = nil
msg := &RemoveNode{}
@ -232,6 +249,7 @@ func DecodeRemoveNode(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetNodeAttribute(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetNodeAttribute{}
@ -247,6 +265,7 @@ func DecodeSetNodeAttribute(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeRemoveNodeAttribute(reader io.Reader) (Message, error) {
var err error = nil
msg := &RemoveNodeAttribute{}
@ -259,6 +278,7 @@ func DecodeRemoveNodeAttribute(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetNodeData(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetNodeData{}
@ -271,6 +291,7 @@ func DecodeSetNodeData(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetCSSData(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetCSSData{}
@ -283,6 +304,7 @@ func DecodeSetCSSData(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetNodeScroll(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetNodeScroll{}
@ -298,6 +320,7 @@ func DecodeSetNodeScroll(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetInputTarget(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetInputTarget{}
@ -310,6 +333,7 @@ func DecodeSetInputTarget(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetInputValue(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetInputValue{}
@ -325,6 +349,7 @@ func DecodeSetInputValue(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetInputChecked(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetInputChecked{}
@ -337,6 +362,7 @@ func DecodeSetInputChecked(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeMouseMove(reader io.Reader) (Message, error) {
var err error = nil
msg := &MouseMove{}
@ -349,6 +375,7 @@ func DecodeMouseMove(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeConsoleLog(reader io.Reader) (Message, error) {
var err error = nil
msg := &ConsoleLog{}
@ -361,6 +388,7 @@ func DecodeConsoleLog(reader io.Reader) (Message, error) {
return msg, err
}
func DecodePageLoadTiming(reader io.Reader) (Message, error) {
var err error = nil
msg := &PageLoadTiming{}
@ -394,6 +422,7 @@ func DecodePageLoadTiming(reader io.Reader) (Message, error) {
return msg, err
}
func DecodePageRenderTiming(reader io.Reader) (Message, error) {
var err error = nil
msg := &PageRenderTiming{}
@ -424,6 +453,7 @@ func DecodeJSExceptionDeprecated(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIntegrationEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &IntegrationEvent{}
@ -445,6 +475,7 @@ func DecodeIntegrationEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeRawCustomEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &RawCustomEvent{}
@ -457,6 +488,7 @@ func DecodeRawCustomEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeUserID(reader io.Reader) (Message, error) {
var err error = nil
msg := &UserID{}
@ -466,6 +498,7 @@ func DecodeUserID(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeUserAnonymousID(reader io.Reader) (Message, error) {
var err error = nil
msg := &UserAnonymousID{}
@ -475,6 +508,7 @@ func DecodeUserAnonymousID(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeMetadata(reader io.Reader) (Message, error) {
var err error = nil
msg := &Metadata{}
@ -487,6 +521,7 @@ func DecodeMetadata(reader io.Reader) (Message, error) {
return msg, err
}
func DecodePageEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &PageEvent{}
@ -544,6 +579,7 @@ func DecodePageEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeInputEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &InputEvent{}
@ -565,6 +601,7 @@ func DecodeInputEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeClickEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &ClickEvent{}
@ -586,6 +623,7 @@ func DecodeClickEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeErrorEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &ErrorEvent{}
@ -610,6 +648,7 @@ func DecodeErrorEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeResourceEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &ResourceEvent{}
@ -652,6 +691,7 @@ func DecodeResourceEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeCustomEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &CustomEvent{}
@ -670,6 +710,7 @@ func DecodeCustomEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeCSSInsertRule(reader io.Reader) (Message, error) {
var err error = nil
msg := &CSSInsertRule{}
@ -685,6 +726,7 @@ func DecodeCSSInsertRule(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeCSSDeleteRule(reader io.Reader) (Message, error) {
var err error = nil
msg := &CSSDeleteRule{}
@ -697,6 +739,7 @@ func DecodeCSSDeleteRule(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeFetch(reader io.Reader) (Message, error) {
var err error = nil
msg := &Fetch{}
@ -724,6 +767,7 @@ func DecodeFetch(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeProfiler(reader io.Reader) (Message, error) {
var err error = nil
msg := &Profiler{}
@ -742,6 +786,7 @@ func DecodeProfiler(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeOTable(reader io.Reader) (Message, error) {
var err error = nil
msg := &OTable{}
@ -754,6 +799,7 @@ func DecodeOTable(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeStateAction(reader io.Reader) (Message, error) {
var err error = nil
msg := &StateAction{}
@ -763,6 +809,7 @@ func DecodeStateAction(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeStateActionEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &StateActionEvent{}
@ -778,6 +825,7 @@ func DecodeStateActionEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeRedux(reader io.Reader) (Message, error) {
var err error = nil
msg := &Redux{}
@ -793,6 +841,7 @@ func DecodeRedux(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeVuex(reader io.Reader) (Message, error) {
var err error = nil
msg := &Vuex{}
@ -805,6 +854,7 @@ func DecodeVuex(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeMobX(reader io.Reader) (Message, error) {
var err error = nil
msg := &MobX{}
@ -817,6 +867,7 @@ func DecodeMobX(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeNgRx(reader io.Reader) (Message, error) {
var err error = nil
msg := &NgRx{}
@ -832,6 +883,7 @@ func DecodeNgRx(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeGraphQL(reader io.Reader) (Message, error) {
var err error = nil
msg := &GraphQL{}
@ -850,6 +902,7 @@ func DecodeGraphQL(reader io.Reader) (Message, error) {
return msg, err
}
func DecodePerformanceTrack(reader io.Reader) (Message, error) {
var err error = nil
msg := &PerformanceTrack{}
@ -868,6 +921,7 @@ func DecodePerformanceTrack(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeGraphQLEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &GraphQLEvent{}
@ -892,6 +946,7 @@ func DecodeGraphQLEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeFetchEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &FetchEvent{}
@ -922,6 +977,7 @@ func DecodeFetchEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeDOMDrop(reader io.Reader) (Message, error) {
var err error = nil
msg := &DOMDrop{}
@ -931,6 +987,7 @@ func DecodeDOMDrop(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeResourceTiming(reader io.Reader) (Message, error) {
var err error = nil
msg := &ResourceTiming{}
@ -961,6 +1018,7 @@ func DecodeResourceTiming(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeConnectionInformation(reader io.Reader) (Message, error) {
var err error = nil
msg := &ConnectionInformation{}
@ -973,6 +1031,7 @@ func DecodeConnectionInformation(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetPageVisibility(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetPageVisibility{}
@ -982,6 +1041,7 @@ func DecodeSetPageVisibility(reader io.Reader) (Message, error) {
return msg, err
}
func DecodePerformanceTrackAggr(reader io.Reader) (Message, error) {
var err error = nil
msg := &PerformanceTrackAggr{}
@ -1030,6 +1090,7 @@ func DecodePerformanceTrackAggr(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeLongTask(reader io.Reader) (Message, error) {
var err error = nil
msg := &LongTask{}
@ -1057,6 +1118,7 @@ func DecodeLongTask(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetNodeAttributeURLBased(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetNodeAttributeURLBased{}
@ -1075,6 +1137,7 @@ func DecodeSetNodeAttributeURLBased(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeSetCSSDataURLBased(reader io.Reader) (Message, error) {
var err error = nil
msg := &SetCSSDataURLBased{}
@ -1090,6 +1153,7 @@ func DecodeSetCSSDataURLBased(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIssueEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &IssueEvent{}
@ -1114,6 +1178,7 @@ func DecodeIssueEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeTechnicalInfo(reader io.Reader) (Message, error) {
var err error = nil
msg := &TechnicalInfo{}
@ -1126,6 +1191,7 @@ func DecodeTechnicalInfo(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeCustomIssue(reader io.Reader) (Message, error) {
var err error = nil
msg := &CustomIssue{}
@ -1138,6 +1204,7 @@ func DecodeCustomIssue(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeAssetCache(reader io.Reader) (Message, error) {
var err error = nil
msg := &AssetCache{}
@ -1147,6 +1214,7 @@ func DecodeAssetCache(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeCSSInsertRuleURLBased(reader io.Reader) (Message, error) {
var err error = nil
msg := &CSSInsertRuleURLBased{}
@ -1165,6 +1233,7 @@ func DecodeCSSInsertRuleURLBased(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeMouseClick(reader io.Reader) (Message, error) {
var err error = nil
msg := &MouseClick{}
@ -1183,6 +1252,7 @@ func DecodeMouseClick(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeCreateIFrameDocument(reader io.Reader) (Message, error) {
var err error = nil
msg := &CreateIFrameDocument{}
@ -1195,6 +1265,7 @@ func DecodeCreateIFrameDocument(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeAdoptedSSReplaceURLBased(reader io.Reader) (Message, error) {
var err error = nil
msg := &AdoptedSSReplaceURLBased{}
@ -1210,6 +1281,7 @@ func DecodeAdoptedSSReplaceURLBased(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeAdoptedSSReplace(reader io.Reader) (Message, error) {
var err error = nil
msg := &AdoptedSSReplace{}
@ -1222,6 +1294,7 @@ func DecodeAdoptedSSReplace(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeAdoptedSSInsertRuleURLBased(reader io.Reader) (Message, error) {
var err error = nil
msg := &AdoptedSSInsertRuleURLBased{}
@ -1240,6 +1313,7 @@ func DecodeAdoptedSSInsertRuleURLBased(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeAdoptedSSInsertRule(reader io.Reader) (Message, error) {
var err error = nil
msg := &AdoptedSSInsertRule{}
@ -1255,6 +1329,7 @@ func DecodeAdoptedSSInsertRule(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeAdoptedSSDeleteRule(reader io.Reader) (Message, error) {
var err error = nil
msg := &AdoptedSSDeleteRule{}
@ -1267,6 +1342,7 @@ func DecodeAdoptedSSDeleteRule(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeAdoptedSSAddOwner(reader io.Reader) (Message, error) {
var err error = nil
msg := &AdoptedSSAddOwner{}
@ -1279,6 +1355,7 @@ func DecodeAdoptedSSAddOwner(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeAdoptedSSRemoveOwner(reader io.Reader) (Message, error) {
var err error = nil
msg := &AdoptedSSRemoveOwner{}
@ -1291,6 +1368,7 @@ func DecodeAdoptedSSRemoveOwner(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeZustand(reader io.Reader) (Message, error) {
var err error = nil
msg := &Zustand{}
@ -1348,6 +1426,7 @@ func DecodeIOSBatchMeta(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSSessionStart(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSSessionStart{}
@ -1384,6 +1463,7 @@ func DecodeIOSSessionStart(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSSessionEnd(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSSessionEnd{}
@ -1393,6 +1473,7 @@ func DecodeIOSSessionEnd(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSMetadata(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSMetadata{}
@ -1411,6 +1492,7 @@ func DecodeIOSMetadata(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSCustomEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSCustomEvent{}
@ -1429,6 +1511,7 @@ func DecodeIOSCustomEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSUserID(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSUserID{}
@ -1444,6 +1527,7 @@ func DecodeIOSUserID(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSUserAnonymousID(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSUserAnonymousID{}
@ -1459,6 +1543,7 @@ func DecodeIOSUserAnonymousID(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSScreenChanges(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSScreenChanges{}
@ -1483,6 +1568,7 @@ func DecodeIOSScreenChanges(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSCrash(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSCrash{}
@ -1504,6 +1590,7 @@ func DecodeIOSCrash(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSScreenEnter(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSScreenEnter{}
@ -1522,6 +1609,7 @@ func DecodeIOSScreenEnter(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSScreenLeave(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSScreenLeave{}
@ -1540,6 +1628,7 @@ func DecodeIOSScreenLeave(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSClickEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSClickEvent{}
@ -1561,6 +1650,7 @@ func DecodeIOSClickEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSInputEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSInputEvent{}
@ -1582,6 +1672,7 @@ func DecodeIOSInputEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSPerformanceEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSPerformanceEvent{}
@ -1600,6 +1691,7 @@ func DecodeIOSPerformanceEvent(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSLog(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSLog{}
@ -1618,6 +1710,7 @@ func DecodeIOSLog(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSInternalError(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSInternalError{}
@ -1633,6 +1726,7 @@ func DecodeIOSInternalError(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSNetworkCall(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSNetworkCall{}
@ -1666,6 +1760,7 @@ func DecodeIOSNetworkCall(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSPerformanceAggregated(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSPerformanceAggregated{}
@ -1714,6 +1809,7 @@ func DecodeIOSPerformanceAggregated(reader io.Reader) (Message, error) {
return msg, err
}
func DecodeIOSIssueEvent(reader io.Reader) (Message, error) {
var err error = nil
msg := &IOSIssueEvent{}
@ -1735,6 +1831,8 @@ func DecodeIOSIssueEvent(reader io.Reader) (Message, error) {
return msg, err
}
func ReadMessage(t uint64, reader io.Reader) (Message, error) {
switch t {

65
ee/backend/internal/storage/encryptor.go Normal file
View file

@ -0,0 +1,65 @@
package storage
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"errors"
"fmt"
"math/rand"
)
const letterSet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
func GenerateEncryptionKey() []byte {
return append(generateRandomBytes(16), generateRandomBytes(16)...)
}
func generateRandomBytes(size int) []byte {
b := make([]byte, size)
for i := range b {
b[i] = letterSet[rand.Int63()%int64(len(letterSet))]
}
return b
}
func fillLastBlock(rawText []byte, blockSize int) []byte {
padding := blockSize - len(rawText)%blockSize
padText := bytes.Repeat([]byte{byte(padding)}, padding)
return append(rawText, padText...)
}
func EncryptData(data, fullKey []byte) ([]byte, error) {
if len(fullKey) != 32 {
return nil, errors.New("wrong format of encryption key")
}
key, iv := fullKey[:16], fullKey[16:]
// Fill the last block of data by zeros
paddedData := fillLastBlock(data, aes.BlockSize)
// Create new AES cipher with CBC encryptor
block, err := aes.NewCipher(key)
if err != nil {
return nil, fmt.Errorf("cbc encryptor failed: %s", err)
}
mode := cipher.NewCBCEncrypter(block, iv)
// Encrypting data
ciphertext := make([]byte, len(paddedData))
mode.CryptBlocks(ciphertext, paddedData)
// Return encrypted data
return ciphertext, nil
}
func DecryptData(data, fullKey []byte) ([]byte, error) {
if len(fullKey) != 32 {
return nil, errors.New("wrong format of encryption key")
}
key, iv := fullKey[:16], fullKey[16:]
block, err := aes.NewCipher(key)
if err != nil {
return nil, fmt.Errorf("cbc encryptor failed: %s", err)
}
cbc := cipher.NewCBCDecrypter(block, iv)
res := make([]byte, len(data))
cbc.CryptBlocks(res, data)
return res, nil
}

4
ee/backend/pkg/failover/failover.go
View file

@ -89,7 +89,9 @@ func (s *sessionFinderImpl) worker() {
}
func (s *sessionFinderImpl) findSession(sessionID, timestamp, partition uint64) {
err := s.storage.UploadSessionFiles(sessionID)
sessEnd := &messages.SessionEnd{Timestamp: timestamp}
sessEnd.SetSessionID(sessionID)
err := s.storage.UploadSessionFiles(sessEnd)
if err == nil {
log.Printf("found session: %d in partition: %d, original: %d",
sessionID, partition, sessionID%numberOfPartitions)

3
ee/connectors/msgcodec/messages.py
View file

@ -66,8 +66,9 @@ class SessionStart(Message):
class SessionEnd(Message):
__id__ = 3
def __init__(self, timestamp):
def __init__(self, timestamp, encryption_key):
self.timestamp = timestamp
self.encryption_key = encryption_key
class SetPageLocation(Message):

3
ee/connectors/msgcodec/msgcodec.py
View file

@ -125,7 +125,8 @@ class MessageCodec(Codec):
if message_id == 3:
return SessionEnd(
timestamp=self.read_uint(reader)
timestamp=self.read_uint(reader),
encryption_key=self.read_string(reader)
)
if message_id == 4:

1
mobs/messages.rb
View file

@ -48,6 +48,7 @@ end
# end
message 3, 'SessionEnd', :tracker => false, :replayer => false do
uint 'Timestamp'
string 'EncryptionKey'
end
message 4, 'SetPageLocation' do
string 'URL'