jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Devtools separation (#752)

Browse source 
* feat (backend+frontend/player): writing  devtools-related messages into a separate file
This commit is contained in:
Alex K 2022-10-07 16:20:48 +02:00 • committed by GitHub
parent a1bc1bbb78
commit c3fcda45d4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 2839 additions and 2960 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
backend/cmd/sink/main.go
View file

@ -87,8 +87,16 @@ func main() {
// Write encoded message with index to session file
data := msg.EncodeWithIndex()
if err := writer.Write(msg.SessionID(), data); err != nil {
log.Printf("Writer error: %v\n", err)
if messages.IsDOMType(msg.TypeID()) {
if err := writer.WriteDOM(msg.SessionID(), data); err != nil {
log.Printf("DOM Writer error: %v\n", err)
}
}
if !messages.IsDOMType(msg.TypeID()) || msg.TypeID() == messages.MsgTimestamp {
// TODO: write only necessary timestamps
if err := writer.WriteDEV(msg.SessionID(), data); err != nil {
log.Printf("Devtools Writer error: %v\n", err)
}
}
// [METRICS] Increase the number of written to the files messages and the message size

7
backend/cmd/storage/main.go
View file

@ -4,7 +4,6 @@ import (
"log"
"os"
"os/signal"
"strconv"
"syscall"
"time"
@ -44,10 +43,10 @@ func main() {
},
messages.NewMessageIterator(
func(msg messages.Message) {
m := msg.(*messages.SessionEnd)
if err := srv.UploadKey(strconv.FormatUint(msg.SessionID(), 10), 5); err != nil {
sesEnd := msg.(*messages.SessionEnd)
if err := srv.UploadSessionFiles(msg.SessionID()); err != nil {
log.Printf("can't find session: %d", msg.SessionID())
sessionFinder.Find(msg.SessionID(), m.Timestamp)
sessionFinder.Find(msg.SessionID(), sesEnd.Timestamp)
}
// Log timestamp of last processed session
counter.Update(msg.SessionID(), time.UnixMilli(msg.Meta().Batch().Timestamp()))

51
backend/internal/sink/oswriter/oswriter.go
View file

@ -3,6 +3,7 @@ package oswriter
import (
"math"
"os"
"path/filepath"
"strconv"
"time"
)
@ -10,26 +11,26 @@ import (
type Writer struct {
ulimit int
dir string
files map[uint64]*os.File
atimes map[uint64]int64
files map[string]*os.File
atimes map[string]int64
}
func NewWriter(ulimit uint16, dir string) *Writer {
return &Writer{
ulimit: int(ulimit),
dir: dir + "/",
files: make(map[uint64]*os.File),
atimes: make(map[uint64]int64),
files: make(map[string]*os.File),
atimes: make(map[string]int64),
}
}
func (w *Writer) open(key uint64) (*os.File, error) {
file, ok := w.files[key]
func (w *Writer) open(fname string) (*os.File, error) {
file, ok := w.files[fname]
if ok {
return file, nil
}
if len(w.atimes) == w.ulimit {
var m_k uint64
var m_k string
var m_t int64 = math.MaxInt64
for k, t := range w.atimes {
if t < m_t {
@ -37,21 +38,28 @@ func (w *Writer) open(key uint64) (*os.File, error) {
m_t = t
}
}
if err := w.Close(m_k); err != nil {
if err := w.close(m_k); err != nil {
return nil, err
}
}
file, err := os.OpenFile(w.dir+strconv.FormatUint(key, 10), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
// mkdir if not exist
pathTo := w.dir + filepath.Dir(fname)
if _, err := os.Stat(pathTo); os.IsNotExist(err) {
os.MkdirAll(pathTo, 0644)
}
file, err := os.OpenFile(w.dir+fname, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
return nil, err
}
w.files[key] = file
w.atimes[key] = time.Now().Unix()
w.files[fname] = file
w.atimes[fname] = time.Now().Unix()
return file, nil
}
func (w *Writer) Close(key uint64) error {
file := w.files[key]
func (w *Writer) close(fname string) error {
file := w.files[fname]
if file == nil {
return nil
}
@ -61,17 +69,24 @@ func (w *Writer) Close(key uint64) error {
if err := file.Close(); err != nil {
return err
}
delete(w.files, key)
delete(w.atimes, key)
delete(w.files, fname)
delete(w.atimes, fname)
return nil
}
func (w *Writer) Write(key uint64, data []byte) error {
file, err := w.open(key)
func (w *Writer) WriteDOM(sid uint64, data []byte) error {
return w.write(strconv.FormatUint(sid, 10)+"/dom.mob", data)
}
func (w *Writer) WriteDEV(sid uint64, data []byte) error {
return w.write(strconv.FormatUint(sid, 10)+"/devtools.mob", data)
}
func (w *Writer) write(fname string, data []byte) error {
file, err := w.open(fname)
if err != nil {
return err
}
// TODO: add check for the number of recorded bytes to file
_, err = file.Write(data)
return err
}

114
backend/internal/storage/storage.go
View file

@ -16,13 +16,16 @@ import (
)
type Storage struct {
cfg *config.Config
s3 *storage.S3
startBytes []byte
totalSessions syncfloat64.Counter
sessionSize syncfloat64.Histogram
readingTime syncfloat64.Histogram
archivingTime syncfloat64.Histogram
cfg *config.Config
s3 *storage.S3
startBytes []byte
totalSessions syncfloat64.Counter
sessionDOMSize syncfloat64.Histogram
sessionDevtoolsSize syncfloat64.Histogram
readingDOMTime syncfloat64.Histogram
readingTime syncfloat64.Histogram
archivingTime syncfloat64.Histogram
}
func New(cfg *config.Config, s3 *storage.S3, metrics *monitoring.Metrics) (*Storage, error) {
@ -37,10 +40,14 @@ func New(cfg *config.Config, s3 *storage.S3, metrics *monitoring.Metrics) (*Stor
if err != nil {
log.Printf("can't create sessions_total metric: %s", err)
}
sessionSize, err := metrics.RegisterHistogram("sessions_size")
sessionDOMSize, err := metrics.RegisterHistogram("sessions_size")
if err != nil {
log.Printf("can't create session_size metric: %s", err)
}
sessionDevtoolsSize, err := metrics.RegisterHistogram("sessions_dt_size")
if err != nil {
log.Printf("can't create sessions_dt_size metric: %s", err)
}
readingTime, err := metrics.RegisterHistogram("reading_duration")
if err != nil {
log.Printf("can't create reading_duration metric: %s", err)
@ -50,17 +57,30 @@ func New(cfg *config.Config, s3 *storage.S3, metrics *monitoring.Metrics) (*Stor
log.Printf("can't create archiving_duration metric: %s", err)
}
return &Storage{
cfg: cfg,
s3: s3,
startBytes: make([]byte, cfg.FileSplitSize),
totalSessions: totalSessions,
sessionSize: sessionSize,
readingTime: readingTime,
archivingTime: archivingTime,
cfg: cfg,
s3: s3,
startBytes: make([]byte, cfg.FileSplitSize),
totalSessions: totalSessions,
sessionDOMSize: sessionDOMSize,
sessionDevtoolsSize: sessionDevtoolsSize,
readingTime: readingTime,
archivingTime: archivingTime,
}, nil
}
func (s *Storage) UploadKey(key string, retryCount int) error {
func (s *Storage) UploadSessionFiles(sessID uint64) error {
sessionDir := strconv.FormatUint(sessID, 10)
if err := s.uploadKey(sessID, sessionDir+"/dom.mob", true, 5); err != nil {
return err
}
if err := s.uploadKey(sessID, sessionDir+"/devtools.mob", false, 4); err != nil {
return err
}
return nil
}
// TODO: make a bit cleaner
func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCount int) error {
if retryCount <= 0 {
return nil
}
@ -68,7 +88,6 @@ func (s *Storage) UploadKey(key string, retryCount int) error {
start := time.Now()
file, err := os.Open(s.cfg.FSDir + "/" + key)
if err != nil {
sessID, _ := strconv.ParseUint(key, 10, 64)
return fmt.Errorf("File open error: %v; sessID: %s, part: %d, sessStart: %s\n",
err, key, sessID%16,
time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))),
@ -76,33 +95,40 @@ func (s *Storage) UploadKey(key string, retryCount int) error {
}
defer file.Close()
nRead, err := file.Read(s.startBytes)
if err != nil {
sessID, _ := strconv.ParseUint(key, 10, 64)
log.Printf("File read error: %s; sessID: %s, part: %d, sessStart: %s",
err,
key,
sessID%16,
time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))),
)
time.AfterFunc(s.cfg.RetryTimeout, func() {
s.UploadKey(key, retryCount-1)
})
return nil
}
s.readingTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()))
if shouldSplit {
nRead, err := file.Read(s.startBytes)
if err != nil {
log.Printf("File read error: %s; sessID: %s, part: %d, sessStart: %s",
err,
key,
sessID%16,
time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))),
)
time.AfterFunc(s.cfg.RetryTimeout, func() {
s.uploadKey(sessID, key, shouldSplit, retryCount-1)
})
return nil
}
s.readingTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()))
start = time.Now()
startReader := bytes.NewBuffer(s.startBytes[:nRead])
if err := s.s3.Upload(s.gzipFile(startReader), key, "application/octet-stream", true); err != nil {
log.Fatalf("Storage: start upload failed. %v\n", err)
}
if nRead == s.cfg.FileSplitSize {
if err := s.s3.Upload(s.gzipFile(file), key+"e", "application/octet-stream", true); err != nil {
start = time.Now()
startReader := bytes.NewBuffer(s.startBytes[:nRead])
if err := s.s3.Upload(s.gzipFile(startReader), key+"s", "application/octet-stream", true); err != nil {
log.Fatalf("Storage: start upload failed. %v\n", err)
}
if nRead == s.cfg.FileSplitSize {
if err := s.s3.Upload(s.gzipFile(file), key+"e", "application/octet-stream", true); err != nil {
log.Fatalf("Storage: end upload failed. %v\n", err)
}
}
s.archivingTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()))
} else {
start = time.Now()
if err := s.s3.Upload(s.gzipFile(file), key+"s", "application/octet-stream", true); err != nil {
log.Fatalf("Storage: end upload failed. %v\n", err)
}
s.archivingTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()))
}
s.archivingTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()))
// Save metrics
var fileSize float64 = 0
@ -113,8 +139,12 @@ func (s *Storage) UploadKey(key string, retryCount int) error {
fileSize = float64(fileInfo.Size())
}
ctx, _ := context.WithTimeout(context.Background(), time.Millisecond*200)
if shouldSplit {
s.totalSessions.Add(ctx, 1)
s.sessionDOMSize.Record(ctx, fileSize)
} else {
s.sessionDevtoolsSize.Record(ctx, fileSize)
}
s.sessionSize.Record(ctx, fileSize)
s.totalSessions.Add(ctx, 1)
return nil
}

6
backend/pkg/messages/filters.go
View file

@ -2,9 +2,13 @@
package messages
func IsReplayerType(id int) bool {
return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 22 == id || 37 == id || 38 == id || 39 == id || 40 == id || 41 == id || 44 == id || 45 == id || 46 == id || 47 == id || 48 == id || 49 == id || 54 == id || 55 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 79 == id || 127 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id
return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 22 == id || 37 == id || 38 == id || 39 == id || 40 == id || 41 == id || 44 == id || 45 == id || 46 == id || 47 == id || 48 == id || 49 == id || 54 == id || 55 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 79 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id
}
func IsIOSType(id int) bool {
return 107 == id || 90 == id || 91 == id || 92 == id || 93 == id || 94 == id || 95 == id || 96 == id || 97 == id || 98 == id || 99 == id || 100 == id || 101 == id || 102 == id || 103 == id || 104 == id || 105 == id || 110 == id || 111 == id
}
func IsDOMType(id int) bool {
return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 54 == id || 55 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id
}

12
backend/pkg/messages/legacy-message-transform.go
View file

@ -1,14 +1,6 @@
package messages
func transformDeprecated(msg Message) Message {
switch m := msg.(type) {
case *MouseClickDepricated:
return &MouseClick{
ID: m.ID,
HesitationTime: m.HesitationTime,
Label: m.Label,
}
default:
return msg
}
// transform legacy message here if needed
return msg
}

2299
backend/pkg/messages/messages.go
View file

File diff suppressed because it is too large Load diff

2963
backend/pkg/messages/read-message.go
View file

File diff suppressed because it is too large Load diff

17
ee/connectors/msgcodec/messages.py
View file

@ -213,15 +213,6 @@ class MouseMove(Message):
self.y = y
class MouseClickDepricated(Message):
__id__ = 21
def __init__(self, id, hesitation_time, label):
self.id = id
self.hesitation_time = hesitation_time
self.label = label
class ConsoleLog(Message):
__id__ = 22
@ -752,6 +743,14 @@ class Zustand(Message):
self.state = state
class SessionSearch(Message):
__id__ = 127
def __init__(self, timestamp, partition):
self.timestamp = timestamp
self.partition = partition
class IOSBatchMeta(Message):
__id__ = 107

13
ee/connectors/msgcodec/msgcodec.py
View file

@ -237,13 +237,6 @@ class MessageCodec(Codec):
y=self.read_uint(reader)
)
if message_id == 21:
return MouseClickDepricated(
id=self.read_uint(reader),
hesitation_time=self.read_uint(reader),
label=self.read_string(reader)
)
if message_id == 22:
return ConsoleLog(
level=self.read_string(reader),
@ -668,6 +661,12 @@ class MessageCodec(Codec):
state=self.read_string(reader)
)
if message_id == 127:
return SessionSearch(
timestamp=self.read_uint(reader),
partition=self.read_uint(reader)
)
if message_id == 107:
return IOSBatchMeta(
timestamp=self.read_uint(reader),

2
frontend/app/components/Session_/Player/Controls/Controls.js
View file

@ -113,7 +113,6 @@ function getStorageName(type) {
graphqlCount: state.graphqlListNow.length,
exceptionsCount: state.exceptionsListNow.length,
showExceptions: state.exceptionsList.length > 0,
showLongtasks: state.longtasksList.length > 0,
liveTimeTravel: state.liveTimeTravel,
}))
@connect(
@ -181,7 +180,6 @@ export default class Controls extends React.Component {
nextProps.graphqlCount !== this.props.graphqlCount ||
nextProps.showExceptions !== this.props.showExceptions ||
nextProps.exceptionsCount !== this.props.exceptionsCount ||
nextProps.showLongtasks !== this.props.showLongtasks ||
nextProps.liveTimeTravel !== this.props.liveTimeTravel ||
nextProps.skipInterval !== this.props.skipInterval
)

2
frontend/app/player/MessageDistributor/Lists.ts
View file

@ -1,7 +1,7 @@
import type { Message } from './messages'
import ListWalker from './managers/ListWalker';
export const LIST_NAMES = ["redux", "mobx", "vuex", "zustand", "ngrx", "graphql", "exceptions", "profiles", "longtasks"] as const;
export const LIST_NAMES = ["redux", "mobx", "vuex", "zustand", "ngrx", "graphql", "exceptions", "profiles"] as const;
export const INITIAL_STATE = {}
LIST_NAMES.forEach(name => {

185
frontend/app/player/MessageDistributor/MessageDistributor.ts
View file

@ -27,7 +27,7 @@ import ActivityManager from './managers/ActivityManager';
import AssistManager from './managers/AssistManager';
import MFileReader from './messages/MFileReader';
import { loadFiles, checkUnprocessedMobs } from './network/loadFiles';
import { loadFiles, requestEFSDom, requestEFSDevtools } from './network/loadFiles';
import { INITIAL_STATE as SUPER_INITIAL_STATE, State as SuperState } from './StatedScreen/StatedScreen';
import { INITIAL_STATE as ASSIST_INITIAL_STATE, State as AssistState } from './managers/AssistManager';
@ -49,6 +49,7 @@ export interface State extends SuperState, AssistState {
domBuildingTime?: any,
loadTime?: any,
error: boolean,
devtoolsLoading: boolean
}
export const INITIAL_STATE: State = {
...SUPER_INITIAL_STATE,
@ -56,7 +57,8 @@ export const INITIAL_STATE: State = {
...ASSIST_INITIAL_STATE,
performanceChartData: [],
skipIntervals: [],
error: false
error: false,
devtoolsLoading: false,
};
@ -100,12 +102,11 @@ export default class MessageDistributor extends StatedScreen {
private readonly lists = initLists();
private activityManager: ActivityManager | null = null;
private fileReader: MFileReader;
private sessionStart: number;
private navigationStartOffset: number = 0;
private lastMessageTime: number = 0;
private lastRecordedMessageTime: number = 0;
private lastMessageInFileTime: number = 0;
constructor(private readonly session: any /*Session*/, config: any, live: boolean) {
super();
@ -143,43 +144,19 @@ export default class MessageDistributor extends StatedScreen {
}
}
private waitingForFiles: boolean = false
private onFileSuccessRead() {
this.windowNodeCounter.reset()
if (this.activityManager) {
this.activityManager.end()
update({
skipIntervals: this.activityManager.list
})
}
this.waitingForFiles = false
this.setMessagesLoading(false)
}
private readAndDistributeMessages(byteArray: Uint8Array, onReadCb?: (msg: Message) => void) {
private parseAndDistributeMessages(fileReader: MFileReader, onMessage?: (msg: Message) => void) {
const msgs: Array<Message> = []
if (!this.fileReader) {
this.fileReader = new MFileReader(new Uint8Array(), this.sessionStart)
}
this.fileReader.append(byteArray)
let next: ReturnType<MFileReader['next']>
while (next = this.fileReader.next()) {
while (next = fileReader.next()) {
const [msg, index] = next
this.distributeMessage(msg, index)
msgs.push(msg)
onReadCb?.(msg)
onMessage?.(msg)
}
logger.info("Messages count: ", msgs.length, msgs)
return msgs
}
private processStateUpdates(msgs: Message[]) {
// @ts-ignore Hack for upet (TODO: fix ordering in one mutation in tracker(removes first))
const headChildrenIds = msgs.filter(m => m.parentID === 1).map(m => m.id);
this.pagesManager.sortPages((m1, m2) => {
@ -204,7 +181,11 @@ export default class MessageDistributor extends StatedScreen {
}
return 0;
})
}
private waitingForFiles: boolean = false
private onFileReadSuccess = () => {
const stateToUpdate: {[key:string]: any} = {
performanceChartData: this.performanceTrackManager.chartData,
performanceAvaliability: this.performanceTrackManager.avaliability,
@ -212,79 +193,87 @@ export default class MessageDistributor extends StatedScreen {
LIST_NAMES.forEach(key => {
stateToUpdate[ `${ key }List` ] = this.lists[ key ].list
})
if (this.activityManager) {
this.activityManager.end()
stateToUpdate.skipIntervals = this.activityManager.list
}
update(stateToUpdate)
}
private onFileReadFailed = (e: any) => {
logger.error(e)
update({ error: true })
toast.error('Error requesting a session file')
}
private onFileReadFinally = () => {
this.waitingForFiles = false
this.setMessagesLoading(false)
}
private loadMessages() {
private loadMessages() {
const createNewParser = () => {
// Each time called - new fileReader created
const fileReader = new MFileReader(new Uint8Array(), this.sessionStart)
return (b: Uint8Array) => {
fileReader.append(b)
this.parseAndDistributeMessages(fileReader)
this.setMessagesLoading(false)
}
}
this.setMessagesLoading(true)
this.waitingForFiles = true
const onData = (byteArray: Uint8Array) => {
const msgs = this.readAndDistributeMessages(byteArray)
this.processStateUpdates(msgs)
}
loadFiles(this.session.mobsUrl,
onData
loadFiles(this.session.domURL, createNewParser())
.catch(() => // do if only the first file missing (404) (?)
requestEFSDom(this.session.sessionId)
.then(createNewParser())
// Fallback to back Compatability with mobsUrl
.catch(e =>
loadFiles(this.session.mobsUrl, createNewParser())
)
)
.then(() => this.onFileSuccessRead())
.catch(async () => {
checkUnprocessedMobs(this.session.sessionId)
.then(file => file ? onData(file) : Promise.reject('No session file'))
.then(() => this.onFileSuccessRead())
.catch((e) => {
logger.error(e)
update({ error: true })
toast.error('Error getting a session replay file')
})
.finally(() => {
this.waitingForFiles = false
this.setMessagesLoading(false)
})
.then(this.onFileReadSuccess)
.catch(this.onFileReadFailed)
.finally(this.onFileReadFinally)
})
// load devtools
update({ devtoolsLoading: true })
loadFiles(this.session.devtoolsURL, createNewParser())
.catch(() =>
requestEFSDevtools(this.session.sessionId)
.then(createNewParser())
)
//.catch() // not able to download the devtools file
.finally(() => update({ devtoolsLoading: false }))
}
public async reloadWithUnprocessedFile() {
// assist will pause and skip messages to prevent timestamp related errors
this.assistManager.toggleTimeTravelJump()
this.reloadMessageManagers()
this.setMessagesLoading(true)
this.waitingForFiles = true
reloadWithUnprocessedFile() {
const onData = (byteArray: Uint8Array) => {
const onReadCallback = () => this.setLastRecordedMessageTime(this.lastMessageTime)
const msgs = this.readAndDistributeMessages(byteArray, onReadCallback)
this.processStateUpdates(msgs)
const onMessage = (msg: Message) => { this.lastMessageInFileTime = msg.time }
this.parseAndDistributeMessages(new MFileReader(byteArray, this.sessionStart), onMessage)
}
// unpausing assist
const unpauseAssist = () => {
this.assistManager.toggleTimeTravelJump()
const updateState = () =>
update({
liveTimeTravel: true,
});
}
try {
const unprocessedFile = await checkUnprocessedMobs(this.session.sessionId)
Promise.resolve(onData(unprocessedFile))
.then(() => this.onFileSuccessRead())
.then(unpauseAssist)
} catch (unprocessedFilesError) {
logger.error(unprocessedFilesError)
update({ error: true })
toast.error('Error getting a session replay file')
this.assistManager.toggleTimeTravelJump()
} finally {
this.waitingForFiles = false
this.setMessagesLoading(false)
}
// assist will pause and skip messages to prevent timestamp related errors
this.assistManager.toggleTimeTravelJump()
this.reloadMessageManagers()
this.windowNodeCounter.reset()
this.setMessagesLoading(true)
this.waitingForFiles = true
return requestEFSDom(this.session.sessionId)
.then(onData)
.then(updateState)
.then(this.onFileReadSuccess)
.catch(this.onFileReadFailed)
.finally(this.onFileReadFinally)
.then(() => {
this.assistManager.toggleTimeTravelJump()
})
}
private reloadMessageManagers() {
@ -381,7 +370,7 @@ export default class MessageDistributor extends StatedScreen {
}
}
private decodeMessage(msg: any, keys: Array<string>) {
private decodeStateMessage(msg: any, keys: Array<string>) {
const decoded = {};
try {
keys.forEach(key => {
@ -461,34 +450,34 @@ export default class MessageDistributor extends StatedScreen {
this.decoder.set(msg.key, msg.value);
break;
case "redux":
decoded = this.decodeMessage(msg, ["state", "action"]);
decoded = this.decodeStateMessage(msg, ["state", "action"]);
logger.log(decoded)
if (decoded != null) {
this.lists.redux.append(decoded);
}
break;
case "ng_rx":
decoded = this.decodeMessage(msg, ["state", "action"]);
decoded = this.decodeStateMessage(msg, ["state", "action"]);
logger.log(decoded)
if (decoded != null) {
this.lists.ngrx.append(decoded);
}
break;
case "vuex":
decoded = this.decodeMessage(msg, ["state", "mutation"]);
decoded = this.decodeStateMessage(msg, ["state", "mutation"]);
logger.log(decoded)
if (decoded != null) {
this.lists.vuex.append(decoded);
}
break;
case "zustand":
decoded = this.decodeMessage(msg, ["state", "mutation"])
decoded = this.decodeStateMessage(msg, ["state", "mutation"])
logger.log(decoded)
if (decoded != null) {
this.lists.zustand.append(decoded)
}
case "mob_x":
decoded = this.decodeMessage(msg, ["payload"]);
decoded = this.decodeStateMessage(msg, ["payload"]);
logger.log(decoded)
if (decoded != null) {
@ -501,12 +490,6 @@ export default class MessageDistributor extends StatedScreen {
case "profiler":
this.lists.profiles.append(msg);
break;
case "long_task":
this.lists.longtasks.append({
...msg,
time: msg.timestamp - this.sessionStart,
});
break;
default:
switch (msg.tp) {
case "create_document":
@ -548,11 +531,7 @@ export default class MessageDistributor extends StatedScreen {
this.assistManager.clear();
}
public setLastRecordedMessageTime(time: number) {
this.lastRecordedMessageTime = time;
}
public getLastRecordedMessageTime(): number {
return this.lastRecordedMessageTime;
getLastRecordedMessageTime(): number {
return this.lastMessageInFileTime;
}
}

15
frontend/app/player/MessageDistributor/messages/MFileReader.ts
View file

@ -18,12 +18,20 @@ export default class MFileReader extends RawMessageReader {
if (this.p === 0) return false
for (let i = 7; i >= 0; i--) {
if (this.buf[ this.p + i ] !== this.buf[ this.pLastMessageID + i ]) {
return this.buf[ this.p + i ] - this.buf[ this.pLastMessageID + i ] < 0
return this.buf[ this.p + i ] < this.buf[ this.pLastMessageID + i ]
}
}
return false
}
private getLastMessageID(): number {
let id = 0
for (let i = 0; i< 8; i++) {
id += this.buf[ this.p + i ] * 2**(8*i)
}
return id
}
private readRawMessage(): RawMessage | null {
this.skip(8)
try {
@ -67,11 +75,12 @@ export default class MFileReader extends RawMessageReader {
return this.next()
}
const index = this.getLastMessageID()
const msg = Object.assign(rMsg, {
time: this.currentTime,
_index: this.pLastMessageID,
_index: index,
})
return [msg, this.pLastMessageID]
return [msg, index]
}
}

38
frontend/app/player/MessageDistributor/network/loadFiles.ts
View file

@ -3,13 +3,11 @@ import APIClient from 'App/api_client';
const NO_NTH_FILE = "nnf"
const NO_UNPROCESSED_FILES = "nuf"
const getUnprocessedFileLink = (sessionId: string) => '/unprocessed/' + sessionId
type onDataCb = (data: Uint8Array) => void
export const loadFiles = (
urls: string[],
onData: onDataCb,
onData: onDataCb,
): Promise<void> => {
const firstFileURL = urls[0]
urls = urls.slice(1)
@ -41,28 +39,32 @@ export const loadFiles = (
})
}
export const checkUnprocessedMobs = async (sessionId: string) => {
try {
const api = new APIClient()
const res = await api.fetch(getUnprocessedFileLink(sessionId))
if (res.status >= 400) {
throw NO_UNPROCESSED_FILES
}
const byteArray = await processAPIStreamResponse(res, false)
return byteArray
} catch (e) {
throw e
}
export async function requestEFSDom(sessionId: string) {
return await requestEFSMobFile(sessionId, "dom.mob")
}
const processAPIStreamResponse = (response: Response, isFirstFile: boolean) => {
export async function requestEFSDevtools(sessionId: string) {
return await requestEFSMobFile(sessionId, "devtools.mob")
}
async function requestEFSMobFile(sessionId: string, filename: string) {
const api = new APIClient()
const res = await api.fetch('/unprocessed/' + sessionId + '/' + filename)
if (res.status >= 400) {
throw NO_UNPROCESSED_FILES
}
return await processAPIStreamResponse(res, false)
}
const processAPIStreamResponse = (response: Response, isMainFile: boolean) => {
return new Promise<ArrayBuffer>((res, rej) => {
if (response.status === 404 && !isFirstFile) {
if (response.status === 404 && !isMainFile) {
return rej(NO_NTH_FILE)
}
if (response.status >= 400) {
return rej(
isFirstFile ? `no start file. status code ${ response.status }`
isMainFile ? `no start file. status code ${ response.status }`
: `Bad endfile status code ${response.status}`
)
}

7
frontend/app/types/session/session.ts
View file

@ -40,7 +40,8 @@ export default Record({
filterId: '',
messagesUrl: '',
domURL: [],
mobsUrl: [],
devtoolsURL: [],
mobsUrl: [], // @depricated
userBrowser: '',
userBrowserVersion: '?',
userCountry: '',
@ -95,6 +96,7 @@ export default Record({
sessionId,
sessionID,
domURL = [],
devtoolsURL= [],
mobsUrl = [],
notes = [],
...session
@ -166,8 +168,9 @@ export default Record({
issues: issuesList,
sessionId: sessionId || sessionID,
userId: session.userId || session.userID,
domURL: Array.isArray(domURL) ? domURL : [ domURL ],
mobsUrl: Array.isArray(mobsUrl) ? mobsUrl : [ mobsUrl ],
domURL,
devtoolsURL,
notes,
notesWithEvents: List(notesWithEvents),
};

34
mobs/messages.rb
View file

@ -125,13 +125,8 @@ message 20, 'MouseMove' do
uint 'X'
uint 'Y'
end
# Depricated since OpenReplay 1.2.0 (tracker version?)
message 21, 'MouseClickDepricated', :tracker => false, :replayer => false do
uint 'ID'
uint 'HesitationTime'
string 'Label'
end
message 22, 'ConsoleLog' do
# 21
message 22, 'ConsoleLog', :replayer => :devtools do
string 'Level'
string 'Value'
end
@ -250,7 +245,7 @@ message 38, 'CSSDeleteRule' do
uint 'Index'
end
message 39, 'Fetch' do
message 39, 'Fetch', :replayer => :devtools do
string 'Method'
string 'URL'
string 'Request'
@ -259,16 +254,17 @@ message 39, 'Fetch' do
uint 'Timestamp'
uint 'Duration'
end
message 40, 'Profiler' do
message 40, 'Profiler', :replayer => :devtools do
string 'Name'
uint 'Duration'
string 'Args'
string 'Result'
end
message 41, 'OTable' do
message 41, 'OTable', :replayer => :devtools do
string 'Key'
string 'Value'
end
# Do we use that?
message 42, 'StateAction', :replayer => false do
string 'Type'
end
@ -277,36 +273,37 @@ message 43, 'StateActionEvent', :tracker => false, :replayer => false do
uint 'Timestamp'
string 'Type'
end
message 44, 'Redux' do
message 44, 'Redux', :replayer => :devtools do
string 'Action'
string 'State'
uint 'Duration'
end
message 45, 'Vuex' do
message 45, 'Vuex', :replayer => :devtools do
string 'Mutation'
string 'State'
end
message 46, 'MobX' do
message 46, 'MobX', :replayer => :devtools do
string 'Type'
string 'Payload'
end
message 47, 'NgRx' do
message 47, 'NgRx', :replayer => :devtools do
string 'Action'
string 'State'
uint 'Duration'
end
message 48, 'GraphQL' do
message 48, 'GraphQL', :replayer => :devtools do
string 'OperationKind'
string 'OperationName'
string 'Variables'
string 'Response'
end
message 49, 'PerformanceTrack' do
message 49, 'PerformanceTrack' do #, :replayer => :devtools --> requires player performance refactoring (now is tied with nodes counter)
int 'Frames'
int 'Ticks'
uint 'TotalJSHeapSize'
uint 'UsedJSHeapSize'
end
# next 2 should be removed after refactoring backend/pkg/handlers/custom/eventMapper.go (move "wrapping" logic to pg connector insertion)
message 50, 'GraphQLEvent', :tracker => false, :replayer => false do
uint 'MessageID'
uint 'Timestamp'
@ -362,6 +359,7 @@ message 56, 'PerformanceTrackAggr', :tracker => false, :replayer => false do
uint 'MaxUsedJSHeapSize'
end
## 57 58
#Depricated (since 3.0.?)
message 59, 'LongTask' do
uint 'Timestamp'
uint 'Duration'
@ -464,12 +462,12 @@ end
# string 'Styles'
# string 'BaseURL'
# end
message 79, 'Zustand' do
message 79, 'Zustand', :replayer => :devtools do
string 'Mutation'
string 'State'
end
message 127, 'SessionSearch' do
message 127, 'SessionSearch', :tracker => false, :replayer => false do
uint 'Timestamp'
uint 'Partition'
end

6
mobs/templates/backend~pkg~messages~filters.go.erb
View file

@ -2,9 +2,13 @@
package messages
func IsReplayerType(id int) bool {
return <%= $messages.select { |msg| msg.replayer }.map{ |msg| "#{msg.id} == id" }.join(' || ') %>
return <%= $messages.select { |msg| msg.replayer != false }.map{ |msg| "#{msg.id} == id" }.join(' || ') %>
}
func IsIOSType(id int) bool {
return <%= $messages.select { |msg| msg.context == :ios }.map{ |msg| "#{msg.id} == id"}.join(' || ') %>
}
func IsDOMType(id int) bool {
return <%= $messages.select { |msg| msg.replayer == true }.map{ |msg| "#{msg.id} == id" }.join(' || ') %>
}

4
mobs/templates/frontend~app~player~MessageDistributor~messages~RawMessageReader.ts.erb
View file

@ -17,7 +17,7 @@ export default class RawMessageReader extends PrimitiveReader {
if (tp === null) { return resetPointer() }
switch (tp) {
<% $messages.select { |msg| msg.replayer }.each do |msg| %>
<% $messages.select { |msg| msg.replayer != false }.each do |msg| %>
case <%= msg.id %>: {
<%= msg.attributes.map { |attr|
" const #{attr.name.camel_case} = this.read#{attr.type.to_s.pascal_case}(); if (#{attr.name.camel_case} === null) { return resetPointer() }" }.join "\n" %>
@ -27,7 +27,7 @@ export default class RawMessageReader extends PrimitiveReader {
" #{attr.name.camel_case}," }.join "\n" %>
};
}
<% end %>
<% end %>
default:
throw new Error(`Unrecognizable message type: ${ tp }; Pointer at the position ${this.p} of ${this.buf.length}`)
return null;

4
mobs/templates/frontend~app~player~MessageDistributor~messages~message.ts.erb
View file

@ -4,11 +4,11 @@
import type { Timed } from './timed'
import type { RawMessage } from './raw'
import type {
<%= $messages.select { |msg| msg.replayer }.map { |msg| " Raw#{msg.name.snake_case.pascal_case}," }.join "\n" %>
<%= $messages.select { |msg| msg.replayer != false }.map { |msg| " Raw#{msg.name.snake_case.pascal_case}," }.join "\n" %>
} from './raw'
export type Message = RawMessage & Timed
<% $messages.select { |msg| msg.replayer }.each do |msg| %>
<% $messages.select { |msg| msg.replayer != false }.each do |msg| %>
export type <%= msg.name.snake_case.pascal_case %> = Raw<%= msg.name.snake_case.pascal_case %> & Timed
<% end %>

4
mobs/templates/frontend~app~player~MessageDistributor~messages~raw.ts.erb
View file

@ -1,11 +1,11 @@
// Auto-generated, do not edit
/* eslint-disable */
<% $messages.select { |msg| msg.replayer }.each do |msg| %>
<% $messages.select { |msg| msg.replayer != false }.each do |msg| %>
export interface Raw<%= msg.name.snake_case.pascal_case %> {
tp: "<%= msg.name.snake_case %>",
<%= msg.attributes.map { |attr| " #{attr.name.camel_case}: #{attr.type_js}," }.join "\n" %>
}
<% end %>
export type RawMessage = <%= $messages.select { |msg| msg.replayer }.map { |msg| "Raw#{msg.name.snake_case.pascal_case}" }.join " | " %>;
export type RawMessage = <%= $messages.select { |msg| msg.replayer != false }.map { |msg| "Raw#{msg.name.snake_case.pascal_case}" }.join " | " %>;

2
mobs/templates/frontend~app~player~MessageDistributor~messages~tracker-legacy.ts.erb
View file

@ -3,5 +3,5 @@
// Auto-generated, do not edit
export const TP_MAP = {
<%= $messages.select { |msg| msg.tracker || msg.replayer }.map { |msg| " #{msg.id}: \"#{msg.name.snake_case}\"," }.join "\n" %>
<%= $messages.select { |msg| msg.tracker || msg.replayer != false }.map { |msg| " #{msg.id}: \"#{msg.name.snake_case}\"," }.join "\n" %>
} as const

2
mobs/templates/frontend~app~player~MessageDistributor~messages~tracker.ts.erb
View file

@ -14,7 +14,7 @@ export type TrackerMessage = <%= $messages.select { |msg| msg.tracker }.map { |m
export default function translate(tMsg: TrackerMessage): RawMessage | null {
switch(tMsg[0]) {
<% $messages.select { |msg| msg.replayer & msg.tracker }.each do |msg| %>
<% $messages.select { |msg| msg.replayer != false && msg.tracker }.each do |msg| %>
case <%= msg.id %>: {
return {
tp: "<%= msg.name.snake_case %>",