jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'backend' into dev

Browse source
This commit is contained in:
ShiKhu 2022-04-28 17:03:25 +02:00
commit c2d1bcdb35
51 changed files with 3105 additions and 3155 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
backend/Dockerfile.bundle
View file

@ -1,4 +1,4 @@
FROM golang:1.13-alpine3.10 AS prepare
FROM golang:1.18-alpine3.15 AS prepare
RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash
@ -13,7 +13,7 @@ FROM prepare AS build
COPY pkg pkg
COPY services services
RUN for name in alerts assets db ender http integrations sink storage;do CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o bin/$name -tags musl openreplay/backend/services/$name; done
RUN for name in assets db ender http integrations sink storage;do CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o bin/$name -tags musl openreplay/backend/services/$name; done
FROM alpine
@ -26,8 +26,9 @@ ENV TZ=UTC \
MAXMINDDB_FILE=/root/geoip.mmdb \
UAPARSER_FILE=/root/regexes.yaml \
HTTP_PORT=80 \
BEACON_SIZE_LIMIT=1000000 \
BEACON_SIZE_LIMIT=7000000 \
KAFKA_USE_SSL=true \
KAFKA_MAX_POLL_INTERVAL_MS=400000 \
REDIS_STREAMS_MAX_LEN=3000 \
TOPIC_RAW_WEB=raw \
TOPIC_RAW_IOS=raw-ios \
@ -42,10 +43,10 @@ ENV TZ=UTC \
AWS_REGION_WEB=eu-central-1 \
AWS_REGION_IOS=eu-west-1 \
AWS_REGION_ASSETS=eu-central-1 \
CACHE_ASSETS=false \
CACHE_ASSETS=true \
ASSETS_SIZE_LIMIT=6291456 \
FS_CLEAN_HRS=12
FS_CLEAN_HRS=12 \
LOG_QUEUE_STATS_INTERVAL_SEC=60
RUN mkdir $FS_DIR
#VOLUME [ $FS_DIR ] # Uncomment in case of using Bind mount.

25
backend/pkg/dev/profiling/profiling.go
View file

@ -1,24 +1,23 @@
package profiling
import (
"log"
"net/http"
"github.com/gorilla/mux"
_ "net/http/pprof"
)
"github.com/gorilla/mux"
"log"
"net/http"
_ "net/http/pprof"
)
func Profile() {
go func() {
router := mux.NewRouter()
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
log.Println("Starting profiler...")
if err := http.ListenAndServe(":6060", router); err != nil {
panic(err)
}
router := mux.NewRouter()
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
log.Println("Starting profiler...")
if err := http.ListenAndServe(":6060", router); err != nil {
panic(err)
}
}()
}
/*
docker run -p 6060:6060 -e REQUIRED_ENV=http://value -e ANOTHER_ENV=anothervalue workername
@ -34,4 +33,4 @@ go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30
THEN
https://www.speedscope.app/
*/
*/

2
backend/pkg/env/aws.go vendored
View file

@ -19,7 +19,7 @@ func AWSSessionOnRegion(region string) *_session.Session {
if AWS_ENDPOINT != "" {
config.Endpoint = aws.String(AWS_ENDPOINT)
config.DisableSSL = aws.Bool(true)
config.S3ForcePathStyle = aws.Bool(true)
config.S3ForcePathStyle = aws.Bool(true)
}
aws_session, err := _session.NewSession(config)
if err != nil {

7
backend/pkg/env/vars.go vendored
View file

@ -22,7 +22,7 @@ func Uint64(key string) uint64 {
v := String(key)
n, err := strconv.ParseUint(v, 10, 64)
if err != nil {
log.Fatalln(key + " has a wrong value. ", err)
log.Fatalln(key+" has a wrong value. ", err)
}
return n
}
@ -31,12 +31,13 @@ func Uint16(key string) uint16 {
v := String(key)
n, err := strconv.ParseUint(v, 10, 16)
if err != nil {
log.Fatalln(key + " has a wrong value. ", err)
log.Fatalln(key+" has a wrong value. ", err)
}
return uint16(n)
}
const MAX_INT = uint64(^uint(0) >> 1)
func Int(key string) int {
val := Uint64(key)
if val > MAX_INT {
@ -54,4 +55,4 @@ func Bool(key string) bool {
return true
}
return false
}
}

4
backend/pkg/env/worker-id.go vendored
View file

@ -5,9 +5,9 @@ import (
)
func hashHostname(hostname string) uint16 {
var h uint16 ;
var h uint16
for i, b := range hostname {
h += uint16(i+1)*uint16(b)
h += uint16(i+1) * uint16(b)
}
return h
}

2
backend/pkg/flakeid/flakeid.go
View file

@ -8,7 +8,7 @@ const (
TIMESTAMP_MAX = 1<<TIMESTAMP_SIZE - 1
TIMESTAMP_SHIFT = SEQ_ID_SIZE + SHARD_ID_SHIFT
SHARD_ID_SHIFT = SEQ_ID_SIZE
EPOCH = 1550000000000
EPOCH = 1550000000000
)
func compose(timestamp uint64, shardID uint16, seqID byte) uint64 {

1
backend/pkg/hashid/hashid.go
View file

@ -16,7 +16,6 @@ func IssueID(projectID uint32, e *messages.IssueEvent) string {
return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil))
}
func IOSCrashID(projectID uint32, crash *messages.IOSCrash) string {
hash := fnv.New128a()
hash.Write([]byte(crash.Name))

6
backend/pkg/intervals/intervals.go
View file

@ -5,7 +5,7 @@ const HEARTBEAT_INTERVAL = 2 * 60 * 1000
const INTEGRATIONS_REQUEST_INTERVAL = 1 * 60 * 1000
const EVENTS_PAGE_EVENT_TIMEOUT = 2 * 60 * 1000
const EVENTS_INPUT_EVENT_TIMEOUT = 2 * 60 * 1000
const EVENTS_PERFORMANCE_AGGREGATION_TIMEOUT = 2 * 60 * 1000
const EVENTS_SESSION_END_TIMEOUT = HEARTBEAT_INTERVAL + 30 * 1000
const EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS = HEARTBEAT_INTERVAL + 3 * 60 * 1000
const EVENTS_PERFORMANCE_AGGREGATION_TIMEOUT = 2 * 60 * 1000
const EVENTS_SESSION_END_TIMEOUT = HEARTBEAT_INTERVAL + 30*1000
const EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS = HEARTBEAT_INTERVAL + 3*60*1000
const EVENTS_BACK_COMMIT_GAP = EVENTS_SESSION_END_TIMEOUT_WITH_INTEGRATIONS + 1*60*1000

99
backend/pkg/log/queue.go
View file

@ -1,77 +1,72 @@
package log
import (
"time"
"fmt"
"log"
"fmt"
"log"
"time"
"openreplay/backend/pkg/messages"
"openreplay/backend/pkg/queue/types"
//"openreplay/backend/pkg/env"
"openreplay/backend/pkg/messages"
"openreplay/backend/pkg/queue/types"
//"openreplay/backend/pkg/env"
)
type partitionStats struct {
maxts int64
mints int64
lastts int64
lastID uint64
count int
maxts int64
mints int64
lastts int64
lastID uint64
count int
}
type queueStats struct {
prts map[int32]*partitionStats
tick <-chan time.Time
prts map[int32]*partitionStats
tick <-chan time.Time
}
func NewQueueStats(sec int)*queueStats {
return &queueStats{
prts: make(map[int32]*partitionStats),
tick: time.Tick(time.Duration(sec) * time.Second),
}
func NewQueueStats(sec int) *queueStats {
return &queueStats{
prts: make(map[int32]*partitionStats),
tick: time.Tick(time.Duration(sec) * time.Second),
}
}
func (qs *queueStats) HandleAndLog(sessionID uint64, m *types.Meta) {
prti := int32(sessionID % 16) // TODO use GetKeyPartition from kafka/key.go
prt, ok := qs.prts[prti]
if !ok {
qs.prts[prti] = &partitionStats{}
prt = qs.prts[prti]
}
prti := int32(sessionID % 16) // TODO use GetKeyPartition from kafka/key.go
prt, ok := qs.prts[prti]
if !ok {
qs.prts[prti] = &partitionStats{}
prt = qs.prts[prti]
}
if prt.maxts < m.Timestamp {
prt.maxts = m.Timestamp
}
if prt.mints > m.Timestamp || prt.mints == 0 {
prt.mints = m.Timestamp
}
prt.lastts = m.Timestamp
prt.lastID = m.ID
prt.count += 1
if prt.maxts < m.Timestamp {
prt.maxts = m.Timestamp
}
if prt.mints > m.Timestamp || prt.mints == 0 {
prt.mints = m.Timestamp
}
prt.lastts = m.Timestamp
prt.lastID = m.ID
prt.count += 1
select {
case <-qs.tick:
qs.LogThenReset()
default:
}
select {
case <-qs.tick:
qs.LogThenReset()
default:
}
}
func (qs *queueStats) LogThenReset() {
s := "Queue Statistics: "
for i, p := range qs.prts {
s = fmt.Sprintf("%v | %v:: lastTS %v, lastID %v, count %v, maxTS %v, minTS %v",
s, i, p.lastts, p.lastID, p.count, p.maxts, p.mints)
}
log.Println(s)
// reset
qs.prts = make(map[int32]*partitionStats)
s := "Queue Statistics: "
for i, p := range qs.prts {
s = fmt.Sprintf("%v | %v:: lastTS %v, lastID %v, count %v, maxTS %v, minTS %v",
s, i, p.lastts, p.lastID, p.count, p.maxts, p.mints)
}
log.Println(s)
// reset
qs.prts = make(map[int32]*partitionStats)
}
// TODO: list of message id to log (mb filter function with callback in messages/utils.go or something)
func LogMessage(s string, sessionID uint64, msg messages.Message, m *types.Meta) {
log.Printf("%v | SessionID: %v, Queue info: %v, Message: %v", s, sessionID, m, msg)
log.Printf("%v | SessionID: %v, Queue info: %v, Message: %v", s, sessionID, m, msg)
}

2
backend/pkg/messages/facade.go
View file

@ -36,6 +36,6 @@ func Encode(msg Message) []byte {
// }
func GetMessageTypeID(b []byte) (uint64, error) {
reader := bytes.NewReader(b)
reader := bytes.NewReader(b)
return ReadUint(reader)
}

3
backend/pkg/messages/filters.go
View file

@ -1,9 +1,8 @@
// Auto-generated, do not edit
package messages
func IsReplayerType(id uint64) bool {
return 0 == id || 2 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 22 == id || 37 == id || 38 == id || 39 == id || 40 == id || 41 == id || 44 == id || 45 == id || 46 == id || 47 == id || 48 == id || 49 == id || 54 == id || 55 == id || 59 == id || 69 == id || 70 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id
return 0 == id || 2 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 22 == id || 37 == id || 38 == id || 39 == id || 40 == id || 41 == id || 44 == id || 45 == id || 46 == id || 47 == id || 48 == id || 49 == id || 54 == id || 55 == id || 59 == id || 69 == id || 70 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id
}
func IsIOSType(id uint64) bool {

118
backend/pkg/messages/get-timestamp.go
View file

@ -1,65 +1,63 @@
// Auto-generated, do not edit
package messages
func GetTimestamp(message Message) uint64 {
switch msg := message.(type) {
case *IOSBatchMeta:
return msg.Timestamp
case *IOSSessionStart:
return msg.Timestamp
case *IOSSessionEnd:
return msg.Timestamp
case *IOSMetadata:
return msg.Timestamp
case *IOSCustomEvent:
return msg.Timestamp
case *IOSUserID:
return msg.Timestamp
case *IOSUserAnonymousID:
return msg.Timestamp
case *IOSScreenChanges:
return msg.Timestamp
case *IOSCrash:
return msg.Timestamp
case *IOSScreenEnter:
return msg.Timestamp
case *IOSScreenLeave:
return msg.Timestamp
case *IOSClickEvent:
return msg.Timestamp
case *IOSInputEvent:
return msg.Timestamp
case *IOSPerformanceEvent:
return msg.Timestamp
case *IOSLog:
return msg.Timestamp
case *IOSInternalError:
return msg.Timestamp
case *IOSNetworkCall:
return msg.Timestamp
case *IOSIssueEvent:
return msg.Timestamp
}
return uint64(message.Meta().Timestamp)
}
switch msg := message.(type) {
case *IOSBatchMeta:
return msg.Timestamp
case *IOSSessionStart:
return msg.Timestamp
case *IOSSessionEnd:
return msg.Timestamp
case *IOSMetadata:
return msg.Timestamp
case *IOSCustomEvent:
return msg.Timestamp
case *IOSUserID:
return msg.Timestamp
case *IOSUserAnonymousID:
return msg.Timestamp
case *IOSScreenChanges:
return msg.Timestamp
case *IOSCrash:
return msg.Timestamp
case *IOSScreenEnter:
return msg.Timestamp
case *IOSScreenLeave:
return msg.Timestamp
case *IOSClickEvent:
return msg.Timestamp
case *IOSInputEvent:
return msg.Timestamp
case *IOSPerformanceEvent:
return msg.Timestamp
case *IOSLog:
return msg.Timestamp
case *IOSInternalError:
return msg.Timestamp
case *IOSNetworkCall:
return msg.Timestamp
case *IOSIssueEvent:
return msg.Timestamp
}
return uint64(message.Meta().Timestamp)
}

15
backend/pkg/messages/legacy-message-transform.go
View file

@ -1,21 +1,20 @@
package messages
func transformDepricated(msg Message) Message {
switch m := msg.(type) {
case *MouseClickDepricated:
meta := m.Meta()
meta := m.Meta()
meta.TypeID = 33
return &MouseClick{
meta: meta,
ID: m.ID,
meta: meta,
ID: m.ID,
HesitationTime: m.HesitationTime,
Label: m.Label,
Label: m.Label,
// Selector: '',
}
// case *FetchDepricated:
// return &Fetch {
// Method: m.Method,
// Method: m.Method,
// URL: m.URL,
// Request: m.Request,
// Response: m.Response,
@ -25,8 +24,6 @@ func transformDepricated(msg Message) Message {
// // Headers: ''
// }
default:
return msg
return msg
}
}

2304
backend/pkg/messages/messages.go
View file

File diff suppressed because it is too large Load diff

3
backend/pkg/messages/performance/performance.go
View file

@ -4,7 +4,6 @@ import (
"math"
)
func TimeDiff(t1 uint64, t2 uint64) uint64 {
if t1 < t2 {
return 0
@ -30,4 +29,4 @@ func CPURateFromTickRate(tickRate float64) uint64 {
func CPURate(ticks int64, dt uint64) uint64 {
return CPURateFromTickRate(TickRate(ticks, dt))
}
}

6
backend/pkg/messages/primitives.go
View file

@ -1,9 +1,9 @@
package messages
import (
"encoding/json"
"errors"
"io"
"encoding/json"
"log"
)
@ -37,7 +37,7 @@ func ReadData(reader io.Reader) ([]byte, error) {
}
return p, nil
}
func ReadUint(reader io.Reader) (uint64, error) {
var x uint64
var s uint
@ -152,4 +152,4 @@ func WriteJson(v interface{}, buf []byte, p int) int {
return WriteString("null", buf, p)
}
return WriteData(data, buf, p)
}
}

2846
backend/pkg/messages/read-message.go
View file

File diff suppressed because it is too large Load diff

2
backend/pkg/pprof/pprof.go
View file

@ -8,6 +8,6 @@ import (
func StartProfilingServer() {
go func() {
log.Println(http.ListenAndServe("localhost:6060", nil))
log.Println(http.ListenAndServe(":6060", nil))
}()
}

13
backend/pkg/redisstream/producer.go
View file

@ -6,25 +6,24 @@ import (
"openreplay/backend/pkg/env"
)
type Producer struct {
redis *redis.Client
maxLenApprox int64
redis *redis.Client
maxLenApprox int64
}
func NewProducer() *Producer {
return &Producer{
redis: getRedisClient(),
redis: getRedisClient(),
maxLenApprox: int64(env.Uint64("REDIS_STREAMS_MAX_LEN")),
}
}
func (p *Producer) Produce(topic string, key uint64, value []byte) error {
args := &redis.XAddArgs{
args := &redis.XAddArgs{
Stream: topic,
Values: map[string]interface{}{
"sessionID": key,
"value": value,
"value": value,
},
}
args.MaxLenApprox = p.maxLenApprox
@ -35,7 +34,7 @@ func (p *Producer) Produce(topic string, key uint64, value []byte) error {
}
return nil
}
func (p *Producer) Close(_ int) {
// noop
}

8
backend/pkg/redisstream/redis.go
View file

@ -2,15 +2,13 @@ package redisstream
import (
"log"
"github.com/go-redis/redis"
"openreplay/backend/pkg/env"
)
var redisClient *redis.Client
var redisClient *redis.Client
func getRedisClient() *redis.Client {
if redisClient != nil {
@ -23,4 +21,4 @@ func getRedisClient() *redis.Client {
log.Fatalln(err)
}
return redisClient
}
}

41
backend/pkg/storage/s3.go
View file

@ -2,8 +2,8 @@ package storage
import (
"io"
"strconv"
"sort"
"strconv"
_s3 "github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
@ -12,18 +12,17 @@ import (
)
type S3 struct {
uploader *s3manager.Uploader
svc *_s3.S3
bucket *string
uploader *s3manager.Uploader
svc *_s3.S3
bucket *string
}
func NewS3(region string, bucket string) *S3 {
sess := env.AWSSessionOnRegion(region)
return &S3{
uploader: s3manager.NewUploader(sess),
svc: _s3.New(sess), // AWS Docs: "These clients are safe to use concurrently."
bucket: &bucket,
svc: _s3.New(sess), // AWS Docs: "These clients are safe to use concurrently."
bucket: &bucket,
}
}
@ -35,14 +34,14 @@ func (s3 *S3) Upload(reader io.Reader, key string, contentType string, gzipped b
contentEncoding = &gzipStr
}
_, err := s3.uploader.Upload(&s3manager.UploadInput{
Body: reader,
Bucket: s3.bucket,
Key: &key,
ContentType: &contentType,
CacheControl: &cacheControl,
Body: reader,
Bucket: s3.bucket,
Key: &key,
ContentType: &contentType,
CacheControl: &cacheControl,
ContentEncoding: contentEncoding,
})
return err
})
return err
}
func (s3 *S3) Get(key string) (io.ReadCloser, error) {
@ -67,8 +66,8 @@ func (s3 *S3) Exists(key string) bool {
return false
}
const MAX_RETURNING_COUNT = 40
func (s3 *S3) GetFrequentlyUsedKeys(projectID uint64) ([]string, error) {
prefix := strconv.FormatUint(projectID, 10) + "/"
output, err := s3.svc.ListObjectsV2(&_s3.ListObjectsV2Input{
@ -82,7 +81,7 @@ func (s3 *S3) GetFrequentlyUsedKeys(projectID uint64) ([]string, error) {
list := output.Contents
max := len(list)
if (max > MAX_RETURNING_COUNT) {
if max > MAX_RETURNING_COUNT {
max = MAX_RETURNING_COUNT
sort.Slice(list, func(i, j int) bool {
return list[i].LastModified.After(*(list[j].LastModified))
@ -91,8 +90,8 @@ func (s3 *S3) GetFrequentlyUsedKeys(projectID uint64) ([]string, error) {
var keyList []string
s := len(prefix)
for _, obj := range list[:max] {
keyList = append(keyList, (*obj.Key)[s:])
}
return keyList, nil
}
for _, obj := range list[:max] {
keyList = append(keyList, (*obj.Key)[s:])
}
return keyList, nil
}

2
backend/pkg/url/assets/css.go
View file

@ -39,7 +39,7 @@ func unquote(str string) (string, string) {
}
func ExtractURLsFromCSS(css string) []string {
indexes := cssUrlsIndex(css)
indexes := cssUrlsIndex(css)
urls := make([]string, len(indexes))
for _, idx := range indexes {

10
backend/pkg/url/method.go
View file

@ -1,12 +1,12 @@
package url
var METHODS = []string{ "GET", "HEAD", "POST" , "PUT" , "DELETE" , "CONNECT" , "OPTIONS" , "TRACE" , "PATCH" }
var METHODS = []string{"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"}
func EnsureMethod(method string) string {
for _, m := range METHODS {
if m == method {
return method
}
if m == method {
return method
}
}
return ""
}
}

8
backend/services/assets/jsexception.go
View file

@ -1,16 +1,14 @@
package main
package main
import (
"encoding/json"
"strings"
)
type frame struct {
FileName string `json:"fileName"`
}
func extractJSExceptionSources(payload *string) ([]string, error) {
var frameList []frame
err := json.Unmarshal([]byte(*payload), &frameList)
@ -25,8 +23,8 @@ func extractJSExceptionSources(payload *string) ([]string, error) {
fn := strings.Split(f.FileName, "?")[0]
if strings.HasPrefix(fn, "http") && !presentedFileName[fn] {
fileNamesList = append(fileNamesList, f.FileName)
presentedFileName[fn] = true
presentedFileName[fn] = true
}
}
return fileNamesList, nil
}
}

1
backend/services/assets/main.go
View file

@ -66,6 +66,7 @@ func main() {
os.Exit(0)
case err := <-cacher.Errors:
log.Printf("Error while caching: %v", err)
// TODO: notify user
case <-tick:
cacher.UpdateTimeouts()
default:

13
backend/services/db/heuristics/anr.go
View file

@ -1,23 +1,22 @@
package heuristics
import (
. "openreplay/backend/pkg/messages"
. "openreplay/backend/pkg/messages"
)
const MIN_TIME_AFTER_LAST_HEARTBEAT = 60 * 1000
type anr struct {
readyMessageStore
lastLabel string
lastLabel string
lastHeartbeatTimestamp uint64
lastHeartbeatIndex uint64
lastHeartbeatIndex uint64
}
func (h *anr) buildIf(timestamp uint64) {
if h.lastHeartbeatTimestamp != 0 && h.lastHeartbeatTimestamp + MIN_TIME_AFTER_LAST_HEARTBEAT <= timestamp {
if h.lastHeartbeatTimestamp != 0 && h.lastHeartbeatTimestamp+MIN_TIME_AFTER_LAST_HEARTBEAT <= timestamp {
m := &IOSIssueEvent{
Type: "anr",
Type: "anr",
ContextString: h.lastLabel,
//Context: "{}",
//Payload: fmt.SPrint
@ -49,4 +48,4 @@ func (h *anr) HandleMessage(msg Message) {
case *IOSSessionEnd:
h.buildIf(m.Timestamp)
}
}
}

17
backend/services/db/heuristics/clickrage.go
View file

@ -1,26 +1,25 @@
package heuristics
import (
. "openreplay/backend/pkg/messages"
. "openreplay/backend/pkg/messages"
)
const CLICK_TIME_DIFF = 200
const MIN_CLICKS_IN_A_ROW = 3
type clickrage struct {
readyMessageStore
lastTimestamp uint64
lastLabel string
lastTimestamp uint64
lastLabel string
firstInARawTimestamp uint64
firstInARawSeqIndex uint64
countsInARow int
firstInARawSeqIndex uint64
countsInARow int
}
func (h *clickrage) build() {
if h.countsInARow >= MIN_CLICKS_IN_A_ROW {
m := &IOSIssueEvent{
Type: "click_rage",
Type: "click_rage",
ContextString: h.lastLabel,
//Context: "{}",
//Payload: fmt.SPrint
@ -39,7 +38,7 @@ func (h *clickrage) build() {
func (h *clickrage) HandleMessage(msg Message) {
switch m := msg.(type) {
case *IOSClickEvent:
if h.lastTimestamp + CLICK_TIME_DIFF < m.Timestamp && h.lastLabel == m.Label {
if h.lastTimestamp+CLICK_TIME_DIFF < m.Timestamp && h.lastLabel == m.Label {
h.lastTimestamp = m.Timestamp
h.countsInARow += 1
return
@ -55,4 +54,4 @@ func (h *clickrage) HandleMessage(msg Message) {
case *IOSSessionEnd:
h.build()
}
}
}

13
backend/services/db/heuristics/heuristics.go
View file

@ -1,8 +1,8 @@
package heuristics
import (
. "openreplay/backend/pkg/messages"
. "openreplay/backend/pkg/db/types"
. "openreplay/backend/pkg/db/types"
. "openreplay/backend/pkg/messages"
)
type MessageHandler interface {
@ -19,7 +19,6 @@ type Handler interface {
type mainHandler map[uint64]*sessHandler
func NewHandler() mainHandler {
return make(mainHandler)
}
@ -43,8 +42,10 @@ func (m mainHandler) HandleMessage(session *Session, msg Message) {
}
func (m mainHandler) IterateSessionReadyMessages(sessionID uint64, iter func(msg Message)) {
s, ok := m[ sessionID ]
if !ok { return }
s, ok := m[sessionID]
if !ok {
return
}
s.IterateReadyMessages(iter)
if s.IsEnded() {
delete(m, sessionID)
@ -61,5 +62,3 @@ func (m mainHandler) IterateReadyMessages(iter func(sessionID uint64, msg Messag
}
}
}

25
backend/services/db/heuristics/performance.go
View file

@ -1,31 +1,30 @@
package heuristics
import (
. "openreplay/backend/pkg/messages"
. "openreplay/backend/pkg/messages"
)
const AGGR_TIME = 15 * 60 * 1000
type valueAggregator struct {
sum float64
sum float64
count float64
}
func (va *valueAggregator) aggregate() uint64 {
if va.count == 0 {
return 0
}
return uint64(va.sum/va.count)
return uint64(va.sum / va.count)
}
type performanceAggregator struct {
readyMessageStore
pa *IOSPerformanceAggregated
fps valueAggregator
cpu valueAggregator
memory valueAggregator
battery valueAggregator
pa *IOSPerformanceAggregated
fps valueAggregator
cpu valueAggregator
memory valueAggregator
battery valueAggregator
}
func (h *performanceAggregator) build(timestamp uint64) {
@ -56,7 +55,7 @@ func (h *performanceAggregator) HandleMessage(msg Message) {
if h.pa.TimestampStart == 0 {
h.pa.TimestampStart = m.Timestamp
}
if h.pa.TimestampStart + AGGR_TIME <= m.Timestamp {
if h.pa.TimestampStart+AGGR_TIME <= m.Timestamp {
h.build(m.Timestamp)
}
switch m.Name {
@ -96,8 +95,8 @@ func (h *performanceAggregator) HandleMessage(msg Message) {
if m.Value > h.pa.MaxBattery {
h.pa.MaxBattery = m.Value
}
}
}
case *IOSSessionEnd:
h.build(m.Timestamp)
}
}
}

5
backend/services/db/heuristics/readyMessageStore.go
View file

@ -1,10 +1,9 @@
package heuristics
import (
. "openreplay/backend/pkg/messages"
. "openreplay/backend/pkg/messages"
)
type readyMessageStore struct {
store []Message
}
@ -18,4 +17,4 @@ func (s *readyMessageStore) IterateReadyMessages(cb func(msg Message)) {
cb(msg)
}
s.store = nil
}
}

12
backend/services/db/heuristics/session.go
View file

@ -1,18 +1,16 @@
package heuristics
import (
. "openreplay/backend/pkg/messages"
. "openreplay/backend/pkg/db/types"
. "openreplay/backend/pkg/db/types"
. "openreplay/backend/pkg/messages"
)
type sessHandler struct {
session *Session
session *Session
handlers []Handler
ended bool
ended bool
}
func newSessHandler(session *Session) *sessHandler {
return &sessHandler{
session: session,
@ -44,4 +42,4 @@ func (s *sessHandler) IterateReadyMessages(cb func(msg Message)) {
func (s *sessHandler) IsEnded() bool {
return s.ended
}
}

20
backend/services/db/stats.go
View file

@ -1,25 +1,23 @@
package main
import (
. "openreplay/backend/pkg/messages"
. "openreplay/backend/pkg/db/types"
. "openreplay/backend/pkg/messages"
)
func initStats() {
// noop
// noop
}
func insertStats(session *Session, msg Message) error {
switch m := msg.(type) {
// Web
case *PerformanceTrackAggr:
return pg.InsertWebStatsPerformance(session.SessionID, m)
case *ResourceEvent:
return pg.InsertWebStatsResourceEvent(session.SessionID, m)
case *LongTask:
return pg.InsertWebStatsLongtask(session.SessionID, m)
// Web
case *PerformanceTrackAggr:
return pg.InsertWebStatsPerformance(session.SessionID, m)
case *ResourceEvent:
return pg.InsertWebStatsResourceEvent(session.SessionID, m)
case *LongTask:
return pg.InsertWebStatsLongtask(session.SessionID, m)
// IOS
// case *IOSPerformanceAggregated:

9
backend/services/ender/builder/builderMap.go
View file

@ -6,7 +6,6 @@ import (
type builderMap map[uint64]*builder
func NewBuilderMap() builderMap {
return make(builderMap)
}
@ -28,8 +27,10 @@ func (m builderMap) HandleMessage(sessionID uint64, msg Message, messageID uint6
}
func (m builderMap) IterateSessionReadyMessages(sessionID uint64, operatingTs int64, iter func(msg Message)) {
b, ok := m[ sessionID ]
if !ok { return }
b, ok := m[sessionID]
if !ok {
return
}
sessionEnded := b.checkTimeouts(operatingTs)
b.iterateReadyMessage(iter)
if sessionEnded {
@ -48,5 +49,3 @@ func (m builderMap) IterateReadyMessages(operatingTs int64, iter func(sessionID
}
}
}

26
backend/services/ender/builder/clikRageDetector.go
View file

@ -1,34 +1,32 @@
package builder
import (
"encoding/json"
"encoding/json"
. "openreplay/backend/pkg/messages"
)
const CLICK_TIME_DIFF = 300
const MIN_CLICKS_IN_A_ROW = 3
type clickRageDetector struct {
lastTimestamp uint64
lastLabel string
lastTimestamp uint64
lastLabel string
firstInARawTimestamp uint64
firstInARawMessageId uint64
countsInARow int
countsInARow int
}
func (crd *clickRageDetector) Build() *IssueEvent {
var i *IssueEvent
if crd.countsInARow >= MIN_CLICKS_IN_A_ROW {
payload, _ := json.Marshal(struct{Count int }{crd.countsInARow,})
payload, _ := json.Marshal(struct{ Count int }{crd.countsInARow})
i = &IssueEvent{
Type: "click_rage",
Type: "click_rage",
ContextString: crd.lastLabel,
Payload: string(payload), // TODO: json encoder
Timestamp: crd.firstInARawTimestamp,
MessageID: crd.firstInARawMessageId,
Payload: string(payload), // TODO: json encoder
Timestamp: crd.firstInARawTimestamp,
MessageID: crd.firstInARawMessageId,
}
}
crd.lastTimestamp = 0
@ -39,8 +37,8 @@ func (crd *clickRageDetector) Build() *IssueEvent {
return i
}
func (crd *clickRageDetector) HandleMouseClick(msg *MouseClick, messageID uint64, timestamp uint64) *IssueEvent {
if crd.lastTimestamp + CLICK_TIME_DIFF > timestamp && crd.lastLabel == msg.Label {
func (crd *clickRageDetector) HandleMouseClick(msg *MouseClick, messageID uint64, timestamp uint64) *IssueEvent {
if crd.lastTimestamp+CLICK_TIME_DIFF > timestamp && crd.lastLabel == msg.Label {
crd.lastTimestamp = timestamp
crd.countsInARow += 1
return nil
@ -54,4 +52,4 @@ func (crd *clickRageDetector) HandleMouseClick(msg *MouseClick, messageID uint6
crd.countsInARow = 1
}
return i
}
}

29
backend/services/ender/builder/cpuIssueFinder.go
View file

@ -3,20 +3,19 @@ package builder
import (
"encoding/json"
"openreplay/backend/pkg/messages/performance"
. "openreplay/backend/pkg/messages"
"openreplay/backend/pkg/messages/performance"
)
const CPU_THRESHOLD = 70 // % out of 100
const CPU_THRESHOLD = 70 // % out of 100
const CPU_MIN_DURATION_TRIGGER = 6 * 1000
type cpuIssueFinder struct {
startTimestamp uint64
startMessageID uint64
lastTimestamp uint64
maxRate uint64
contextString string
lastTimestamp uint64
maxRate uint64
contextString string
}
func (f *cpuIssueFinder) Build() *IssueEvent {
@ -35,16 +34,16 @@ func (f *cpuIssueFinder) Build() *IssueEvent {
return nil
}
payload, _ := json.Marshal(struct{
payload, _ := json.Marshal(struct {
Duration uint64
Rate uint64
}{duration,maxRate})
Rate uint64
}{duration, maxRate})
return &IssueEvent{
Type: "cpu",
Timestamp: timestamp,
MessageID: messageID,
Type: "cpu",
Timestamp: timestamp,
MessageID: messageID,
ContextString: f.contextString,
Payload: string(payload),
Payload: string(payload),
}
}
@ -52,8 +51,6 @@ func (f *cpuIssueFinder) HandleSetPageLocation(msg *SetPageLocation) {
f.contextString = msg.URL
}
func (f *cpuIssueFinder) HandlePerformanceTrack(msg *PerformanceTrack, messageID uint64, timestamp uint64) *IssueEvent {
dt := performance.TimeDiff(timestamp, f.lastTimestamp)
if dt == 0 {
@ -82,5 +79,3 @@ func (f *cpuIssueFinder) HandlePerformanceTrack(msg *PerformanceTrack, messageID
return nil
}

24
backend/services/ender/builder/deadClickDetector.go
View file

@ -4,25 +4,23 @@ import (
. "openreplay/backend/pkg/messages"
)
const CLICK_RELATION_TIME = 1400
type deadClickDetector struct {
lastMouseClick *MouseClick
lastTimestamp uint64
lastMessageID uint64
inputIDSet map[uint64]bool
lastMouseClick *MouseClick
lastTimestamp uint64
lastMessageID uint64
inputIDSet map[uint64]bool
}
func (d *deadClickDetector) HandleReaction(timestamp uint64) *IssueEvent {
var i *IssueEvent
if d.lastMouseClick != nil && d.lastTimestamp + CLICK_RELATION_TIME < timestamp {
if d.lastMouseClick != nil && d.lastTimestamp+CLICK_RELATION_TIME < timestamp {
i = &IssueEvent{
Type: "dead_click",
Type: "dead_click",
ContextString: d.lastMouseClick.Label,
Timestamp: d.lastTimestamp,
MessageID: d.lastMessageID,
Timestamp: d.lastTimestamp,
MessageID: d.lastMessageID,
}
}
d.inputIDSet = nil
@ -53,8 +51,8 @@ func (d *deadClickDetector) HandleMessage(msg Message, messageID uint64, timesta
d.lastMouseClick = m
d.lastTimestamp = timestamp
d.lastMessageID = messageID
case *SetNodeAttribute,
*RemoveNodeAttribute,
case *SetNodeAttribute,
*RemoveNodeAttribute,
*CreateElementNode,
*CreateTextNode,
*MoveNode,
@ -66,5 +64,3 @@ func (d *deadClickDetector) HandleMessage(msg Message, messageID uint64, timesta
}
return i
}

11
backend/services/ender/builder/domDropDetector.go
View file

@ -4,14 +4,13 @@ import (
. "openreplay/backend/pkg/messages"
)
type domDropDetector struct {
removedCount int
removedCount int
lastDropTimestamp uint64
}
const DROP_WINDOW = 200 //ms
const CRITICAL_COUNT = 1 // Our login page contains 20. But on crush it removes only roots (1-3 nodes).
const DROP_WINDOW = 200 //ms
const CRITICAL_COUNT = 1 // Our login page contains 20. But on crush it removes only roots (1-3 nodes).
func (dd *domDropDetector) HandleNodeCreation() {
dd.removedCount = 0
@ -19,7 +18,7 @@ func (dd *domDropDetector) HandleNodeCreation() {
}
func (dd *domDropDetector) HandleNodeRemoval(ts uint64) {
if dd.lastDropTimestamp + DROP_WINDOW > ts {
if dd.lastDropTimestamp+DROP_WINDOW > ts {
dd.removedCount += 1
} else {
dd.removedCount = 1
@ -27,7 +26,6 @@ func (dd *domDropDetector) HandleNodeRemoval(ts uint64) {
dd.lastDropTimestamp = ts
}
func (dd *domDropDetector) Build() *DOMDrop {
var domDrop *DOMDrop
if dd.removedCount >= CRITICAL_COUNT {
@ -39,4 +37,3 @@ func (dd *domDropDetector) Build() *DOMDrop {
dd.lastDropTimestamp = 0
return domDrop
}

11
backend/services/ender/builder/inputEventBuilder.go
View file

@ -7,9 +7,9 @@ import (
type inputLabels map[uint64]string
type inputEventBuilder struct {
inputEvent *InputEvent
inputLabels inputLabels
inputID uint64
inputEvent *InputEvent
inputLabels inputLabels
inputID uint64
}
func NewInputEventBuilder() *inputEventBuilder {
@ -18,7 +18,6 @@ func NewInputEventBuilder() *inputEventBuilder {
return ieBuilder
}
func (b *inputEventBuilder) ClearLabels() {
b.inputLabels = make(inputLabels)
}
@ -57,11 +56,11 @@ func (b *inputEventBuilder) HasInstance() bool {
return b.inputEvent != nil
}
func (b * inputEventBuilder) GetTimestamp() uint64 {
func (b *inputEventBuilder) GetTimestamp() uint64 {
if b.inputEvent == nil {
return 0
}
return b.inputEvent.Timestamp;
return b.inputEvent.Timestamp
}
func (b *inputEventBuilder) Build() *InputEvent {

24
backend/services/ender/builder/memoryIssueFinder.go
View file

@ -1,21 +1,21 @@
package builder
import (
"math"
"encoding/json"
"math"
. "openreplay/backend/pkg/messages"
)
const MIN_COUNT = 3
const MEM_RATE_THRESHOLD = 300 // % to average
const MEM_RATE_THRESHOLD = 300 // % to average
type memoryIssueFinder struct {
startMessageID uint64
startTimestamp uint64
rate int
count float64
sum float64
sum float64
contextString string
}
@ -23,13 +23,13 @@ func (f *memoryIssueFinder) Build() *IssueEvent {
if f.startTimestamp == 0 {
return nil
}
payload, _ := json.Marshal(struct{Rate int }{f.rate - 100,})
payload, _ := json.Marshal(struct{ Rate int }{f.rate - 100})
i := &IssueEvent{
Type: "memory",
Timestamp: f.startTimestamp,
MessageID: f.startMessageID,
Type: "memory",
Timestamp: f.startTimestamp,
MessageID: f.startMessageID,
ContextString: f.contextString,
Payload: string(payload),
Payload: string(payload),
}
f.startTimestamp = 0
f.startMessageID = 0
@ -48,8 +48,8 @@ func (f *memoryIssueFinder) HandlePerformanceTrack(msg *PerformanceTrack, messag
return nil
}
average := f.sum/f.count
rate := int(math.Round(float64(msg.UsedJSHeapSize)/average * 100))
average := f.sum / f.count
rate := int(math.Round(float64(msg.UsedJSHeapSize) / average * 100))
f.sum += float64(msg.UsedJSHeapSize)
f.count++
@ -68,5 +68,3 @@ func (f *memoryIssueFinder) HandlePerformanceTrack(msg *PerformanceTrack, messag
return nil
}

16
backend/services/ender/builder/pageEventBuilder.go
View file

@ -5,8 +5,8 @@ import (
)
type pageEventBuilder struct {
pageEvent *PageEvent
firstTimingHandled bool
pageEvent *PageEvent
firstTimingHandled bool
}
func (b *pageEventBuilder) buildIfTimingsComplete() *PageEvent {
@ -28,7 +28,7 @@ func (b *pageEventBuilder) HandleSetPageLocation(msg *SetPageLocation, messageID
}
}
func (b * pageEventBuilder) HandlePageLoadTiming(msg *PageLoadTiming) *PageEvent {
func (b *pageEventBuilder) HandlePageLoadTiming(msg *PageLoadTiming) *PageEvent {
if !b.HasInstance() {
return nil
}
@ -62,7 +62,7 @@ func (b * pageEventBuilder) HandlePageLoadTiming(msg *PageLoadTiming) *PageEvent
return b.buildIfTimingsComplete()
}
func (b * pageEventBuilder) HandlePageRenderTiming(msg *PageRenderTiming) *PageEvent {
func (b *pageEventBuilder) HandlePageRenderTiming(msg *PageRenderTiming) *PageEvent {
if !b.HasInstance() {
return nil
}
@ -76,16 +76,16 @@ func (b *pageEventBuilder) HasInstance() bool {
return b.pageEvent != nil
}
func (b * pageEventBuilder) GetTimestamp() uint64 {
func (b *pageEventBuilder) GetTimestamp() uint64 {
if b.pageEvent == nil {
return 0
}
return b.pageEvent.Timestamp;
return b.pageEvent.Timestamp
}
func (b * pageEventBuilder) Build() *PageEvent {
func (b *pageEventBuilder) Build() *PageEvent {
pageEvent := b.pageEvent
b.pageEvent = nil
b.firstTimingHandled = false
return pageEvent
}
}

23
backend/services/ender/builder/performanceTrackAggrBuilder.go
View file

@ -3,22 +3,20 @@ package builder
import (
"math"
"openreplay/backend/pkg/messages/performance"
. "openreplay/backend/pkg/messages"
"openreplay/backend/pkg/messages/performance"
)
type performanceTrackAggrBuilder struct {
performanceTrackAggr *PerformanceTrackAggr
lastTimestamp uint64
count float64
sumFrameRate float64
sumTickRate float64
sumTotalJSHeapSize float64
sumUsedJSHeapSize float64
performanceTrackAggr *PerformanceTrackAggr
lastTimestamp uint64
count float64
sumFrameRate float64
sumTickRate float64
sumTotalJSHeapSize float64
sumUsedJSHeapSize float64
}
func (b *performanceTrackAggrBuilder) start(timestamp uint64) {
b.performanceTrackAggr = &PerformanceTrackAggr{
TimestampStart: timestamp,
@ -39,7 +37,7 @@ func (b *performanceTrackAggrBuilder) HandlePerformanceTrack(msg *PerformanceTra
}
frameRate := performance.FrameRate(msg.Frames, dt)
tickRate := performance.TickRate(msg.Ticks, dt)
tickRate := performance.TickRate(msg.Ticks, dt)
fps := uint64(math.Round(frameRate))
cpu := performance.CPURateFromTickRate(tickRate)
@ -84,7 +82,7 @@ func (b *performanceTrackAggrBuilder) GetStartTimestamp() uint64 {
if b.performanceTrackAggr == nil {
return 0
}
return b.performanceTrackAggr.TimestampStart;
return b.performanceTrackAggr.TimestampStart
}
func (b *performanceTrackAggrBuilder) Build() *PerformanceTrackAggr {
@ -106,4 +104,3 @@ func (b *performanceTrackAggrBuilder) Build() *PerformanceTrackAggr {
b.lastTimestamp = 0
return performanceTrackAggr
}

4
backend/services/http/assets.go
View file

@ -1,8 +1,8 @@
package main
import (
"openreplay/backend/pkg/url/assets"
"openreplay/backend/pkg/messages"
"openreplay/backend/pkg/url/assets"
)
func sendAssetForCache(sessionID uint64, baseURL string, relativeURL string) {
@ -33,4 +33,4 @@ func handleCSS(sessionID uint64, baseURL string, css string) string {
return rewriter.RewriteCSS(sessionID, baseURL, css)
}
return assets.ResolveCSS(baseURL, css)
}
}

2
backend/services/http/handlers-depricated.go
View file

@ -1 +1 @@
package main
package main

256
backend/services/http/ios-device.go
View file

@ -1,138 +1,138 @@
package main
import (
"strings"
"strings"
)
func MapIOSDevice(identifier string) string {
switch identifier {
case "iPod5,1":
return "iPod touch (5th generation)"
case "iPod7,1":
return "iPod touch (6th generation)"
case "iPod9,1":
return "iPod touch (7th generation)"
case "iPhone3,1", "iPhone3,2", "iPhone3,3":
return "iPhone 4"
case "iPhone4,1":
return "iPhone 4s"
case "iPhone5,1", "iPhone5,2":
return "iPhone 5"
case "iPhone5,3", "iPhone5,4":
return "iPhone 5c"
case "iPhone6,1", "iPhone6,2":
return "iPhone 5s"
case "iPhone7,2":
return "iPhone 6"
case "iPhone7,1":
return "iPhone 6 Plus"
case "iPhone8,1":
return "iPhone 6s"
case "iPhone8,2":
return "iPhone 6s Plus"
case "iPhone8,4":
return "iPhone SE"
case "iPhone9,1", "iPhone9,3":
return "iPhone 7"
case "iPhone9,2", "iPhone9,4":
return "iPhone 7 Plus"
case "iPhone10,1", "iPhone10,4":
return "iPhone 8"
case "iPhone10,2", "iPhone10,5":
return "iPhone 8 Plus"
case "iPhone10,3", "iPhone10,6":
return "iPhone X"
case "iPhone11,2":
return "iPhone XS"
case "iPhone11,4", "iPhone11,6":
return "iPhone XS Max"
case "iPhone11,8":
return "iPhone XR"
case "iPhone12,1":
return "iPhone 11"
case "iPhone12,3":
return "iPhone 11 Pro"
case "iPhone12,5":
return "iPhone 11 Pro Max"
case "iPhone12,8":
return "iPhone SE (2nd generation)"
case "iPhone13,1":
return "iPhone 12 mini"
case "iPhone13,2":
return "iPhone 12"
case "iPhone13,3":
return "iPhone 12 Pro"
case "iPhone13,4":
return "iPhone 12 Pro Max"
case "iPad2,1", "iPad2,2", "iPad2,3", "iPad2,4":
return "iPad 2"
case "iPad3,1", "iPad3,2", "iPad3,3":
return "iPad (3rd generation)"
case "iPad3,4", "iPad3,5", "iPad3,6":
return "iPad (4th generation)"
case "iPad6,11", "iPad6,12":
return "iPad (5th generation)"
case "iPad7,5", "iPad7,6":
return "iPad (6th generation)"
case "iPad7,11", "iPad7,12":
return "iPad (7th generation)"
case "iPad11,6", "iPad11,7":
return "iPad (8th generation)"
case "iPad4,1", "iPad4,2", "iPad4,3":
return "iPad Air"
case "iPad5,3", "iPad5,4":
return "iPad Air 2"
case "iPad11,3", "iPad11,4":
return "iPad Air (3rd generation)"
case "iPad13,1", "iPad13,2":
return "iPad Air (4th generation)"
case "iPad2,5", "iPad2,6", "iPad2,7":
return "iPad mini"
case "iPad4,4", "iPad4,5", "iPad4,6":
return "iPad mini 2"
case "iPad4,7", "iPad4,8", "iPad4,9":
return "iPad mini 3"
case "iPad5,1", "iPad5,2":
return "iPad mini 4"
case "iPad11,1", "iPad11,2":
return "iPad mini (5th generation)"
case "iPad6,3", "iPad6,4":
return "iPad Pro (9.7-inch)"
case "iPad7,3", "iPad7,4":
return "iPad Pro (10.5-inch)"
case "iPad8,1", "iPad8,2", "iPad8,3", "iPad8,4":
return "iPad Pro (11-inch) (1st generation)"
case "iPad8,9", "iPad8,10":
return "iPad Pro (11-inch) (2nd generation)"
case "iPad6,7", "iPad6,8":
return "iPad Pro (12.9-inch) (1st generation)"
case "iPad7,1", "iPad7,2":
return "iPad Pro (12.9-inch) (2nd generation)"
case "iPad8,5", "iPad8,6", "iPad8,7", "iPad8,8":
return "iPad Pro (12.9-inch) (3rd generation)"
case "iPad8,11", "iPad8,12":
return "iPad Pro (12.9-inch) (4th generation)"
case "AppleTV5,3":
return "Apple TV"
case "AppleTV6,2":
return "Apple TV 4K"
case "AudioAccessory1,1":
return "HomePod"
case "AudioAccessory5,1":
return "HomePod mini"
case "i386", "x86_64":
return "Simulator"
default:
return identifier
}
switch identifier {
case "iPod5,1":
return "iPod touch (5th generation)"
case "iPod7,1":
return "iPod touch (6th generation)"
case "iPod9,1":
return "iPod touch (7th generation)"
case "iPhone3,1", "iPhone3,2", "iPhone3,3":
return "iPhone 4"
case "iPhone4,1":
return "iPhone 4s"
case "iPhone5,1", "iPhone5,2":
return "iPhone 5"
case "iPhone5,3", "iPhone5,4":
return "iPhone 5c"
case "iPhone6,1", "iPhone6,2":
return "iPhone 5s"
case "iPhone7,2":
return "iPhone 6"
case "iPhone7,1":
return "iPhone 6 Plus"
case "iPhone8,1":
return "iPhone 6s"
case "iPhone8,2":
return "iPhone 6s Plus"
case "iPhone8,4":
return "iPhone SE"
case "iPhone9,1", "iPhone9,3":
return "iPhone 7"
case "iPhone9,2", "iPhone9,4":
return "iPhone 7 Plus"
case "iPhone10,1", "iPhone10,4":
return "iPhone 8"
case "iPhone10,2", "iPhone10,5":
return "iPhone 8 Plus"
case "iPhone10,3", "iPhone10,6":
return "iPhone X"
case "iPhone11,2":
return "iPhone XS"
case "iPhone11,4", "iPhone11,6":
return "iPhone XS Max"
case "iPhone11,8":
return "iPhone XR"
case "iPhone12,1":
return "iPhone 11"
case "iPhone12,3":
return "iPhone 11 Pro"
case "iPhone12,5":
return "iPhone 11 Pro Max"
case "iPhone12,8":
return "iPhone SE (2nd generation)"
case "iPhone13,1":
return "iPhone 12 mini"
case "iPhone13,2":
return "iPhone 12"
case "iPhone13,3":
return "iPhone 12 Pro"
case "iPhone13,4":
return "iPhone 12 Pro Max"
case "iPad2,1", "iPad2,2", "iPad2,3", "iPad2,4":
return "iPad 2"
case "iPad3,1", "iPad3,2", "iPad3,3":
return "iPad (3rd generation)"
case "iPad3,4", "iPad3,5", "iPad3,6":
return "iPad (4th generation)"
case "iPad6,11", "iPad6,12":
return "iPad (5th generation)"
case "iPad7,5", "iPad7,6":
return "iPad (6th generation)"
case "iPad7,11", "iPad7,12":
return "iPad (7th generation)"
case "iPad11,6", "iPad11,7":
return "iPad (8th generation)"
case "iPad4,1", "iPad4,2", "iPad4,3":
return "iPad Air"
case "iPad5,3", "iPad5,4":
return "iPad Air 2"
case "iPad11,3", "iPad11,4":
return "iPad Air (3rd generation)"
case "iPad13,1", "iPad13,2":
return "iPad Air (4th generation)"
case "iPad2,5", "iPad2,6", "iPad2,7":
return "iPad mini"
case "iPad4,4", "iPad4,5", "iPad4,6":
return "iPad mini 2"
case "iPad4,7", "iPad4,8", "iPad4,9":
return "iPad mini 3"
case "iPad5,1", "iPad5,2":
return "iPad mini 4"
case "iPad11,1", "iPad11,2":
return "iPad mini (5th generation)"
case "iPad6,3", "iPad6,4":
return "iPad Pro (9.7-inch)"
case "iPad7,3", "iPad7,4":
return "iPad Pro (10.5-inch)"
case "iPad8,1", "iPad8,2", "iPad8,3", "iPad8,4":
return "iPad Pro (11-inch) (1st generation)"
case "iPad8,9", "iPad8,10":
return "iPad Pro (11-inch) (2nd generation)"
case "iPad6,7", "iPad6,8":
return "iPad Pro (12.9-inch) (1st generation)"
case "iPad7,1", "iPad7,2":
return "iPad Pro (12.9-inch) (2nd generation)"
case "iPad8,5", "iPad8,6", "iPad8,7", "iPad8,8":
return "iPad Pro (12.9-inch) (3rd generation)"
case "iPad8,11", "iPad8,12":
return "iPad Pro (12.9-inch) (4th generation)"
case "AppleTV5,3":
return "Apple TV"
case "AppleTV6,2":
return "Apple TV 4K"
case "AudioAccessory1,1":
return "HomePod"
case "AudioAccessory5,1":
return "HomePod mini"
case "i386", "x86_64":
return "Simulator"
default:
return identifier
}
}
func GetIOSDeviceType(identifier string) string {
if strings.Contains(identifier, "iPhone") {
return "mobile" //"phone"
}
if strings.Contains(identifier, "iPad") {
return "tablet"
}
return "other"
if strings.Contains(identifier, "iPhone") {
return "mobile" //"phone"
}
if strings.Contains(identifier, "iPad") {
return "tablet"
}
return "other"
}

2
backend/services/http/uuid.go
View file

@ -12,4 +12,4 @@ func getUUID(u *string) string {
}
}
return uuid.New().String()
}
}

24
backend/services/integrations/clientManager/manager.go
View file

@ -7,38 +7,36 @@ import (
"openreplay/backend/services/integrations/integration"
)
type manager struct {
clientMap integration.ClientMap
Events chan *integration.SessionErrorEvent
Errors chan error
RequestDataUpdates chan postgres.Integration // not pointer because it could change in other thread
clientMap integration.ClientMap
Events chan *integration.SessionErrorEvent
Errors chan error
RequestDataUpdates chan postgres.Integration // not pointer because it could change in other thread
}
func NewManager() *manager {
return &manager {
clientMap: make(integration.ClientMap),
return &manager{
clientMap: make(integration.ClientMap),
RequestDataUpdates: make(chan postgres.Integration, 100),
Events: make(chan *integration.SessionErrorEvent, 100),
Errors: make(chan error, 100),
Events: make(chan *integration.SessionErrorEvent, 100),
Errors: make(chan error, 100),
}
}
func (m* manager) Update(i *postgres.Integration) error {
func (m *manager) Update(i *postgres.Integration) error {
key := strconv.Itoa(int(i.ProjectID)) + i.Provider
if i.Options == nil {
delete(m.clientMap, key)
return nil
}
c, exists := m.clientMap[ key ]
c, exists := m.clientMap[key]
if !exists {
c, err := integration.NewClient(i, m.RequestDataUpdates, m.Events, m.Errors)
if err != nil {
return err
}
m.clientMap[ key ] = c
m.clientMap[key] = c
return nil
}
return c.Update(i)

43
backend/services/integrations/integration/cloudwatch.go
View file

@ -2,43 +2,40 @@ package integration
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"strings"
"regexp"
"openreplay/backend/pkg/messages"
"regexp"
"strings"
)
var reIsException = regexp.MustCompile(`(?i)exception|error`)
type cloudwatch struct {
AwsAccessKeyId string // `json:"aws_access_key_id"`
AwsSecretAccessKey string // `json:"aws_secret_access_key"`
LogGroupName string // `json:"log_group_name"`
Region string // `json:"region"`
AwsAccessKeyId string // `json:"aws_access_key_id"`
AwsSecretAccessKey string // `json:"aws_secret_access_key"`
LogGroupName string // `json:"log_group_name"`
Region string // `json:"region"`
}
func (cw *cloudwatch) Request(c *client) error {
startTs := int64(c.getLastMessageTimestamp() + 1) // From next millisecond
startTs := int64(c.getLastMessageTimestamp() + 1) // From next millisecond
//endTs := utils.CurrentTimestamp()
sess, err := session.NewSession(aws.NewConfig().
WithRegion(cw.Region).
WithCredentials(
credentials.NewStaticCredentials(cw.AwsAccessKeyId, cw.AwsSecretAccessKey, ""),
),
WithRegion(cw.Region).
WithCredentials(
credentials.NewStaticCredentials(cw.AwsAccessKeyId, cw.AwsSecretAccessKey, ""),
),
)
if err != nil {
return err
}
svc := cloudwatchlogs.New(sess)
filterOptions := new(cloudwatchlogs.FilterLogEventsInput).
SetStartTime(startTs). // Inclusively both startTime and endTime
SetStartTime(startTs). // Inclusively both startTime and endTime
// SetEndTime(endTs). // Default nil?
// SetLimit(10000). // Default 10000
SetLogGroupName(cw.LogGroupName).
@ -56,7 +53,7 @@ func (cw *cloudwatch) Request(c *client) error {
}
if !reIsException.MatchString(*e.Message) { // too weak condition ?
continue
}
}
token, err := GetToken(*e.Message)
if err != nil {
c.errChan <- err
@ -72,18 +69,18 @@ func (cw *cloudwatch) Request(c *client) error {
//SessionID: sessionID,
Token: token,
RawErrorEvent: &messages.RawErrorEvent{
Source: "cloudwatch",
Timestamp: timestamp, // e.IngestionTime ??
Name: name,
Payload: strings.ReplaceAll(e.String(), "\n", ""),
Source: "cloudwatch",
Timestamp: timestamp, // e.IngestionTime ??
Name: name,
Payload: strings.ReplaceAll(e.String(), "\n", ""),
},
}
}
if output.NextToken == nil {
break;
break
}
filterOptions.NextToken = output.NextToken
}
return nil
}
}

6
backend/services/integrations/integration/elasticsearch.go
View file

@ -53,14 +53,14 @@ func (es *elasticsearch) Request(c *client) error {
"query": map[string]interface{}{
"bool": map[string]interface{}{
"filter": []map[string]interface{}{
map[string]interface{}{
{
"match": map[string]interface{}{
"message": map[string]interface{}{
"query": "openReplaySessionToken=", // asayer_session_id=
},
},
},
map[string]interface{}{
{
"range": map[string]interface{}{
"utc_time": map[string]interface{}{
"gte": strconv.FormatUint(gteTs, 10),
@ -68,7 +68,7 @@ func (es *elasticsearch) Request(c *client) error {
},
},
},
map[string]interface{}{
{
"term": map[string]interface{}{
"tags": "error",
},

66
backend/services/integrations/integration/rollbar.go
View file

@ -1,15 +1,15 @@
package integration
import (
"net/http"
"encoding/json"
"errors"
"fmt"
"time"
"strings"
"strconv"
"io"
"io/ioutil"
"errors"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
"openreplay/backend/pkg/messages"
)
@ -17,42 +17,42 @@ import (
// Old name: asayerSessionId
// QUERY: what can be modified?
const RB_QUERY =
"SELECT item.id, item.title,body.message.openReplaySessionToken,item.level,"+
" item.counter,item.environment,body.crash_report.raw,body.message.body,timestamp"+
" FROM item_occurrence"+
" WHERE body.message.openReplaySessionToken != null"+
" AND timestamp>= %v"+
" AND item.level>30"+
" ORDER BY timestamp"+
const RB_QUERY = "SELECT item.id, item.title,body.message.openReplaySessionToken,item.level," +
" item.counter,item.environment,body.crash_report.raw,body.message.body,timestamp" +
" FROM item_occurrence" +
" WHERE body.message.openReplaySessionToken != null" +
" AND timestamp>= %v" +
" AND item.level>30" +
" ORDER BY timestamp" +
" LIMIT 1000"
// ASC by default
// \n\t symbols can spoil the request body, so it wouldn't work (OR probably it happend because of job hashing)
/*
- `read` Access Token required
- timstamp in seconds
- `read` Access Token required
- timstamp in seconds
*/
type rollbar struct {
AccessToken string // `json:"access_token"`
AccessToken string // `json:"access_token"`
}
type rollbarJobResponce struct {
Err int
Err int
Message string
Result struct {
Result struct {
Id int
}
}
type rollbarJobStatusResponce struct {
Err int
Err int
Result struct {
Status string
Result struct {
Rows [][] json.Number
Columns[] string
Rows [][]json.Number
Columns []string
}
}
}
@ -65,7 +65,7 @@ type rollbarEvent map[string]string
*/
func (rb *rollbar) Request(c *client) error {
fromTs := c.getLastMessageTimestamp() + 1000 // From next second
c.setLastMessageTimestamp(fromTs) // anti-job-hashing
c.setLastMessageTimestamp(fromTs) // anti-job-hashing
fromTsSec := fromTs / 1e3
query := fmt.Sprintf(RB_QUERY, fromTsSec)
jsonBody := fmt.Sprintf(`{
@ -111,7 +111,7 @@ func (rb *rollbar) Request(c *client) error {
tick := time.Tick(5 * time.Second)
for {
<- tick
<-tick
resp, err = http.DefaultClient.Do(req)
if err != nil {
return err // continue + timeout/maxAttempts
@ -131,14 +131,14 @@ func (rb *rollbar) Request(c *client) error {
e := make(rollbarEvent)
for i, col := range jobStatus.Result.Result.Columns {
//if len(row) <= i { error }
e[ col ] = row[ i ].String() // here I make them all string. That's not good
e[col] = row[i].String() // here I make them all string. That's not good
}
// sessionID, err := strconv.ParseUint(e[ "body.message.asayerSessionId" ], 10, 64)
// if err != nil {
// c.errChan <- err
// continue
// }
if e[ "body.message.openReplaySessionToken" ] == "" {
if e["body.message.openReplaySessionToken"] == "" {
c.errChan <- errors.New("Token is empty!")
continue
}
@ -147,7 +147,7 @@ func (rb *rollbar) Request(c *client) error {
c.errChan <- err
continue
}
timestampSec, err := strconv.ParseUint(e[ "timestamp" ], 10, 64)
timestampSec, err := strconv.ParseUint(e["timestamp"], 10, 64)
if err != nil {
c.errChan <- err
continue
@ -155,22 +155,22 @@ func (rb *rollbar) Request(c *client) error {
timestamp := timestampSec * 1000
c.setLastMessageTimestamp(timestamp)
c.evChan <- &SessionErrorEvent{
Token: e[ "body.message.openReplaySessionToken" ],
Token: e["body.message.openReplaySessionToken"],
RawErrorEvent: &messages.RawErrorEvent{
Source: "rollbar",
Source: "rollbar",
Timestamp: timestamp,
Name: e[ "item.title" ],
Payload: string(payload),
Name: e["item.title"],
Payload: string(payload),
},
}
}
break
}
if jobStatus.Result.Status != "new" &&
if jobStatus.Result.Status != "new" &&
jobStatus.Result.Status != "running" {
// error
break
}
}
return nil
}
}

19
backend/services/integrations/integration/utils.go
View file

@ -1,34 +1,37 @@
package integration
import (
"fmt"
"regexp"
"strconv"
"strings"
"fmt"
)
var reSessionID = regexp.MustCompile(`(?i)asayer_session_id=([0-9]+)`)
func GetAsayerSessionId(s string) (uint64, error) {
func GetAsayerSessionId(s string) (uint64, error) {
matches := reSessionID.FindStringSubmatch(s)
if len(matches) < 2 {
return 0, fmt.Errorf("'asayer_session_id' not found in '%v' ", s)
}
return strconv.ParseUint(matches[ 1 ], 10, 64)
return strconv.ParseUint(matches[1], 10, 64)
}
func GetLinkFromAngularBrackets(s string) string {
beg := strings.Index(s, "<") + 1
end := strings.Index(s, ">")
if end < 0 { return "" }
if end < 0 {
return ""
}
return strings.TrimSpace(s[beg:end])
}
var reToken = regexp.MustCompile(`(?i)openReplaySessionToken=([0-9a-zA-Z\.]+)`)
func GetToken(s string) (string, error) {
func GetToken(s string) (string, error) {
matches := reToken.FindStringSubmatch(s)
if len(matches) < 2 {
return "", fmt.Errorf("'openReplaySessionToken' not found in '%v' ", s)
}
return matches[ 1 ], nil
}
return matches[1], nil
}

19
backend/services/storage/gzip.go
View file

@ -1,19 +1,18 @@
package main
import (
"io"
gzip "github.com/klauspost/pgzip"
"io"
)
func gzipFile(file io.ReadSeeker) io.Reader {
reader, writer := io.Pipe()
go func() {
gw, _ := gzip.NewWriterLevel(writer, gzip.BestSpeed)
io.Copy(gw, file)
go func() {
gw, _ := gzip.NewWriterLevel(writer, gzip.BestSpeed)
io.Copy(gw, file)
gw.Close()
writer.Close()
}()
return reader
}
gw.Close()
writer.Close()
}()
return reader
}