Merge branch 'dev' into new-frustrations

api/chalicelib/utils/helper.py

@@ -318,4 +318,6 @@ def get_domain():
 def obfuscate(text, keep_last: int = 4):
     if text is None or not isinstance(text, str):
         return text
+    if len(text) <= keep_last:
+        return "*" * len(text)
     return "*" * (len(text) - keep_last) + text[-keep_last:]

api/chalicelib/utils/pg_client.py

@@ -111,6 +111,8 @@ class PostgresClient:
     def __enter__(self):
         if self.cursor is None:
             self.cursor = self.connection.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+            self.cursor.cursor_execute = self.cursor.execute
+            self.cursor.execute = self.__execute
             self.cursor.recreate = self.recreate_cursor
         return self.cursor
 
@@ -136,6 +138,17 @@ class PostgresClient:
                     and not self.unlimited_query:
                 postgreSQL_pool.putconn(self.connection)
 
+    def __execute(self, query, vars=None):
+        try:
+            result = self.cursor.cursor_execute(query=query, vars=vars)
+        except psycopg2.Error as error:
+            logging.error(f"!!! Error of type:{type(error)} while executing query:")
+            logging.error(query)
+            logging.info("starting rollback to allow future execution")
+            self.connection.rollback()
+            raise error
+        return result
+
     def recreate_cursor(self, rollback=False):
         if rollback:
             try:

backend/cmd/heuristics/main.go

@@ -8,12 +8,17 @@ import (
 	"openreplay/backend/pkg/handlers/custom"
 	"openreplay/backend/pkg/handlers/web"
 	"openreplay/backend/pkg/messages"
+	"openreplay/backend/pkg/metrics"
+	heuristicsMetrics "openreplay/backend/pkg/metrics/heuristics"
 	"openreplay/backend/pkg/queue"
 	"openreplay/backend/pkg/sessions"
 	"openreplay/backend/pkg/terminator"
 )
 
 func main() {
+	m := metrics.New()
+	m.Register(heuristicsMetrics.List())
+
 	log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile)
 	cfg := config.New()
 

backend/internal/assets/cacher/cacher.go

@@ -2,9 +2,11 @@ package cacher
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"mime"
 	"net/http"
 	metrics "openreplay/backend/pkg/metrics/assets"
@@ -38,14 +40,43 @@ func (c *cacher) CanCache() bool {
 
 func NewCacher(cfg *config.Config) *cacher {
 	rewriter := assets.NewRewriter(cfg.AssetsOrigin)
+
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: true,
+	}
+
+	if cfg.ClientCertFilePath != "" && cfg.ClientKeyFilePath != "" && cfg.CaCertFilePath != "" {
+
+		var cert tls.Certificate
+		var err error
+
+		cert, err = tls.LoadX509KeyPair(cfg.ClientCertFilePath, cfg.ClientKeyFilePath)
+		if err != nil {
+			log.Fatalf("Error creating x509 keypair from the client cert file %s and client key file %s , Error: %s", err, cfg.ClientCertFilePath, cfg.ClientKeyFilePath)
+		}
+
+		caCert, err := ioutil.ReadFile(cfg.CaCertFilePath)
+		if err != nil {
+			log.Fatalf("Error opening cert file %s, Error: %s", cfg.CaCertFilePath, err)
+		}
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+		tlsConfig = &tls.Config{
+				InsecureSkipVerify: true,
+				Certificates:       []tls.Certificate{cert},
+				RootCAs:            caCertPool,
+			}
+		
+	}
+
 	c := &cacher{
 		timeoutMap: newTimeoutMap(),
 		s3:         storage.NewS3(cfg.AWSRegion, cfg.S3BucketAssets),
 		httpClient: &http.Client{
 			Timeout: time.Duration(6) * time.Second,
 			Transport: &http.Transport{
-				Proxy:           http.ProxyFromEnvironment,
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+				Proxy: http.ProxyFromEnvironment,
+				TLSClientConfig: tlsConfig,
 			},
 		},
 		rewriter:       rewriter,

backend/internal/config/assets/config.go

@@ -15,6 +15,9 @@ type Config struct {
 	AssetsSizeLimit      int               `env:"ASSETS_SIZE_LIMIT,required"`
 	AssetsRequestHeaders map[string]string `env:"ASSETS_REQUEST_HEADERS"`
 	UseProfiler          bool              `env:"PROFILER_ENABLED,default=false"`
+	ClientKeyFilePath    string            `env:"CLIENT_KEY_FILE_PATH"`
+	CaCertFilePath       string            `env:"CA_CERT_FILE_PATH"`
+	ClientCertFilePath   string            `env:"CLIENT_CERT_FILE_PATH"`
 }
 
 func New() *Config {

backend/internal/heuristics/service.go

@@ -1,7 +1,10 @@
 package heuristics
 
 import (
+	"fmt"
 	"log"
+	"openreplay/backend/pkg/messages"
+	metrics "openreplay/backend/pkg/metrics/heuristics"
 	"time"
 
 	"openreplay/backend/internal/config/heuristics"
@@ -35,6 +38,8 @@ func (h *heuristicsImpl) run() {
 		case evt := <-h.events.Events():
 			if err := h.producer.Produce(h.cfg.TopicAnalytics, evt.SessionID(), evt.Encode()); err != nil {
 				log.Printf("can't send new event to queue: %s", err)
+			} else {
+				metrics.IncreaseTotalEvents(messageTypeName(evt))
 			}
 		case <-tick:
 			h.producer.Flush(h.cfg.ProducerTimeout)
@@ -62,3 +67,21 @@ func (h *heuristicsImpl) Stop() {
 	h.consumer.Commit()
 	h.consumer.Close()
 }
+
+func messageTypeName(msg messages.Message) string {
+	switch msg.TypeID() {
+	case 31:
+		return "PageEvent"
+	case 32:
+		return "InputEvent"
+	case 56:
+		return "PerformanceTrackAggr"
+	case 69:
+		return "MouseClick"
+	case 125:
+		m := msg.(*messages.IssueEvent)
+		return fmt.Sprintf("IssueEvent(%s)", m.Type)
+	default:
+		return "unknown"
+	}
+}

backend/internal/storage/storage.go

@@ -95,6 +95,7 @@ func (s *Storage) Upload(msg *messages.SessionEnd) (err error) {
 	if err != nil {
 		if strings.Contains(err.Error(), "big file") {
 			log.Printf("%s, sess: %d", err, msg.SessionID())
+			metrics.IncreaseStorageTotalSkippedSessions()
 			return nil
 		}
 		return err
@@ -110,6 +111,7 @@ func (s *Storage) openSession(filePath string, tp FileType) ([]byte, error) {
 	// Check file size before download into memory
 	info, err := os.Stat(filePath)
 	if err == nil && info.Size() > s.cfg.MaxFileSize {
+		metrics.RecordSkippedSessionSize(float64(info.Size()), tp.String())
 		return nil, fmt.Errorf("big file, size: %d", info.Size())
 	}
 	// Read file into memory

backend/pkg/metrics/heuristics/metrics.go

@@ -0,0 +1,22 @@
+package heuristics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var heuristicsTotalEvents = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: "heuristics",
+		Name:      "events_total",
+		Help:      "A counter displaying the number of all processed events",
+	},
+	[]string{"type"},
+)
+
+func IncreaseTotalEvents(eventType string) {
+	heuristicsTotalEvents.WithLabelValues(eventType).Inc()
+}
+
+func List() []prometheus.Collector {
+	return []prometheus.Collector{
+		heuristicsTotalEvents,
+	}
+}

backend/pkg/metrics/storage/metrics.go

@@ -31,6 +31,32 @@ func IncreaseStorageTotalSessions() {
 	storageTotalSessions.Inc()
 }
 
+var storageSkippedSessionSize = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Namespace: "storage",
+		Name:      "session_size_bytes",
+		Help:      "A histogram displaying the size of each skipped session file in bytes.",
+		Buckets:   common.DefaultSizeBuckets,
+	},
+	[]string{"file_type"},
+)
+
+func RecordSkippedSessionSize(fileSize float64, fileType string) {
+	storageSkippedSessionSize.WithLabelValues(fileType).Observe(fileSize)
+}
+
+var storageTotalSkippedSessions = prometheus.NewCounter(
+	prometheus.CounterOpts{
+		Namespace: "storage",
+		Name:      "sessions_skipped_total",
+		Help:      "A counter displaying the total number of all skipped sessions because of the size limits.",
+	},
+)
+
+func IncreaseStorageTotalSkippedSessions() {
+	storageTotalSkippedSessions.Inc()
+}
+
 var storageSessionReadDuration = prometheus.NewHistogramVec(
 	prometheus.HistogramOpts{
 		Namespace: "storage",

ee/api/chalicelib/core/unlock.py

@@ -4,6 +4,7 @@ from os import environ
 import requests
 from decouple import config
 
+from chalicelib.utils import helper
 from chalicelib.utils.TimeUTC import TimeUTC
 
 
@@ -22,7 +23,7 @@ def check():
         environ["expiration"] = "-1"
         environ["numberOfSeats"] = "0"
         return
-    print(f"validating: {license}")
+    print(f"validating: {helper.obfuscate(license)}")
     r = requests.post('https://api.openreplay.com/os/license', json={"mid": __get_mid(), "license": get_license()})
     if r.status_code != 200 or "errors" in r.json() or not r.json()["data"].get("valid"):
         print("license validation failed")

ee/api/routers/crons/core_dynamic_crons.py

@@ -26,7 +26,7 @@ def unlock_cron() -> None:
 
 
 cron_jobs = [
-    {"func": unlock_cron, "trigger": "cron", "hour": "*"},
+    {"func": unlock_cron, "trigger": CronTrigger(day="*")},
 ]
 
 SINGLE_CRONS = [{"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"),

frontend/app/Router.js

@@ -120,8 +120,6 @@ class Router extends React.Component {
         super(props);
         if (props.isLoggedIn) {
             this.fetchInitialData();
-        } else {
-            props.fetchTenants();
         }
     }
 

frontend/app/api_middleware.js

@@ -40,7 +40,7 @@ export default () => (next) => (action) => {
     });
 };
 
-function parseError(e) {
+export function parseError(e) {
   try {
     return [...JSON.parse(e).errors] || [];
   } catch {

frontend/app/components/Login/Login.js

@@ -11,6 +11,7 @@ import cn from 'classnames';
 import { setJwt } from 'Duck/user';
 import LoginBg from '../../svg/login-illustration.svg';
 import { ENTERPRISE_REQUEIRED } from 'App/constants';
+import { fetchTenants } from 'Duck/user';
 
 const FORGOT_PASSWORD = forgotPassword();
 const SIGNUP_ROUTE = signup();
@@ -24,7 +25,7 @@ export default
     authDetails: state.getIn(['user', 'authDetails']),
     params: new URLSearchParams(props.location.search),
   }),
-  { login, setJwt }
+  { login, setJwt, fetchTenants }
 )
 @withPageTitle('Login - OpenReplay')
 @withRouter
@@ -37,6 +38,7 @@ class Login extends React.Component {
 
   componentDidMount() {
     const { params } = this.props;
+    this.props.fetchTenants();
     const jwt = params.get('jwt');
     if (jwt) {
       this.props.setJwt(jwt);

frontend/app/components/Session/Player/LivePlayer/LivePlayerInst.tsx

@@ -29,7 +29,6 @@ function Player(props: IProps) {
   const screenWrapper = React.useRef<HTMLDivElement>(null);
   const ready = playerContext.store.get().ready
 
-  console.log(ready)
   React.useEffect(() => {
     if (!props.closedLive || isMultiview) {
       const parentElement = findDOMNode(screenWrapper.current) as HTMLDivElement | null; //TODO: good architecture

frontend/app/components/Session/Session.js

@@ -2,7 +2,7 @@ import React from 'react';
 import { useEffect, useState } from 'react';
 import { connect } from 'react-redux';
 import usePageTitle from 'App/hooks/usePageTitle';
-import { fetch as fetchSession } from 'Duck/sessions';
+import { fetchV2 } from "Duck/sessions";
 import { fetchList as fetchSlackList } from 'Duck/integrations/slack';
 import { Link, NoContent, Loader } from 'UI';
 import { sessions as sessionsRoute } from 'App/routes';
@@ -17,14 +17,14 @@ function Session({
 	sessionId,
 	loading,
 	hasErrors,
-	fetchSession,
+	fetchV2,
  }) {
  	usePageTitle("OpenReplay Session Player");
  	const [ initializing, setInitializing ] = useState(true)
 	const { sessionStore }	= useStore();
 	useEffect(() => {
 		if (sessionId != null) {
-			fetchSession(sessionId)
+			fetchV2(sessionId)
 		} else {
 			console.error("No sessionID in route.")
 		}
@@ -63,6 +63,6 @@ export default withPermissions(['SESSION_REPLAY'], '', true)(connect((state, pro
 		session: state.getIn([ 'sessions', 'current' ]),
 	};
 	}, {
-	fetchSession,
 	fetchSlackList,
+	fetchV2,
 })(Session));

frontend/app/components/Session/WebPlayer.tsx

@@ -64,6 +64,12 @@ function WebPlayer(props: any) {
     return () => WebPlayerInst.clean();
   }, [session.sessionId]);
 
+  React.useEffect(() => {
+    if (session.events.length > 0 || session.errors.length > 0) {
+      contextValue.player.updateLists(session)
+    }
+  }, [session.events, session.errors])
+
   const isPlayerReady = contextValue.store?.get().ready
 
   React.useEffect(() => {

frontend/app/components/Session_/Player/Controls/Timeline.tsx

@@ -91,6 +91,7 @@ function Timeline(props: IProps) {
     }
 
     const time = getTime(e);
+    if (!time) return;
     const tz = settingsStore.sessionSettings.timezone.value
     const timeStr = DateTime.fromMillis(props.startedAt + time).setZone(tz).toFormat(`hh:mm:ss a`)
     const timeLineTooltip = {

frontend/app/duck/sessions.ts

@@ -1,18 +1,26 @@
 import { List, Map } from 'immutable';
 import Session from 'Types/session';
 import ErrorStack from 'Types/session/errorStack';
-import { Location } from 'Types/session/event'
+import { EventData, Location } from "Types/session/event";
 import Watchdog from 'Types/watchdog';
 import { clean as cleanParams } from 'App/api_client';
 import withRequestState, { RequestTypes } from './requestStateCreator';
 import { getRE, setSessionFilter, getSessionFilter, compareJsonObjects, cleanSessionFilters } from 'App/utils';
 import { LAST_7_DAYS } from 'Types/app/period';
 import { getDateRangeFromValue } from 'App/dateRange';
+import APIClient from 'App/api_client';
+import { FETCH_ACCOUNT, UPDATE_JWT } from "Duck/user";
+import logger from "App/logger";
+import { parseError } from 'App/api_middleware'
 
 const name = 'sessions';
 const FETCH_LIST = new RequestTypes('sessions/FETCH_LIST');
 const FETCH_AUTOPLAY_LIST = new RequestTypes('sessions/FETCH_AUTOPLAY_LIST');
-const FETCH = new RequestTypes('sessions/FETCH');
+const FETCH = new RequestTypes('sessions/FETCH')
+const FETCHV2 = new RequestTypes('sessions/FETCHV2')
+const FETCH_EVENTS = new RequestTypes('sessions/FETCH_EVENTS');
+const FETCH_NOTES = new RequestTypes('sessions/FETCH_NOTES');
+
 const FETCH_FAVORITE_LIST = new RequestTypes('sessions/FETCH_FAVORITE_LIST');
 const FETCH_LIVE_LIST = new RequestTypes('sessions/FETCH_LIVE_LIST');
 const TOGGLE_FAVORITE = new RequestTypes('sessions/TOGGLE_FAVORITE');
@@ -160,11 +168,82 @@ const reducer = (state = initialState, action: IAction) => {
                     }
                 });
             });
+            return state
+              .set('current', session)
+              .set('eventsIndex', matching)
+              .set('visitedEvents', visitedEvents)
+              .set('host', visitedEvents[0] && visitedEvents[0].host);
+        }
+        case FETCHV2.SUCCESS: {
+            const session = new Session(action.data);
+
             return state
                 .set('current', session)
-                .set('eventsIndex', matching)
-                .set('visitedEvents', visitedEvents)
-                .set('host', visitedEvents[0] && visitedEvents[0].host);
+        }
+        case FETCH_EVENTS.SUCCESS: {
+            const {
+                errors,
+                events,
+                issues,
+                resources,
+                stackEvents,
+                userEvents
+            } = action.data as { errors: any[], events: any[], issues: any[], resources: any[], stackEvents: any[], userEvents: EventData[] };
+            const filterEvents = action.filter.events as Record<string, any>[];
+            const session = state.get('current') as Session;
+            const matching: number[] = [];
+
+            const visitedEvents: Location[] = [];
+            const tmpMap = new Set();
+            events.forEach((event) => {
+                // @ts-ignore assume that event is LocationEvent
+                if (event.type === 'LOCATION' && !tmpMap.has(event.url)) {
+                    // @ts-ignore assume that event is LocationEvent
+                    tmpMap.add(event.url);
+                    // @ts-ignore assume that event is LocationEvent
+                    visitedEvents.push(event);
+                }
+            });
+
+            filterEvents.forEach(({ key, operator, value }) => {
+                events.forEach((e, index) => {
+                    if (key === e.type) {
+                        // @ts-ignore assume that event is LocationEvent
+                        const val = e.type === 'LOCATION' ? e.url : e.value;
+                        if (operator === 'is' && value === val) {
+                            matching.push(index);
+                        }
+                        if (operator === 'contains' && val.includes(value)) {
+                            matching.push(index);
+                        }
+                    }
+                });
+            });
+
+            const newSession = session.addEvents(
+              events,
+              errors,
+              issues,
+              resources,
+              stackEvents,
+              userEvents
+            );
+            
+            const forceUpdate = state.set('current', {})
+            return forceUpdate
+              .set('current', newSession)
+              .set('eventsIndex', matching)
+              .set('visitedEvents', visitedEvents)
+              .set('host', visitedEvents[0] && visitedEvents[0].host);
+        }
+        case FETCH_NOTES.SUCCESS: {
+            const notes = action.data;
+            if (notes.length > 0) {
+                const session = state.get('current') as Session;
+                const newSession = session.addNotes(notes);
+                return state.set('current', newSession);
+            }
+            return state
         }
         case FETCH_FAVORITE_LIST.SUCCESS:
             return state.set('favoriteList', action.data.map(s => new Session(s)));
@@ -321,6 +400,51 @@ export const fetch =
         });
     };
 
+// implementing custom middleware-like request to keep the behavior
+// TODO: move all to mobx
+export const fetchV2 = (sessionId: string) =>
+  (dispatch, getState) => {
+      const apiClient = new APIClient()
+      const apiGet = (url: string, dispatch: any, FAILURE: string) => apiClient.get(url)
+        .then(async (response) => {
+            if (response.status === 403) {
+                dispatch({ type: FETCH_ACCOUNT.FAILURE });
+            }
+            if (!response.ok) {
+                const text = await response.text();
+                return Promise.reject(text);
+            }
+            return response.json();
+        })
+        .then((json) => json || {})
+        .catch(async (e) => {
+            const data = await e.response?.json();
+            logger.error('Error during API request. ', e);
+            return dispatch({ type: FAILURE, errors: data ? parseError(data.errors) : [] });
+        });
+
+    const filter = getState().getIn(['filters', 'appliedFilter'])
+    apiGet(`/sessions/${sessionId}/replay`, dispatch, FETCH.FAILURE)
+      .then(async ({ jwt, errors, data }) => {
+          if (errors) {
+              dispatch({ type: FETCH.FAILURE, errors, data });
+          } else {
+              dispatch({ type: FETCHV2.SUCCESS, data, ...filter });
+
+              let [events, notes] = await Promise.all([
+                  apiGet(`/sessions/${sessionId}/events`, dispatch, FETCH_EVENTS.FAILURE),
+                  apiGet(`/sessions/${sessionId}/notes`, dispatch, FETCH_NOTES.FAILURE),
+              ]);
+              dispatch({ type: FETCH_EVENTS.SUCCESS, data: events.data, filter });
+              dispatch({ type: FETCH_NOTES.SUCCESS, data: notes.data });
+          }
+          if (jwt) {
+              dispatch({ type: UPDATE_JWT, data: jwt });
+          }
+      });
+
+  }
+
 export function clearCurrentSession() {
     return {
         type: CLEAR_CURRENT_SESSION

frontend/app/player/web/MessageManager.ts

@@ -109,7 +109,7 @@ export default class MessageManager {
   private scrollManager: ListWalker<SetViewportScroll> = new ListWalker();
 
   public readonly decoder = new Decoder();
-  private readonly lists: Lists;
+  private lists: Lists;
 
   private activityManager: ActivityManager | null = null;
 
@@ -138,6 +138,18 @@ export default class MessageManager {
     this.activityManager = new ActivityManager(this.session.duration.milliseconds) // only if not-live
   }
 
+  public updateLists(lists: Partial<InitialLists>) {
+    this.lists = new Lists(lists)
+
+    lists?.event?.forEach((e: Record<string, string>) => {
+      if (e.type === EVENT_TYPES.LOCATION) {
+        this.locationEventManager.append(e);
+      }
+    })
+
+    this.state.update({ ...this.lists.getFullListsState() });
+  }
+
   private setCSSLoading = (cssLoading: boolean) => {
     this.screen.displayFrame(!cssLoading)
     this.state.update({ cssLoading, ready: !this.state.get().messagesLoading && !cssLoading })

frontend/app/player/web/WebPlayer.ts

@@ -69,6 +69,21 @@ export default class WebPlayer extends Player {
 
   }
 
+  updateLists = (session: any) => {
+    let lists = {
+      event: session.events || [],
+      stack: session.stackEvents || [],
+      exceptions: session.errors?.map(({ name, ...rest }: any) =>
+        Log({
+          level: LogLevel.ERROR,
+          value: name,
+          ...rest,
+        })
+      ) || [],
+    }
+    this.messageManager.updateLists(lists)
+  }
+
   attach = (parent: HTMLElement, isClickmap?: boolean) => {
     this.screen.attach(parent)
     if (!isClickmap) {

frontend/app/types/session/event.ts

@@ -46,7 +46,7 @@ interface InputEvent extends IEvent {
   duration: number;
 }
 
-interface LocationEvent extends IEvent {
+export interface LocationEvent extends IEvent {
   url: string;
   host: string;
   pageLoad: boolean;

frontend/app/types/session/session.ts

@@ -3,7 +3,8 @@ import SessionEvent, { TYPES, EventData, InjectedEvent } from './event';
 import StackEvent from './stackEvent';
 import SessionError, { IError } from './error';
 import Issue, { IIssue, types as issueTypes } from './issue';
-import { Note } from 'App/services/NotesService'
+import { Note } from 'App/services/NotesService';
+import { toJS } from 'mobx';
 
 const HASH_MOD = 1610612741;
 const HASH_P = 53;
@@ -44,70 +45,70 @@ function hashString(s: string): number {
 }
 
 export interface ISession {
-  sessionId: string,
-  pageTitle: string,
-  active: boolean,
-  siteId: string,
-  projectKey: string,
-  peerId: string,
-  live: boolean,
-  startedAt: number,
-  duration: number,
-  events: InjectedEvent[],
-  stackEvents: StackEvent[],
-  metadata: [],
-  favorite: boolean,
-  filterId?: string,
-  domURL: string[],
-  devtoolsURL: string[],
+  sessionId: string;
+  pageTitle: string;
+  active: boolean;
+  siteId: string;
+  projectKey: string;
+  peerId: string;
+  live: boolean;
+  startedAt: number;
+  duration: number;
+  events: InjectedEvent[];
+  stackEvents: StackEvent[];
+  metadata: [];
+  favorite: boolean;
+  filterId?: string;
+  domURL: string[];
+  devtoolsURL: string[];
   /**
    * @deprecated
    */
-  mobsUrl: string[],
-  userBrowser: string,
-  userBrowserVersion: string,
-  userCountry: string,
-  userDevice: string,
-  userDeviceType: string,
-  isMobile: boolean,
-  userOs: string,
-  userOsVersion: string,
-  userId: string,
-  userAnonymousId: string,
-  userUuid: string,
-  userDisplayName: string,
-  userNumericHash: number,
-  viewed: boolean,
-  consoleLogCount: number,
-  eventsCount: number,
-  pagesCount: number,
-  errorsCount: number,
-  issueTypes: string[],
-  issues: [],
-  referrer: string | null,
-  userDeviceHeapSize: number,
-  userDeviceMemorySize: number,
-  errors: SessionError[],
-  crashes?: [],
-  socket: string,
-  isIOS: boolean,
-  revId: string | null,
-  agentIds?: string[],
-  isCallActive?: boolean,
-  agentToken: string,
-  notes: Note[],
-  notesWithEvents: Array<Note | InjectedEvent>,
-  fileKey: string,
-  platform: string,
-  projectId: string,
-  startTs: number,
-  timestamp: number,
-  backendErrors: number,
-  consoleErrors: number,
-  sessionID?: string,
-  userID: string,
-  userUUID: string,
-  userEvents: any[],
+  mobsUrl: string[];
+  userBrowser: string;
+  userBrowserVersion: string;
+  userCountry: string;
+  userDevice: string;
+  userDeviceType: string;
+  isMobile: boolean;
+  userOs: string;
+  userOsVersion: string;
+  userId: string;
+  userAnonymousId: string;
+  userUuid: string;
+  userDisplayName: string;
+  userNumericHash: number;
+  viewed: boolean;
+  consoleLogCount: number;
+  eventsCount: number;
+  pagesCount: number;
+  errorsCount: number;
+  issueTypes: string[];
+  issues: IIssue[];
+  referrer: string | null;
+  userDeviceHeapSize: number;
+  userDeviceMemorySize: number;
+  errors: SessionError[];
+  crashes?: [];
+  socket: string;
+  isIOS: boolean;
+  revId: string | null;
+  agentIds?: string[];
+  isCallActive?: boolean;
+  agentToken: string;
+  notes: Note[];
+  notesWithEvents: Array<Note | InjectedEvent>;
+  fileKey: string;
+  platform: string;
+  projectId: string;
+  startTs: number;
+  timestamp: number;
+  backendErrors: number;
+  consoleErrors: number;
+  sessionID?: string;
+  userID: string;
+  userUUID: string;
+  userEvents: any[];
 }
 
 const emptyValues = {
@@ -127,68 +128,69 @@ const emptyValues = {
   notes: [],
   metadata: {},
   startedAt: 0,
-}
+};
 
 export default class Session {
-  sessionId: ISession["sessionId"]
-  pageTitle: ISession["pageTitle"]
-  active: ISession["active"]
-  siteId: ISession["siteId"]
-  projectKey: ISession["projectKey"]
-  peerId: ISession["peerId"]
-  live: ISession["live"]
-  startedAt: ISession["startedAt"]
-  duration: ISession["duration"]
-  events: ISession["events"]
-  stackEvents: ISession["stackEvents"]
-  metadata: ISession["metadata"]
-  favorite: ISession["favorite"]
-  filterId?: ISession["filterId"]
-  domURL: ISession["domURL"]
-  devtoolsURL: ISession["devtoolsURL"]
+  sessionId: ISession['sessionId'];
+  pageTitle: ISession['pageTitle'];
+  active: ISession['active'];
+  siteId: ISession['siteId'];
+  projectKey: ISession['projectKey'];
+  peerId: ISession['peerId'];
+  live: ISession['live'];
+  startedAt: ISession['startedAt'];
+  duration: ISession['duration'];
+  events: ISession['events'];
+  stackEvents: ISession['stackEvents'];
+  metadata: ISession['metadata'];
+  favorite: ISession['favorite'];
+  filterId?: ISession['filterId'];
+  domURL: ISession['domURL'];
+  devtoolsURL: ISession['devtoolsURL'];
   /**
    * @deprecated
    */
-  mobsUrl: ISession["mobsUrl"]
-  userBrowser: ISession["userBrowser"]
-  userBrowserVersion: ISession["userBrowserVersion"]
-  userCountry: ISession["userCountry"]
-  userDevice: ISession["userDevice"]
-  userDeviceType: ISession["userDeviceType"]
-  isMobile: ISession["isMobile"]
-  userOs: ISession["userOs"]
-  userOsVersion: ISession["userOsVersion"]
-  userId: ISession["userId"]
-  userAnonymousId: ISession["userAnonymousId"]
-  userUuid: ISession["userUuid"]
-  userDisplayName: ISession["userDisplayName"]
-  userNumericHash: ISession["userNumericHash"]
-  viewed: ISession["viewed"]
-  consoleLogCount: ISession["consoleLogCount"]
-  eventsCount: ISession["eventsCount"]
-  pagesCount: ISession["pagesCount"]
-  errorsCount: ISession["errorsCount"]
-  issueTypes: ISession["issueTypes"]
-  issues: ISession["issues"]
-  referrer: ISession["referrer"]
-  userDeviceHeapSize: ISession["userDeviceHeapSize"]
-  userDeviceMemorySize: ISession["userDeviceMemorySize"]
-  errors: ISession["errors"]
-  crashes?: ISession["crashes"]
-  socket: ISession["socket"]
-  isIOS: ISession["isIOS"]
-  revId: ISession["revId"]
-  agentIds?: ISession["agentIds"]
-  isCallActive?: ISession["isCallActive"]
-  agentToken: ISession["agentToken"]
-  notes: ISession["notes"]
-  notesWithEvents: ISession["notesWithEvents"]
+  mobsUrl: ISession['mobsUrl'];
+  userBrowser: ISession['userBrowser'];
+  userBrowserVersion: ISession['userBrowserVersion'];
+  userCountry: ISession['userCountry'];
+  userDevice: ISession['userDevice'];
+  userDeviceType: ISession['userDeviceType'];
+  isMobile: ISession['isMobile'];
+  userOs: ISession['userOs'];
+  userOsVersion: ISession['userOsVersion'];
+  userId: ISession['userId'];
+  userAnonymousId: ISession['userAnonymousId'];
+  userUuid: ISession['userUuid'];
+  userDisplayName: ISession['userDisplayName'];
+  userNumericHash: ISession['userNumericHash'];
+  viewed: ISession['viewed'];
+  consoleLogCount: ISession['consoleLogCount'];
+  eventsCount: ISession['eventsCount'];
+  pagesCount: ISession['pagesCount'];
+  errorsCount: ISession['errorsCount'];
+  issueTypes: ISession['issueTypes'];
+  issues: Issue[];
+  referrer: ISession['referrer'];
+  userDeviceHeapSize: ISession['userDeviceHeapSize'];
+  userDeviceMemorySize: ISession['userDeviceMemorySize'];
+  errors: ISession['errors'];
+  crashes?: ISession['crashes'];
+  socket: ISession['socket'];
+  isIOS: ISession['isIOS'];
+  revId: ISession['revId'];
+  agentIds?: ISession['agentIds'];
+  isCallActive?: ISession['isCallActive'];
+  agentToken: ISession['agentToken'];
+  notes: ISession['notes'];
+  notesWithEvents: ISession['notesWithEvents'];
   frustrations: Array<IIssue | InjectedEvent>
 
-  fileKey: ISession["fileKey"]
+  fileKey: ISession['fileKey'];
+  durationSeconds: number;
 
   constructor(plainSession?: ISession) {
-    const sessionData = plainSession || (emptyValues as unknown as ISession)
+    const sessionData = plainSession || (emptyValues as unknown as ISession);
     const {
       startTs = 0,
       timestamp = 0,
@@ -205,7 +207,7 @@ export default class Session {
       mobsUrl = [],
       notes = [],
       ...session
-    } = sessionData
+    } = sessionData;
     const duration = Duration.fromMillis(session.duration < 1000 ? 1000 : session.duration);
     const durationSeconds = duration.valueOf();
     const startedAt = +startTs || +timestamp;
@@ -214,38 +216,39 @@ export default class Session {
     const userDeviceType = session.userDeviceType || 'other';
     const isMobile = ['console', 'mobile', 'tablet'].includes(userDeviceType);
 
-    const events: InjectedEvent[] = []
-    const rawEvents: (EventData & { key: number })[] = []
+    const events: InjectedEvent[] = [];
+    const rawEvents: (EventData & { key: number })[] = [];
 
     if (session.events?.length) {
       (session.events as EventData[]).forEach((event: EventData, k) => {
-        const time = event.timestamp - startedAt
+        const time = event.timestamp - startedAt;
         if (event.type !== TYPES.CONSOLE && time <= durationSeconds) {
-          const EventClass = SessionEvent({ ...event, time, key: k })
+          const EventClass = SessionEvent({ ...event, time, key: k });
           if (EventClass) {
             events.push(EventClass);
           }
           rawEvents.push({ ...event, time, key: k });
         }
-      })
+      });
     }
 
-    const stackEventsList: StackEvent[] = []
+    const stackEventsList: StackEvent[] = [];
     if (stackEvents?.length || session.userEvents?.length) {
       const mergedArrays = [...stackEvents, ...session.userEvents]
         .sort((a, b) => a.timestamp - b.timestamp)
-        .map((se) => new StackEvent({ ...se, time: se.timestamp - startedAt }))
+        .map((se) => new StackEvent({ ...se, time: se.timestamp - startedAt }));
       stackEventsList.push(...mergedArrays);
     }
 
-    const exceptions = (errors as IError[]).map(e => new SessionError(e)) || [];
+    const exceptions = (errors as IError[]).map((e) => new SessionError(e)) || [];
 
-    const issuesList = (issues as IIssue[]).map(
-      (i, k) => new Issue({ ...i, time: i.timestamp - startedAt, key: k })) || [];
+    const issuesList =
+      (issues as IIssue[]).map(
+        (i, k) => new Issue({ ...i, time: i.timestamp - startedAt, key: k })
+      ) || [];
 
     const rawNotes = notes;
 
-
     const frustrationEvents = events.filter(ev => {
         if (ev.type === TYPES.CLICK || ev.type === TYPES.INPUT) {
           // @ts-ignore
@@ -262,8 +265,8 @@ export default class Session {
       // @ts-ignore
       const bTs = b.timestamp || b.time;
 
-      return aTs - bTs;
-    }) || [];
+        return aTs - bTs;
+      }) || [];
 
     const mixedEventsWithIssues = mergeEventLists(
       mergeEventLists(rawEvents, rawNotes),
@@ -282,13 +285,14 @@ export default class Session {
       isMobile,
       startedAt,
       duration,
+      durationSeconds,
       userNumericHash: hashString(
         session.userId ||
-        session.userAnonymousId ||
-        session.userUuid ||
-        session.userID ||
-        session.userUUID ||
-        ''
+          session.userAnonymousId ||
+          session.userUuid ||
+          session.userID ||
+          session.userUUID ||
+          ''
       ),
       userDisplayName:
         session.userId || session.userAnonymousId || session.userID || 'Anonymous User',
@@ -299,8 +303,79 @@ export default class Session {
       domURL,
       devtoolsURL,
       notes,
+      notesWithEvents: notesWithEvents,
+    });
       notesWithEvents: mixedEventsWithIssues,
       frustrations: frustrationList,
     })
   }
-}
+
+  addEvents(
+    sessionEvents: EventData[],
+    errors: any[],
+    issues: any[],
+    resources: any[],
+    userEvents: any[],
+    stackEvents: any[]
+  ) {
+    const exceptions = (errors as IError[]).map((e) => new SessionError(e)) || [];
+    const issuesList =
+      (issues as IIssue[]).map(
+        (i, k) => new Issue({ ...i, time: i.timestamp - this.startedAt, key: k })
+      ) || [];
+    const stackEventsList: StackEvent[] = [];
+    if (stackEvents?.length || userEvents?.length) {
+      const mergedArrays = [...stackEvents, ...userEvents]
+        .sort((a, b) => a.timestamp - b.timestamp)
+        .map((se) => new StackEvent({ ...se, time: se.timestamp - this.startedAt }));
+      stackEventsList.push(...mergedArrays);
+    }
+
+    const events: InjectedEvent[] = [];
+    const rawEvents: (EventData & { key: number })[] = [];
+
+    if (sessionEvents.length) {
+      sessionEvents.forEach((event, k) => {
+        const time = event.timestamp - this.startedAt;
+        if (event.type !== TYPES.CONSOLE && time <= this.durationSeconds) {
+          const EventClass = SessionEvent({ ...event, time, key: k });
+          if (EventClass) {
+            events.push(EventClass);
+          }
+          rawEvents.push({ ...event, time, key: k });
+        }
+      });
+    }
+
+    this.events = events;
+    // @ts-ignore
+    this.notesWithEvents = rawEvents;
+    this.errors = exceptions;
+    this.issues = issuesList;
+    // @ts-ignore legacy code? no idea
+    this.resources = resources;
+    this.stackEvents = stackEventsList;
+
+    return this;
+  }
+
+  addNotes(sessionNotes: Note[]) {
+    // @ts-ignore
+    this.notesWithEvents =
+      [...this.notesWithEvents, ...sessionNotes].sort((a, b) => {
+        // @ts-ignore just in case
+        const aTs = a.timestamp || a.time;
+        // @ts-ignore
+        const bTs = b.timestamp || b.time;
+
+        return aTs - bTs;
+      }) || [];
+    this.notes = sessionNotes;
+
+    return this;
+  }
+
+  toJS() {
+    return { ...toJS(this) };
+  }
+}

scripts/helmcharts/openreplay/charts/utilities/templates/efs-cron.yaml

@@ -26,7 +26,6 @@ spec:
           containers:
           - name: efs-cleaner
             image: alpine
-            image: "{{ tpl .Values.efsCleaner.image.repository . }}:{{ .Values.efsCleaner.image.tag | default .Chart.AppVersion }}"
             command:
             - /bin/sh
             - -c
@@ -44,7 +43,6 @@ spec:
             volumeMounts:
             - mountPath: /mnt/efs
               name: datadir
-          restartPolicy: Never
           {{- if eq (tpl .Values.efsCleaner.pvc.name . ) "hostPath" }}
           volumes:
           - name: datadir

scripts/helmcharts/openreplay/charts/utilities/values.yaml

@@ -5,10 +5,6 @@
 replicaCount: 1
 
 efsCleaner:
-  image:
-    repository: "{{ .Values.global.openReplayContainerRegistry }}/alpine"
-    pullPolicy: Always
-    tag: 3.16.1
   retention: 2
   pvc:
     # This can be either persistentVolumeClaim or hostPath.

tracker/tracker-zustand/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openreplay/tracker-zustand",
   "description": "Tracker plugin for Zustand state recording",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "keywords": [
     "zustand",
     "state",
@@ -24,7 +24,7 @@
   },
   "dependencies": {},
   "peerDependencies": {
-    "@openreplay/tracker": "^4.0.1"
+    "@openreplay/tracker": ">=4.0.1"
   },
   "devDependencies": {
     "@openreplay/tracker": "^4.0.1",

tracker/tracker/CHANGELOG.md

@@ -1,5 +1,6 @@
 ## 5.0.1
 
+- Re-init worker after device sleep/hybernation
 - Default text input mode is now Obscured
 - Use `@medv/finder` instead of our own implementation of `getSelector` for better clickmaps experience
 

tracker/tracker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openreplay/tracker",
   "description": "The OpenReplay tracker main package",
-  "version": "5.0.1-beta.2",
+  "version": "5.0.1",
   "keywords": [
     "logging",
     "replay"

tracker/tracker/src/main/app/guards.ts

@@ -11,6 +11,10 @@ export function isElementNode(node: Node): node is Element {
   return node.nodeType === Node.ELEMENT_NODE
 }
 
+export function isCommentNode(node: Node): node is Comment {
+  return node.nodeType === Node.COMMENT_NODE
+}
+
 export function isTextNode(node: Node): node is Text {
   return node.nodeType === Node.TEXT_NODE
 }

tracker/tracker/src/main/app/observer/observer.ts

@@ -10,9 +10,19 @@ import {
   RemoveNode,
 } from '../messages.gen.js'
 import App from '../index.js'
-import { isRootNode, isTextNode, isElementNode, isSVGElement, hasTag } from '../guards.js'
+import {
+  isRootNode,
+  isTextNode,
+  isElementNode,
+  isSVGElement,
+  hasTag,
+  isCommentNode,
+} from '../guards.js'
 
 function isIgnored(node: Node): boolean {
+  if (isCommentNode(node)) {
+    return true
+  }
   if (isTextNode(node)) {
     return false
   }

tracker/tracker/src/main/modules/mouse.ts

@@ -197,7 +197,7 @@ export default function (app: App): void {
             id,
             mouseTarget === target ? Math.round(performance.now() - mouseTargetTime) : 0,
             getTargetLabel(target),
-            getSelector(id, target),
+            isClickable(target) ? getSelector(id, target) : '',
           ),
           true,
         )

tracker/tracker/src/webworker/index.ts

@@ -136,11 +136,17 @@ self.onmessage = ({ data }: any): any => {
 
   if (data.type === 'auth') {
     if (!sender) {
-      throw new Error('WebWorker: sender not initialised. Received auth.')
+      console.debug('WebWorker: sender not initialised. Received auth.')
+      initiateRestart()
+      return
     }
+
     if (!writer) {
-      throw new Error('WebWorker: writer not initialised. Received auth.')
+      console.debug('WebWorker: writer not initialised. Received auth.')
+      initiateRestart()
+      return
     }
+
     sender.authorise(data.token)
     data.beaconSizeLimit && writer.setBeaconSizeLimit(data.beaconSizeLimit)
     return