Changing default encryption to false

fix #1158

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>

.github/workflows/api-ee.yaml

@@ -10,6 +10,7 @@ on:
     branches:
       - dev
       - api-*
+      - v1.11.0-patch
     paths:
       - "ee/api/**"
       - "api/**"

.github/workflows/api.yaml

@@ -10,6 +10,7 @@ on:
     branches:
       - dev
       - api-*
+      - v1.11.0-patch
     paths:
       - "api/**"
       - "!api/.gitignore"

.github/workflows/crons-ee.yaml

@@ -10,6 +10,7 @@ on:
     branches:
       - dev
       - api-*
+      - v1.11.0-patch
     paths:
       - "ee/api/**"
       - "api/**"

api/chalicelib/core/health.py

@@ -14,9 +14,9 @@ def app_connection_string(name, port, path):
 
 
 HEALTH_ENDPOINTS = {
-    "alerts": app_connection_string("alerts-openreplay", 8888, "metrics"),
+    "alerts": app_connection_string("alerts-openreplay", 8888, "health"),
     "assets": app_connection_string("assets-openreplay", 8888, "metrics"),
-    "assist": app_connection_string("assist-openreplay", 8888, "metrics"),
+    "assist": app_connection_string("assist-openreplay", 8888, "health"),
     "chalice": app_connection_string("chalice-openreplay", 8888, "metrics"),
     "db": app_connection_string("db-openreplay", 8888, "metrics"),
     "ender": app_connection_string("ender-openreplay", 8888, "metrics"),

api/chalicelib/core/jobs.py

@@ -1,6 +1,6 @@
 from chalicelib.utils import pg_client, helper
 from chalicelib.utils.TimeUTC import TimeUTC
-from chalicelib.core import sessions, sessions_mobs
+from chalicelib.core import sessions_mobs, sessions_devtool
 
 
 class Actions:
@@ -17,11 +17,9 @@ class JobStatus:
 def get(job_id):
     with pg_client.PostgresClient() as cur:
         query = cur.mogrify(
-            """\
-            SELECT
-                *
-            FROM public.jobs
-            WHERE job_id = %(job_id)s;""",
+            """SELECT *
+               FROM public.jobs
+               WHERE job_id = %(job_id)s;""",
             {"job_id": job_id}
         )
         cur.execute(query=query)
@@ -37,11 +35,9 @@ def get(job_id):
 def get_all(project_id):
     with pg_client.PostgresClient() as cur:
         query = cur.mogrify(
-            """\
-            SELECT
-                *
-            FROM public.jobs
-            WHERE project_id = %(project_id)s;""",
+            """SELECT *
+               FROM public.jobs
+               WHERE project_id = %(project_id)s;""",
             {"project_id": project_id}
         )
         cur.execute(query=query)
@@ -51,23 +47,19 @@ def get_all(project_id):
     return helper.list_to_camel_case(data)
 
 
-def create(project_id, data):
+def create(project_id, user_id):
     with pg_client.PostgresClient() as cur:
-        job = {
-            "status": "scheduled",
-            "project_id": project_id,
-            **data
-        }
+        job = {"status": "scheduled",
+               "project_id": project_id,
+               "action": Actions.DELETE_USER_DATA,
+               "reference_id": user_id,
+               "description": f"Delete user sessions of userId = {user_id}",
+               "start_at": TimeUTC.to_human_readable(TimeUTC.midnight(1))}
 
-        query = cur.mogrify("""\
-            INSERT INTO public.jobs(
-                project_id, description, status, action,
-                reference_id, start_at
-            )
-            VALUES (
-                %(project_id)s, %(description)s, %(status)s, %(action)s,
-                %(reference_id)s, %(start_at)s
-            ) RETURNING *;""", job)
+        query = cur.mogrify(
+            """INSERT INTO public.jobs(project_id, description, status, action,reference_id, start_at)
+               VALUES (%(project_id)s, %(description)s, %(status)s, %(action)s,%(reference_id)s, %(start_at)s)
+               RETURNING *;""", job)
 
         cur.execute(query=query)
 
@@ -90,14 +82,13 @@ def update(job_id, job):
             **job
         }
 
-        query = cur.mogrify("""\
-            UPDATE public.jobs
-            SET
-                updated_at = timezone('utc'::text, now()),
-                status = %(status)s,
-                errors = %(errors)s
-            WHERE
-                job_id = %(job_id)s RETURNING *;""", job_data)
+        query = cur.mogrify(
+            """UPDATE public.jobs
+               SET updated_at = timezone('utc'::text, now()),
+                   status = %(status)s,
+                   errors = %(errors)s
+               WHERE job_id = %(job_id)s
+               RETURNING *;""", job_data)
 
         cur.execute(query=query)
 
@@ -113,61 +104,64 @@ def format_datetime(r):
     r["start_at"] = TimeUTC.datetime_to_timestamp(r["start_at"])
 
 
+def __get_session_ids_by_user_ids(project_id, user_ids):
+    with pg_client.PostgresClient() as cur:
+        query = cur.mogrify(
+            """SELECT session_id 
+               FROM public.sessions
+               WHERE project_id = %(project_id)s 
+                    AND user_id IN %(userId)s
+               LIMIT 1000;""",
+            {"project_id": project_id, "userId": tuple(user_ids)})
+        cur.execute(query=query)
+        ids = cur.fetchall()
+    return [s["session_id"] for s in ids]
+
+
+def __delete_sessions_by_session_ids(session_ids):
+    with pg_client.PostgresClient(unlimited_query=True) as cur:
+        query = cur.mogrify(
+            """DELETE FROM public.sessions
+               WHERE session_id IN %(session_ids)s""",
+            {"session_ids": tuple(session_ids)}
+        )
+        cur.execute(query=query)
+
+
 def get_scheduled_jobs():
     with pg_client.PostgresClient() as cur:
         query = cur.mogrify(
-            """\
-            SELECT * FROM public.jobs
-            WHERE status = %(status)s AND start_at <= (now() at time zone 'utc');""",
-            {"status": JobStatus.SCHEDULED}
-        )
+            """SELECT * 
+               FROM public.jobs
+               WHERE status = %(status)s 
+                    AND start_at <= (now() at time zone 'utc');""",
+            {"status": JobStatus.SCHEDULED})
         cur.execute(query=query)
         data = cur.fetchall()
-        for record in data:
-            format_datetime(record)
-
     return helper.list_to_camel_case(data)
 
 
 def execute_jobs():
     jobs = get_scheduled_jobs()
-    if len(jobs) == 0:
-        # No jobs to execute
-        return
-
     for job in jobs:
-        print(f"job can be executed {job['id']}")
+        print(f"Executing jobId:{job['jobId']}")
         try:
             if job["action"] == Actions.DELETE_USER_DATA:
-                session_ids = sessions.get_session_ids_by_user_ids(
-                    project_id=job["projectId"], user_ids=job["referenceId"]
-                )
-
-                sessions.delete_sessions_by_session_ids(session_ids)
-                sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"])
+                session_ids = __get_session_ids_by_user_ids(project_id=job["projectId"],
+                                                            user_ids=[job["referenceId"]])
+                if len(session_ids) > 0:
+                    print(f"Deleting {len(session_ids)} sessions")
+                    __delete_sessions_by_session_ids(session_ids)
+                    sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"])
+                    sessions_devtool.delete_mobs(session_ids=session_ids, project_id=job["projectId"])
             else:
-                raise Exception(f"The action {job['action']} not supported.")
+                raise Exception(f"The action '{job['action']}' not supported.")
 
             job["status"] = JobStatus.COMPLETED
-            print(f"job completed {job['id']}")
+            print(f"Job completed {job['jobId']}")
         except Exception as e:
             job["status"] = JobStatus.FAILED
-            job["error"] = str(e)
-            print(f"job failed {job['id']}")
+            job["errors"] = str(e)
+            print(f"Job failed {job['jobId']}")
 
-        update(job["job_id"], job)
-
-
-def group_user_ids_by_project_id(jobs, now):
-    project_id_user_ids = {}
-    for job in jobs:
-        if job["startAt"] > now:
-            continue
-
-        project_id = job["projectId"]
-        if project_id not in project_id_user_ids:
-            project_id_user_ids[project_id] = []
-
-        project_id_user_ids[project_id].append(job)
-
-    return project_id_user_ids
+        update(job["jobId"], job)

api/chalicelib/core/sessions.py

@@ -1065,47 +1065,6 @@ def get_session_user(project_id, user_id):
     return helper.dict_to_camel_case(data)
 
 
-def get_session_ids_by_user_ids(project_id, user_ids):
-    with pg_client.PostgresClient() as cur:
-        query = cur.mogrify(
-            """\
-            SELECT session_id FROM public.sessions
-            WHERE
-                project_id = %(project_id)s AND user_id IN %(userId)s;""",
-            {"project_id": project_id, "userId": tuple(user_ids)}
-        )
-        ids = cur.execute(query=query)
-    return ids
-
-
-def delete_sessions_by_session_ids(session_ids):
-    with pg_client.PostgresClient(unlimited_query=True) as cur:
-        query = cur.mogrify(
-            """\
-            DELETE FROM public.sessions
-            WHERE
-                session_id IN %(session_ids)s;""",
-            {"session_ids": tuple(session_ids)}
-        )
-        cur.execute(query=query)
-
-    return True
-
-
-def delete_sessions_by_user_ids(project_id, user_ids):
-    with pg_client.PostgresClient(unlimited_query=True) as cur:
-        query = cur.mogrify(
-            """\
-            DELETE FROM public.sessions
-            WHERE
-                project_id = %(project_id)s AND user_id IN %(userId)s;""",
-            {"project_id": project_id, "userId": tuple(user_ids)}
-        )
-        cur.execute(query=query)
-
-    return True
-
-
 def count_all():
     with pg_client.PostgresClient(unlimited_query=True) as cur:
         cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions")

api/chalicelib/core/sessions_devtool.py

@@ -24,3 +24,9 @@ def get_urls(session_id, project_id, check_existence: bool = True):
             ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900)
         ))
     return results
+
+
+def delete_mobs(project_id, session_ids):
+    for session_id in session_ids:
+        for k in __get_devtools_keys(project_id=project_id, session_id=session_id):
+            s3.schedule_for_deletion(config("sessions_bucket"), k)

api/chalicelib/core/sessions_mobs.py

@@ -57,5 +57,6 @@ def get_ios(session_id):
 
 def delete_mobs(project_id, session_ids):
     for session_id in session_ids:
-        for k in __get_mob_keys(project_id=project_id, session_id=session_id):
+        for k in __get_mob_keys(project_id=project_id, session_id=session_id) \
+                 + __get_mob_keys_deprecated(session_id=session_id):
             s3.schedule_for_deletion(config("sessions_bucket"), k)

api/chalicelib/core/sessions_replay.py

@@ -69,9 +69,11 @@ def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_
                                       if e['source'] == "js_exception"][:500]
                     data['userEvents'] = events.get_customs_by_session_id(project_id=project_id,
                                                                           session_id=session_id)
-                    data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id)
-                    data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id)
-                    data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id)
+                    data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id,
+                                                            check_existence=False)
+                    data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False)
+                    data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id,
+                                                                    check_existence=False)
                     data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id,
                                                                     start_ts=data["startTs"], duration=data["duration"])
 
@@ -126,9 +128,11 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat
                 if data["platform"] == 'ios':
                     data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id)
                 else:
-                    data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id)
-                    data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id)
-                    data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id)
+                    data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id,
+                                                            check_existence=False)
+                    data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False)
+                    data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id,
+                                                                    check_existence=False)
 
                 data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data)
                 data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id,

api/chalicelib/utils/s3.py

@@ -110,11 +110,14 @@ def rename(source_bucket, source_key, target_bucket, target_key):
 
 
 def schedule_for_deletion(bucket, key):
+    if not exists(bucket, key):
+        return False
     s3 = __get_s3_resource()
     s3_object = s3.Object(bucket, key)
     s3_object.copy_from(CopySource={'Bucket': bucket, 'Key': key},
-                        Expires=datetime.now() + timedelta(days=7),
+                        Expires=datetime.utcnow() + timedelta(days=config("SCH_DELETE_DAYS", cast=int, default=30)),
                         MetadataDirective='REPLACE')
+    return True
 
 
 def generate_file_key(project_id, key):

api/env.default

@@ -53,4 +53,5 @@ PRESIGNED_URL_EXPIRATION=3600
 ASSIST_JWT_EXPIRATION=144000
 ASSIST_JWT_SECRET=
 PYTHONUNBUFFERED=1
-REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379
+REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379
+SCH_DELETE_DAYS=30

api/routers/crons/core_dynamic_crons.py

@@ -1,5 +1,4 @@
 from apscheduler.triggers.cron import CronTrigger
-from apscheduler.triggers.interval import IntervalTrigger
 
 from chalicelib.core import telemetry
 from chalicelib.core import weekly_report, jobs
@@ -20,7 +19,7 @@ async def telemetry_cron() -> None:
 cron_jobs = [
     {"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"),
      "misfire_grace_time": 60 * 60, "max_instances": 1},
-    {"func": run_scheduled_jobs, "trigger": IntervalTrigger(minutes=1),
+    {"func": run_scheduled_jobs, "trigger": CronTrigger(day_of_week="*", hour=0, minute=15),
      "misfire_grace_time": 20, "max_instances": 1},
     {"func": weekly_report2, "trigger": CronTrigger(day_of_week="mon", hour=5),
      "misfire_grace_time": 60 * 60, "max_instances": 1}

api/routers/subs/v1_api.py

@@ -2,7 +2,6 @@ from fastapi import Depends, Body
 
 import schemas
 from chalicelib.core import sessions, events, jobs, projects
-from chalicelib.utils.TimeUTC import TimeUTC
 from or_dependencies import OR_context
 from routers.base import get_routers
 
@@ -15,7 +14,7 @@ async def get_user_sessions(projectKey: str, userId: str, start_date: int = None
     if projectId is None:
         return {"errors": ["invalid projectKey"]}
     return {
-        'data': sessions.get_user_sessions(
+        "data": sessions.get_user_sessions(
             project_id=projectId,
             user_id=userId,
             start_date=start_date,
@@ -30,7 +29,7 @@ async def get_session_events(projectKey: str, sessionId: int):
     if projectId is None:
         return {"errors": ["invalid projectKey"]}
     return {
-        'data': events.get_by_session_id(
+        "data": events.get_by_session_id(
             project_id=projectId,
             session_id=sessionId
         )
@@ -43,7 +42,7 @@ async def get_user_details(projectKey: str, userId: str):
     if projectId is None:
         return {"errors": ["invalid projectKey"]}
     return {
-        'data': sessions.get_session_user(
+        "data": sessions.get_session_user(
             project_id=projectId,
             user_id=userId
         )
@@ -55,14 +54,8 @@ async def schedule_to_delete_user_data(projectKey: str, userId: str):
     projectId = projects.get_internal_project_id(projectKey)
     if projectId is None:
         return {"errors": ["invalid projectKey"]}
-    data = {"action": "delete_user_data",
-            "reference_id": userId,
-            "description": f"Delete user sessions of userId = {userId}",
-            "start_at": TimeUTC.to_human_readable(TimeUTC.midnight(1))}
-    record = jobs.create(project_id=projectId, data=data)
-    return {
-        'data': record
-    }
+    record = jobs.create(project_id=projectId, user_id=userId)
+    return {"data": record}
 
 
 @app_apikey.get('/v1/{projectKey}/jobs', tags=["api"])
@@ -70,16 +63,12 @@ async def get_jobs(projectKey: str):
     projectId = projects.get_internal_project_id(projectKey)
     if projectId is None:
         return {"errors": ["invalid projectKey"]}
-    return {
-        'data': jobs.get_all(project_id=projectId)
-    }
+    return {"data": jobs.get_all(project_id=projectId)}
 
 
 @app_apikey.get('/v1/{projectKey}/jobs/{jobId}', tags=["api"])
 async def get_job(projectKey: str, jobId: int):
-    return {
-        'data': jobs.get(job_id=jobId)
-    }
+    return {"data": jobs.get(job_id=jobId)}
 
 
 @app_apikey.delete('/v1/{projectKey}/jobs/{jobId}', tags=["api"])
@@ -93,9 +82,7 @@ async def cancel_job(projectKey: str, jobId: int):
         return {"errors": ["The request job has already been canceled/completed."]}
 
     job["status"] = "cancelled"
-    return {
-        'data': jobs.update(job_id=jobId, job=job)
-    }
+    return {"data": jobs.update(job_id=jobId, job=job)}
 
 
 @app_apikey.get('/v1/projects', tags=["api"])
@@ -104,15 +91,13 @@ async def get_projects(context: schemas.CurrentContext = Depends(OR_context)):
     for record in records:
         del record['projectId']
 
-    return {
-        'data': records
-    }
+    return {"data": records}
 
 
 @app_apikey.get('/v1/projects/{projectKey}', tags=["api"])
 async def get_project(projectKey: str, context: schemas.CurrentContext = Depends(OR_context)):
     return {
-        'data': projects.get_project_by_key(tenant_id=context.tenant_id, project_key=projectKey)
+        "data": projects.get_project_by_key(tenant_id=context.tenant_id, project_key=projectKey)
     }
 
 
@@ -125,5 +110,5 @@ async def create_project(data: schemas.CreateProjectSchema = Body(...),
         data=data,
         skip_authorization=True
     )
-    del record['data']['projectId']
+    del record["data"]['projectId']
     return record

backend/Dockerfile

@@ -84,7 +84,8 @@ ENV TZ=UTC \
     CH_PASSWORD="" \
     CH_DATABASE="default" \
     # Max file size to process, default to 100MB
-    MAX_FILE_SIZE=100000000
+    MAX_FILE_SIZE=100000000 \
+    USE_ENCRYPTION=false
 
 
 RUN if [ "$SERVICE_NAME" = "http" ]; then \

backend/internal/storage/storage.go

@@ -33,10 +33,28 @@ func (t FileType) String() string {
 }
 
 type Task struct {
-	id   string
-	doms *bytes.Buffer
-	dome *bytes.Buffer
-	dev  *bytes.Buffer
+	id     string
+	key    string
+	domRaw []byte
+	devRaw []byte
+	doms   *bytes.Buffer
+	dome   *bytes.Buffer
+	dev    *bytes.Buffer
+}
+
+func (t *Task) SetMob(mob []byte, tp FileType) {
+	if tp == DOM {
+		t.domRaw = mob
+	} else {
+		t.devRaw = mob
+	}
+}
+
+func (t *Task) Mob(tp FileType) []byte {
+	if tp == DOM {
+		return t.domRaw
+	}
+	return t.devRaw
 }
 
 type Storage struct {
@@ -75,7 +93,8 @@ func (s *Storage) Upload(msg *messages.SessionEnd) (err error) {
 	filePath := s.cfg.FSDir + "/" + sessionID
 	// Prepare sessions
 	newTask := &Task{
-		id: sessionID,
+		id:  sessionID,
+		key: msg.EncryptionKey,
 	}
 	wg := &sync.WaitGroup{}
 	wg.Add(2)
@@ -108,6 +127,9 @@ func (s *Storage) Upload(msg *messages.SessionEnd) (err error) {
 }
 
 func (s *Storage) openSession(filePath string, tp FileType) ([]byte, error) {
+	if tp == DEV {
+		filePath += "devtools"
+	}
 	// Check file size before download into memory
 	info, err := os.Stat(filePath)
 	if err == nil && info.Size() > s.cfg.MaxFileSize {
@@ -142,51 +164,87 @@ func (s *Storage) sortSessionMessages(raw []byte) ([]byte, error) {
 }
 
 func (s *Storage) prepareSession(path string, tp FileType, task *Task) error {
-	// Open mob file
-	if tp == DEV {
-		path += "devtools"
-	}
+	// Open session file
 	startRead := time.Now()
 	mob, err := s.openSession(path, tp)
 	if err != nil {
 		return err
 	}
-	metrics.RecordSessionSize(float64(len(mob)), tp.String())
 	metrics.RecordSessionReadDuration(float64(time.Now().Sub(startRead).Milliseconds()), tp.String())
+	metrics.RecordSessionSize(float64(len(mob)), tp.String())
 
-	// Encode and compress session
-	if tp == DEV {
-		start := time.Now()
-		task.dev = s.compressSession(mob)
-		metrics.RecordSessionCompressDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
-	} else {
-		if len(mob) <= s.cfg.FileSplitSize {
-			start := time.Now()
-			task.doms = s.compressSession(mob)
-			metrics.RecordSessionCompressDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
-			return nil
-		}
-		wg := &sync.WaitGroup{}
-		wg.Add(2)
-		var firstPart, secondPart int64
-		go func() {
-			start := time.Now()
-			task.doms = s.compressSession(mob[:s.cfg.FileSplitSize])
-			firstPart = time.Now().Sub(start).Milliseconds()
-			wg.Done()
-		}()
-		go func() {
-			start := time.Now()
-			task.dome = s.compressSession(mob[s.cfg.FileSplitSize:])
-			secondPart = time.Now().Sub(start).Milliseconds()
-			wg.Done()
-		}()
-		wg.Wait()
-		metrics.RecordSessionCompressDuration(float64(firstPart+secondPart), tp.String())
-	}
+	// Put opened session file into task struct
+	task.SetMob(mob, tp)
+
+	// Encrypt and compress session
+	s.packSession(task, tp)
 	return nil
 }
 
+func (s *Storage) packSession(task *Task, tp FileType) {
+	// Prepare mob file
+	mob := task.Mob(tp)
+
+	if tp == DEV || len(mob) <= s.cfg.FileSplitSize {
+		// Encryption
+		start := time.Now()
+		data := s.encryptSession(mob, task.key)
+		metrics.RecordSessionEncryptionDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
+		// Compression
+		start = time.Now()
+		result := s.compressSession(data)
+		metrics.RecordSessionCompressDuration(float64(time.Now().Sub(start).Milliseconds()), tp.String())
+
+		if tp == DOM {
+			task.doms = result
+		} else {
+			task.dev = result
+		}
+		return
+	}
+
+	// Prepare two workers
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+	var firstPart, secondPart, firstEncrypt, secondEncrypt int64
+
+	// DomStart part
+	go func() {
+		// Encryption
+		start := time.Now()
+		data := s.encryptSession(mob[:s.cfg.FileSplitSize], task.key)
+		firstEncrypt = time.Since(start).Milliseconds()
+
+		// Compression
+		start = time.Now()
+		task.doms = s.compressSession(data)
+		firstPart = time.Since(start).Milliseconds()
+
+		// Finish task
+		wg.Done()
+	}()
+	// DomEnd part
+	go func() {
+		// Encryption
+		start := time.Now()
+		data := s.encryptSession(mob[s.cfg.FileSplitSize:], task.key)
+		secondEncrypt = time.Since(start).Milliseconds()
+
+		// Compression
+		start = time.Now()
+		task.dome = s.compressSession(data)
+		secondPart = time.Since(start).Milliseconds()
+
+		// Finish task
+		wg.Done()
+	}()
+	wg.Wait()
+
+	// Record metrics
+	metrics.RecordSessionEncryptionDuration(float64(firstEncrypt+secondEncrypt), tp.String())
+	metrics.RecordSessionCompressDuration(float64(firstPart+secondPart), tp.String())
+}
+
 func (s *Storage) encryptSession(data []byte, encryptionKey string) []byte {
 	var encryptedData []byte
 	var err error

backend/pkg/metrics/storage/metrics.go

@@ -85,18 +85,18 @@ func RecordSessionSortDuration(durMillis float64, fileType string) {
 	storageSessionSortDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
 }
 
-var storageSessionEncodeDuration = prometheus.NewHistogramVec(
+var storageSessionEncryptionDuration = prometheus.NewHistogramVec(
 	prometheus.HistogramOpts{
 		Namespace: "storage",
-		Name:      "encode_duration_seconds",
+		Name:      "encryption_duration_seconds",
 		Help:      "A histogram displaying the duration of encoding for each session in seconds.",
 		Buckets:   common.DefaultDurationBuckets,
 	},
 	[]string{"file_type"},
 )
 
-func RecordSessionEncodeDuration(durMillis float64, fileType string) {
-	storageSessionEncodeDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
+func RecordSessionEncryptionDuration(durMillis float64, fileType string) {
+	storageSessionEncryptionDuration.WithLabelValues(fileType).Observe(durMillis / 1000.0)
 }
 
 var storageSessionCompressDuration = prometheus.NewHistogramVec(
@@ -133,7 +133,7 @@ func List() []prometheus.Collector {
 		storageTotalSessions,
 		storageSessionReadDuration,
 		storageSessionSortDuration,
-		storageSessionEncodeDuration,
+		storageSessionEncryptionDuration,
 		storageSessionCompressDuration,
 		storageSessionUploadDuration,
 	}

ee/api/chalicelib/core/health.py

@@ -15,9 +15,9 @@ def app_connection_string(name, port, path):
 
 
 HEALTH_ENDPOINTS = {
-    "alerts": app_connection_string("alerts-openreplay", 8888, "metrics"),
+    "alerts": app_connection_string("alerts-openreplay", 8888, "health"),
     "assets": app_connection_string("assets-openreplay", 8888, "metrics"),
-    "assist": app_connection_string("assist-openreplay", 8888, "metrics"),
+    "assist": app_connection_string("assist-openreplay", 8888, "health"),
     "chalice": app_connection_string("chalice-openreplay", 8888, "metrics"),
     "db": app_connection_string("db-openreplay", 8888, "metrics"),
     "ender": app_connection_string("ender-openreplay", 8888, "metrics"),

ee/api/chalicelib/core/sessions_devtool.py

@@ -31,3 +31,9 @@ def get_urls(session_id, project_id, context: schemas_ee.CurrentContext, check_e
             ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900)
         ))
     return results
+
+
+def delete_mobs(project_id, session_ids):
+    for session_id in session_ids:
+        for k in __get_devtools_keys(project_id=project_id, session_id=session_id):
+            s3.schedule_for_deletion(config("sessions_bucket"), k)

ee/api/chalicelib/core/sessions_exp.py

@@ -1396,47 +1396,6 @@ def get_session_user(project_id, user_id):
     return helper.dict_to_camel_case(data)
 
 
-def get_session_ids_by_user_ids(project_id, user_ids):
-    with pg_client.PostgresClient() as cur:
-        query = cur.mogrify(
-            """\
-            SELECT session_id FROM public.sessions
-            WHERE
-                project_id = %(project_id)s AND user_id IN %(userId)s;""",
-            {"project_id": project_id, "userId": tuple(user_ids)}
-        )
-        ids = cur.execute(query=query)
-    return ids
-
-
-def delete_sessions_by_session_ids(session_ids):
-    with pg_client.PostgresClient(unlimited_query=True) as cur:
-        query = cur.mogrify(
-            """\
-            DELETE FROM public.sessions
-            WHERE
-                session_id IN %(session_ids)s;""",
-            {"session_ids": tuple(session_ids)}
-        )
-        cur.execute(query=query)
-
-    return True
-
-
-def delete_sessions_by_user_ids(project_id, user_ids):
-    with pg_client.PostgresClient(unlimited_query=True) as cur:
-        query = cur.mogrify(
-            """\
-            DELETE FROM public.sessions
-            WHERE
-                project_id = %(project_id)s AND user_id IN %(userId)s;""",
-            {"project_id": project_id, "userId": tuple(user_ids)}
-        )
-        cur.execute(query=query)
-
-    return True
-
-
 def count_all():
     with ch_client.ClickHouseClient() as cur:
         row = cur.execute(query=f"SELECT COUNT(session_id) AS count FROM {exp_ch_helper.get_main_sessions_table()}")

ee/api/chalicelib/core/sessions_replay.py

@@ -72,10 +72,11 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu
                                       if e['source'] == "js_exception"][:500]
                     data['userEvents'] = events.get_customs_by_session_id(project_id=project_id,
                                                                           session_id=session_id)
-                    data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id)
-                    data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id)
+                    data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id,
+                                                            check_existence=False)
+                    data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False)
                     data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id,
-                                                                    context=context)
+                                                                    context=context, check_existence=False)
                     data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id,
                                                                     start_ts=data["startTs"], duration=data["duration"])
 
@@ -132,10 +133,11 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat
                 if data["platform"] == 'ios':
                     data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id)
                 else:
-                    data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id)
-                    data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id)
+                    data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id,
+                                                            check_existence=False)
+                    data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False)
                     data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id,
-                                                                    context=context)
+                                                                    context=context, check_existence=False)
 
                 data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data)
                 data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id,

ee/api/env.default

@@ -73,4 +73,5 @@ PRESIGNED_URL_EXPIRATION=3600
 ASSIST_JWT_EXPIRATION=144000
 ASSIST_JWT_SECRET=
 KAFKA_SERVERS=kafka.db.svc.cluster.local:9092
-KAFKA_USE_SSL=false
+KAFKA_USE_SSL=false
+SCH_DELETE_DAYS=30

ee/api/routers/crons/core_dynamic_crons.py

@@ -1,5 +1,4 @@
 from apscheduler.triggers.cron import CronTrigger
-from apscheduler.triggers.interval import IntervalTrigger
 from decouple import config
 
 from chalicelib.core import jobs
@@ -29,13 +28,14 @@ cron_jobs = [
     {"func": unlock_cron, "trigger": CronTrigger(day="*")},
 ]
 
-SINGLE_CRONS = [{"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"),
-                 "misfire_grace_time": 60 * 60, "max_instances": 1},
-                {"func": run_scheduled_jobs, "trigger": IntervalTrigger(minutes=60),
-                 "misfire_grace_time": 20, "max_instances": 1},
-                {"func": weekly_report, "trigger": CronTrigger(day_of_week="mon", hour=5),
-                 "misfire_grace_time": 60 * 60, "max_instances": 1}
-                ]
+SINGLE_CRONS = [
+    {"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"),
+     "misfire_grace_time": 60 * 60, "max_instances": 1},
+    {"func": run_scheduled_jobs, "trigger": CronTrigger(day_of_week="*", hour=0, minute=15),
+     "misfire_grace_time": 20, "max_instances": 1},
+    {"func": weekly_report, "trigger": CronTrigger(day_of_week="mon", hour=5),
+     "misfire_grace_time": 60 * 60, "max_instances": 1}
+]
 
 if config("LOCAL_CRONS", default=False, cast=bool):
     cron_jobs += SINGLE_CRONS

frontend/app/components/Session/WebPlayer.tsx

@@ -13,6 +13,7 @@ import PlayerContent from './Player/ReplayPlayer/PlayerContent';
 import { IPlayerContext, PlayerContext, defaultContextValue } from './playerContext';
 import { observer } from 'mobx-react-lite';
 import { Note } from "App/services/NotesService";
+import { useParams } from 'react-router-dom'
 
 const TABS = {
   EVENTS: 'User Steps',
@@ -35,6 +36,7 @@ function WebPlayer(props: any) {
   // @ts-ignore
   const [contextValue, setContextValue] = useState<IPlayerContext>(defaultContextValue);
   let playerInst: IPlayerContext['player'];
+  const params: { sessionId: string } = useParams()
 
   useEffect(() => {
     if (!session.sessionId || contextValue.player !== undefined) return;
@@ -91,13 +93,14 @@ function WebPlayer(props: any) {
   // LAYOUT (TODO: local layout state - useContext or something..)
   useEffect(
     () => () => {
+      console.debug('cleaning up player after', params.sessionId)
       toggleFullscreen(false);
       closeBottomBlock();
       playerInst?.clean();
       // @ts-ignore
       setContextValue(defaultContextValue);
     },
-    []
+    [params.sessionId]
   );
 
   const onNoteClose = () => {

frontend/build.sh

@@ -20,7 +20,9 @@ check_prereq() {
 chart=frontend
 [[ $1 == ee ]] && ee=true
 [[ $PATCH -eq 1 ]] && {
-  image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}'  | awk -F. -v OFS=. '{$NF += 1 ; print}')"
+  __app_version="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/${chart} | xargs | awk '{print $2}'  | awk -F. -v OFS=. '{$NF += 1 ; print}' | cut -d 'v' -f2)"
+  sed -i "s/^VERSION = .*/VERSION = $__app_version/g" .env.sample
+  image_tag="v${__app_version}"
   [[ $ee == "true" ]] && { 
     image_tag="${image_tag}-ee"
   }
@@ -30,8 +32,9 @@ update_helm_release() {
   # Update the chart version
   sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml
   # Update image tags
-  sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml
+  sed -i "s#ppVersion.*#ppVersion: \"v${__app_version}\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml
   # Commit the changes
+  git add .env.sample
   git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml
   git commit -m "chore(helm): Updating $chart image release"
 }

scripts/helmcharts/databases/charts/clickhouse/values.yaml

@@ -84,4 +84,4 @@ nodeSelector: {}
 tolerations: []
 
 affinity: {}
-storageSize: 100G
+storageSize: 100Gi

scripts/helmcharts/openreplay-cli

@@ -186,6 +186,12 @@ function or_helm_upgrade() {
 
 function upgrade_old() {
   old_vars_path="$1"
+  [[ -f $old_vars_path ]] || log err "No configuration file ${BWHITE}$old_vars_path${RED}.
+          If you're updating from version older than ${BWHITE}v1.10.0${RED}, for example ${BWHITE}v1.9.0${RED}:
+            ${BWHITE}openreplay --deprecated-upgrade ~/openreplay_v1.9.0/scripts/helmcharts/vars.yaml${RED}.
+          If you're having a custom installation,
+            ${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED}.
+          "
   or_version=$(busybox awk '/fromVersion/{print $2}' < "${old_vars_path}")
   sudo cp "${old_vars_path}" ${OR_DIR}/vars.yaml.backup."${or_version//\"}"_"$(date +%Y%m%d-%H%M%S)" || log err "Not able to copy old vars.yaml"
   sudo cp "${old_vars_path}" ${OR_DIR}/vars.yaml || log err "Not able to copy old vars.yaml"
@@ -266,6 +272,12 @@ function upgrade() {
   # 3. How to update package. Because openreplay -u will be done from old update script
   # 4. Update from Version
   exists git || log err "Git not found. Please install"
+  [[ -f ${OR_DIR}/vars.yaml ]] || log err "No configuration file ${BWHITE}${OR_DIR}/vars.yaml${RED}.
+        If you're updating from version older than ${BWHITE}v1.10.0${RED}, for example ${BWHITE}v1.9.0${RED}:
+          ${BWHITE}openreplay --deprecated-upgrade ~/openreplay_v1.9.0/scripts/helmcharts/vars.yaml${RED}.
+        If you're having a custom installation,
+          ${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED}.
+        "
   or_version=$(busybox awk '/fromVersion/{print $2}' < "${OR_DIR}/vars.yaml") || {
     log err "${BWHITE}${OR_DIR}/vars.yaml${RED} not found. 
         Please do ${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED}

scripts/helmcharts/openreplay/charts/chalice/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.7
+version: 0.1.11
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-AppVersion: "v1.11.7"
+AppVersion: "v1.11.11"

scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml

@@ -43,10 +43,9 @@ spec:
           {{- .Values.healthCheck | toYaml | nindent 10}}
           {{- end}}
           env:
+            {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }}
             - name: KAFKA_SERVERS
               value: "{{ .Values.global.kafka.kafkaHost }}"
-            - name: REDIS_STRING
-              value: "{{ .Values.global.redis.redisHost }}"
             - name: ch_username
               value: "{{ .Values.global.clickhouse.username }}"
             - name: ch_password
@@ -114,6 +113,8 @@ spec:
               value: '{{ .Values.global.s3.region }}'
             - name: sessions_region
               value: '{{ .Values.global.s3.region }}' 
+            - name: ASSIST_RECORDS_BUCKET
+              value: {{ .Values.global.s3.assistRecordsBucket }}
             - name: sessions_bucket
               value: {{ .Values.global.s3.recordingsBucket }}
             - name: sourcemaps_bucket

scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml

@@ -44,7 +44,7 @@ spec:
           {{- end}}
           env:
             - name: CH_USERNAME
-              value: '{{ .Values.global.clickhouse.userame }}'
+              value: '{{ .Values.global.clickhouse.username }}'
             - name: CH_PASSWORD
               value: '{{ .Values.global.clickhouse.password }}'
             - name: CLICKHOUSE_STRING

scripts/helmcharts/openreplay/charts/frontend/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (frontends://semver.org/)
-version: 0.1.7
+version: 0.1.8
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-AppVersion: "v1.11.6"
+AppVersion: "v1.11.7"

scripts/helmcharts/openreplay/charts/sourcemapreader/templates/deployment.yaml

@@ -51,8 +51,7 @@ spec:
               value: '{{ .Values.global.s3.region }}'
             - name: LICENSE_KEY
               value: '{{ .Values.global.enterpriseEditionLicense }}'
-            - name: REDIS_STRING
-              value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}'
+            {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }}
             - name: KAFKA_SERVERS
               value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}'
             - name: KAFKA_USE_SSL

scripts/helmcharts/openreplay/templates/_helpers.tpl

@@ -65,6 +65,7 @@ Create the name of the service account to use
 Create the environment configuration for REDIS_STRING
 */}}
 {{- define "openreplay.env.redis_string" -}}
+{{- if .enabled }}
 {{- $scheme := (eq (.tls | default dict).enabled true) | ternary "rediss" "redis" -}}
 {{- $auth := "" -}}
 {{- if or .existingSecret .redisPassword -}}
@@ -83,6 +84,7 @@ Create the environment configuration for REDIS_STRING
 - name: REDIS_STRING
   value: '{{ $scheme }}://{{ $auth }}{{ .redisHost }}:{{ .redisPort }}'
 {{- end }}
+{{- end }}
 
 {{/*
 Create the volume mount config for redis TLS certificates

scripts/helmcharts/vars.yaml

@@ -50,7 +50,7 @@ kafka: &kafka
     #   value: "3000000"
 
 redis: &redis
-  # enabled: false
+  enabled: true
   redisHost: "redis-master.db.svc.cluster.local"
   redisPort: "6379"
 
@@ -117,6 +117,7 @@ global:
     assetsBucket: "sessions-assets"
     recordingsBucket: "mobs"
     sourcemapsBucket: "sourcemaps"
+    assistRecordsBucket: "records"
     vaultBucket: "vault-data"
     # This is only for enterpriseEdition
     quickwitBucket: "quickwit"