diff --git a/.github/workflows/api-ee.yaml b/.github/workflows/api-ee.yaml index f9a1730f1..ed81c8972 100644 --- a/.github/workflows/api-ee.yaml +++ b/.github/workflows/api-ee.yaml @@ -10,6 +10,7 @@ on: branches: - dev - api-* + - v1.11.0-patch paths: - "ee/api/**" - "api/**" diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index 8e2f7fa7b..451ae64b5 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -10,6 +10,7 @@ on: branches: - dev - api-* + - v1.11.0-patch paths: - "api/**" - "!api/.gitignore" diff --git a/.github/workflows/crons-ee.yaml b/.github/workflows/crons-ee.yaml index 77c098e4e..b357fea65 100644 --- a/.github/workflows/crons-ee.yaml +++ b/.github/workflows/crons-ee.yaml @@ -10,6 +10,7 @@ on: branches: - dev - api-* + - v1.11.0-patch paths: - "ee/api/**" - "api/**" diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index f1fec41fa..155a73272 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -14,9 +14,9 @@ def app_connection_string(name, port, path): HEALTH_ENDPOINTS = { - "alerts": app_connection_string("alerts-openreplay", 8888, "metrics"), + "alerts": app_connection_string("alerts-openreplay", 8888, "health"), "assets": app_connection_string("assets-openreplay", 8888, "metrics"), - "assist": app_connection_string("assist-openreplay", 8888, "metrics"), + "assist": app_connection_string("assist-openreplay", 8888, "health"), "chalice": app_connection_string("chalice-openreplay", 8888, "metrics"), "db": app_connection_string("db-openreplay", 8888, "metrics"), "ender": app_connection_string("ender-openreplay", 8888, "metrics"), diff --git a/api/chalicelib/core/jobs.py b/api/chalicelib/core/jobs.py index 2d244e366..3c812ad33 100644 --- a/api/chalicelib/core/jobs.py +++ b/api/chalicelib/core/jobs.py @@ -1,6 +1,6 @@ from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC -from chalicelib.core import sessions, sessions_mobs +from chalicelib.core import sessions_mobs, sessions_devtool class Actions: @@ -17,11 +17,9 @@ class JobStatus: def get(job_id): with pg_client.PostgresClient() as cur: query = cur.mogrify( - """\ - SELECT - * - FROM public.jobs - WHERE job_id = %(job_id)s;""", + """SELECT * + FROM public.jobs + WHERE job_id = %(job_id)s;""", {"job_id": job_id} ) cur.execute(query=query) @@ -37,11 +35,9 @@ def get(job_id): def get_all(project_id): with pg_client.PostgresClient() as cur: query = cur.mogrify( - """\ - SELECT - * - FROM public.jobs - WHERE project_id = %(project_id)s;""", + """SELECT * + FROM public.jobs + WHERE project_id = %(project_id)s;""", {"project_id": project_id} ) cur.execute(query=query) @@ -51,23 +47,19 @@ def get_all(project_id): return helper.list_to_camel_case(data) -def create(project_id, data): +def create(project_id, user_id): with pg_client.PostgresClient() as cur: - job = { - "status": "scheduled", - "project_id": project_id, - **data - } + job = {"status": "scheduled", + "project_id": project_id, + "action": Actions.DELETE_USER_DATA, + "reference_id": user_id, + "description": f"Delete user sessions of userId = {user_id}", + "start_at": TimeUTC.to_human_readable(TimeUTC.midnight(1))} - query = cur.mogrify("""\ - INSERT INTO public.jobs( - project_id, description, status, action, - reference_id, start_at - ) - VALUES ( - %(project_id)s, %(description)s, %(status)s, %(action)s, - %(reference_id)s, %(start_at)s - ) RETURNING *;""", job) + query = cur.mogrify( + """INSERT INTO public.jobs(project_id, description, status, action,reference_id, start_at) + VALUES (%(project_id)s, %(description)s, %(status)s, %(action)s,%(reference_id)s, %(start_at)s) + RETURNING *;""", job) cur.execute(query=query) @@ -90,14 +82,13 @@ def update(job_id, job): **job } - query = cur.mogrify("""\ - UPDATE public.jobs - SET - updated_at = timezone('utc'::text, now()), - status = %(status)s, - errors = %(errors)s - WHERE - job_id = %(job_id)s RETURNING *;""", job_data) + query = cur.mogrify( + """UPDATE public.jobs + SET updated_at = timezone('utc'::text, now()), + status = %(status)s, + errors = %(errors)s + WHERE job_id = %(job_id)s + RETURNING *;""", job_data) cur.execute(query=query) @@ -113,61 +104,64 @@ def format_datetime(r): r["start_at"] = TimeUTC.datetime_to_timestamp(r["start_at"]) +def __get_session_ids_by_user_ids(project_id, user_ids): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """SELECT session_id + FROM public.sessions + WHERE project_id = %(project_id)s + AND user_id IN %(userId)s + LIMIT 1000;""", + {"project_id": project_id, "userId": tuple(user_ids)}) + cur.execute(query=query) + ids = cur.fetchall() + return [s["session_id"] for s in ids] + + +def __delete_sessions_by_session_ids(session_ids): + with pg_client.PostgresClient(unlimited_query=True) as cur: + query = cur.mogrify( + """DELETE FROM public.sessions + WHERE session_id IN %(session_ids)s""", + {"session_ids": tuple(session_ids)} + ) + cur.execute(query=query) + + def get_scheduled_jobs(): with pg_client.PostgresClient() as cur: query = cur.mogrify( - """\ - SELECT * FROM public.jobs - WHERE status = %(status)s AND start_at <= (now() at time zone 'utc');""", - {"status": JobStatus.SCHEDULED} - ) + """SELECT * + FROM public.jobs + WHERE status = %(status)s + AND start_at <= (now() at time zone 'utc');""", + {"status": JobStatus.SCHEDULED}) cur.execute(query=query) data = cur.fetchall() - for record in data: - format_datetime(record) - return helper.list_to_camel_case(data) def execute_jobs(): jobs = get_scheduled_jobs() - if len(jobs) == 0: - # No jobs to execute - return - for job in jobs: - print(f"job can be executed {job['id']}") + print(f"Executing jobId:{job['jobId']}") try: if job["action"] == Actions.DELETE_USER_DATA: - session_ids = sessions.get_session_ids_by_user_ids( - project_id=job["projectId"], user_ids=job["referenceId"] - ) - - sessions.delete_sessions_by_session_ids(session_ids) - sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) + session_ids = __get_session_ids_by_user_ids(project_id=job["projectId"], + user_ids=[job["referenceId"]]) + if len(session_ids) > 0: + print(f"Deleting {len(session_ids)} sessions") + __delete_sessions_by_session_ids(session_ids) + sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) + sessions_devtool.delete_mobs(session_ids=session_ids, project_id=job["projectId"]) else: - raise Exception(f"The action {job['action']} not supported.") + raise Exception(f"The action '{job['action']}' not supported.") job["status"] = JobStatus.COMPLETED - print(f"job completed {job['id']}") + print(f"Job completed {job['jobId']}") except Exception as e: job["status"] = JobStatus.FAILED - job["error"] = str(e) - print(f"job failed {job['id']}") + job["errors"] = str(e) + print(f"Job failed {job['jobId']}") - update(job["job_id"], job) - - -def group_user_ids_by_project_id(jobs, now): - project_id_user_ids = {} - for job in jobs: - if job["startAt"] > now: - continue - - project_id = job["projectId"] - if project_id not in project_id_user_ids: - project_id_user_ids[project_id] = [] - - project_id_user_ids[project_id].append(job) - - return project_id_user_ids + update(job["jobId"], job) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 8f98aac83..e7283755e 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1065,47 +1065,6 @@ def get_session_user(project_id, user_id): return helper.dict_to_camel_case(data) -def get_session_ids_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient() as cur: - query = cur.mogrify( - """\ - SELECT session_id FROM public.sessions - WHERE - project_id = %(project_id)s AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)} - ) - ids = cur.execute(query=query) - return ids - - -def delete_sessions_by_session_ids(session_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """\ - DELETE FROM public.sessions - WHERE - session_id IN %(session_ids)s;""", - {"session_ids": tuple(session_ids)} - ) - cur.execute(query=query) - - return True - - -def delete_sessions_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """\ - DELETE FROM public.sessions - WHERE - project_id = %(project_id)s AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)} - ) - cur.execute(query=query) - - return True - - def count_all(): with pg_client.PostgresClient(unlimited_query=True) as cur: cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions") diff --git a/api/chalicelib/core/sessions_devtool.py b/api/chalicelib/core/sessions_devtool.py index 6aab5a5e2..50af2bb39 100644 --- a/api/chalicelib/core/sessions_devtool.py +++ b/api/chalicelib/core/sessions_devtool.py @@ -24,3 +24,9 @@ def get_urls(session_id, project_id, check_existence: bool = True): ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) )) return results + + +def delete_mobs(project_id, session_ids): + for session_id in session_ids: + for k in __get_devtools_keys(project_id=project_id, session_id=session_id): + s3.schedule_for_deletion(config("sessions_bucket"), k) diff --git a/api/chalicelib/core/sessions_mobs.py b/api/chalicelib/core/sessions_mobs.py index 68c64fd2b..fb9f8fa9e 100644 --- a/api/chalicelib/core/sessions_mobs.py +++ b/api/chalicelib/core/sessions_mobs.py @@ -57,5 +57,6 @@ def get_ios(session_id): def delete_mobs(project_id, session_ids): for session_id in session_ids: - for k in __get_mob_keys(project_id=project_id, session_id=session_id): + for k in __get_mob_keys(project_id=project_id, session_id=session_id) \ + + __get_mob_keys_deprecated(session_id=session_id): s3.schedule_for_deletion(config("sessions_bucket"), k) diff --git a/api/chalicelib/core/sessions_replay.py b/api/chalicelib/core/sessions_replay.py index 94e3cc504..68991ceca 100644 --- a/api/chalicelib/core/sessions_replay.py +++ b/api/chalicelib/core/sessions_replay.py @@ -69,9 +69,11 @@ def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_ if e['source'] == "js_exception"][:500] data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, session_id=session_id) - data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) - data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) - data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id) + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False) + data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, start_ts=data["startTs"], duration=data["duration"]) @@ -126,9 +128,11 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat if data["platform"] == 'ios': data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) else: - data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) - data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) - data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id) + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False) + data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, diff --git a/api/chalicelib/utils/s3.py b/api/chalicelib/utils/s3.py index 655628602..6eeacd261 100644 --- a/api/chalicelib/utils/s3.py +++ b/api/chalicelib/utils/s3.py @@ -110,11 +110,14 @@ def rename(source_bucket, source_key, target_bucket, target_key): def schedule_for_deletion(bucket, key): + if not exists(bucket, key): + return False s3 = __get_s3_resource() s3_object = s3.Object(bucket, key) s3_object.copy_from(CopySource={'Bucket': bucket, 'Key': key}, - Expires=datetime.now() + timedelta(days=7), + Expires=datetime.utcnow() + timedelta(days=config("SCH_DELETE_DAYS", cast=int, default=30)), MetadataDirective='REPLACE') + return True def generate_file_key(project_id, key): diff --git a/api/env.default b/api/env.default index 074d9b643..7d5266470 100644 --- a/api/env.default +++ b/api/env.default @@ -53,4 +53,5 @@ PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= PYTHONUNBUFFERED=1 -REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 \ No newline at end of file +REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 +SCH_DELETE_DAYS=30 \ No newline at end of file diff --git a/api/routers/crons/core_dynamic_crons.py b/api/routers/crons/core_dynamic_crons.py index a79ce705d..0fdeb242a 100644 --- a/api/routers/crons/core_dynamic_crons.py +++ b/api/routers/crons/core_dynamic_crons.py @@ -1,5 +1,4 @@ from apscheduler.triggers.cron import CronTrigger -from apscheduler.triggers.interval import IntervalTrigger from chalicelib.core import telemetry from chalicelib.core import weekly_report, jobs @@ -20,7 +19,7 @@ async def telemetry_cron() -> None: cron_jobs = [ {"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"), "misfire_grace_time": 60 * 60, "max_instances": 1}, - {"func": run_scheduled_jobs, "trigger": IntervalTrigger(minutes=1), + {"func": run_scheduled_jobs, "trigger": CronTrigger(day_of_week="*", hour=0, minute=15), "misfire_grace_time": 20, "max_instances": 1}, {"func": weekly_report2, "trigger": CronTrigger(day_of_week="mon", hour=5), "misfire_grace_time": 60 * 60, "max_instances": 1} diff --git a/api/routers/subs/v1_api.py b/api/routers/subs/v1_api.py index 0759d0c31..ca98aaf5f 100644 --- a/api/routers/subs/v1_api.py +++ b/api/routers/subs/v1_api.py @@ -2,7 +2,6 @@ from fastapi import Depends, Body import schemas from chalicelib.core import sessions, events, jobs, projects -from chalicelib.utils.TimeUTC import TimeUTC from or_dependencies import OR_context from routers.base import get_routers @@ -15,7 +14,7 @@ async def get_user_sessions(projectKey: str, userId: str, start_date: int = None if projectId is None: return {"errors": ["invalid projectKey"]} return { - 'data': sessions.get_user_sessions( + "data": sessions.get_user_sessions( project_id=projectId, user_id=userId, start_date=start_date, @@ -30,7 +29,7 @@ async def get_session_events(projectKey: str, sessionId: int): if projectId is None: return {"errors": ["invalid projectKey"]} return { - 'data': events.get_by_session_id( + "data": events.get_by_session_id( project_id=projectId, session_id=sessionId ) @@ -43,7 +42,7 @@ async def get_user_details(projectKey: str, userId: str): if projectId is None: return {"errors": ["invalid projectKey"]} return { - 'data': sessions.get_session_user( + "data": sessions.get_session_user( project_id=projectId, user_id=userId ) @@ -55,14 +54,8 @@ async def schedule_to_delete_user_data(projectKey: str, userId: str): projectId = projects.get_internal_project_id(projectKey) if projectId is None: return {"errors": ["invalid projectKey"]} - data = {"action": "delete_user_data", - "reference_id": userId, - "description": f"Delete user sessions of userId = {userId}", - "start_at": TimeUTC.to_human_readable(TimeUTC.midnight(1))} - record = jobs.create(project_id=projectId, data=data) - return { - 'data': record - } + record = jobs.create(project_id=projectId, user_id=userId) + return {"data": record} @app_apikey.get('/v1/{projectKey}/jobs', tags=["api"]) @@ -70,16 +63,12 @@ async def get_jobs(projectKey: str): projectId = projects.get_internal_project_id(projectKey) if projectId is None: return {"errors": ["invalid projectKey"]} - return { - 'data': jobs.get_all(project_id=projectId) - } + return {"data": jobs.get_all(project_id=projectId)} @app_apikey.get('/v1/{projectKey}/jobs/{jobId}', tags=["api"]) async def get_job(projectKey: str, jobId: int): - return { - 'data': jobs.get(job_id=jobId) - } + return {"data": jobs.get(job_id=jobId)} @app_apikey.delete('/v1/{projectKey}/jobs/{jobId}', tags=["api"]) @@ -93,9 +82,7 @@ async def cancel_job(projectKey: str, jobId: int): return {"errors": ["The request job has already been canceled/completed."]} job["status"] = "cancelled" - return { - 'data': jobs.update(job_id=jobId, job=job) - } + return {"data": jobs.update(job_id=jobId, job=job)} @app_apikey.get('/v1/projects', tags=["api"]) @@ -104,15 +91,13 @@ async def get_projects(context: schemas.CurrentContext = Depends(OR_context)): for record in records: del record['projectId'] - return { - 'data': records - } + return {"data": records} @app_apikey.get('/v1/projects/{projectKey}', tags=["api"]) async def get_project(projectKey: str, context: schemas.CurrentContext = Depends(OR_context)): return { - 'data': projects.get_project_by_key(tenant_id=context.tenant_id, project_key=projectKey) + "data": projects.get_project_by_key(tenant_id=context.tenant_id, project_key=projectKey) } @@ -125,5 +110,5 @@ async def create_project(data: schemas.CreateProjectSchema = Body(...), data=data, skip_authorization=True ) - del record['data']['projectId'] + del record["data"]['projectId'] return record diff --git a/api/schemas.py b/api/schemas.py index a7f1eb683..81089e2b2 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -649,6 +649,36 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw): value: Union[List[Union[_SessionSearchEventRaw, str]], str] = Field(...) +def transform_old_FilterType(cls, values): + if values.get("type") is None: + return values + values["type"] = { + "USEROS": FilterType.user_os.value, + "USERBROWSER": FilterType.user_browser.value, + "USERDEVICE": FilterType.user_device.value, + "USERCOUNTRY": FilterType.user_country.value, + "USERID": FilterType.user_id.value, + "USERANONYMOUSID": FilterType.user_anonymous_id.value, + "REFERRER": FilterType.referrer.value, + "REVID": FilterType.rev_id.value, + "USEROS_IOS": FilterType.user_os_ios.value, + "USERDEVICE_IOS": FilterType.user_device_ios.value, + "USERCOUNTRY_IOS": FilterType.user_country_ios.value, + "USERID_IOS": FilterType.user_id_ios.value, + "USERANONYMOUSID_IOS": FilterType.user_anonymous_id_ios.value, + "REVID_IOS": FilterType.rev_id_ios.value, + "DURATION": FilterType.duration.value, + "PLATFORM": FilterType.platform.value, + "METADATA": FilterType.metadata.value, + "ISSUE": FilterType.issue.value, + "EVENTS_COUNT": FilterType.events_count.value, + "UTM_SOURCE": FilterType.utm_source.value, + "UTM_MEDIUM": FilterType.utm_medium.value, + "UTM_CAMPAIGN": FilterType.utm_campaign.value + }.get(values["type"], values["type"]) + return values + + class SessionSearchFilterSchema(__MixedSearchFilter): is_event: bool = Field(False, const=False) # TODO: remove this if there nothing broken from the UI @@ -660,35 +690,7 @@ class SessionSearchFilterSchema(__MixedSearchFilter): source: Optional[Union[ErrorSource, str]] = Field(default=None) filters: List[IssueFilterSchema] = Field(default=[]) - @root_validator(pre=True) - def transform(cls, values): - if values.get("type") is None: - return values - values["type"] = { - "USEROS": FilterType.user_os.value, - "USERBROWSER": FilterType.user_browser.value, - "USERDEVICE": FilterType.user_device.value, - "USERCOUNTRY": FilterType.user_country.value, - "USERID": FilterType.user_id.value, - "USERANONYMOUSID": FilterType.user_anonymous_id.value, - "REFERRER": FilterType.referrer.value, - "REVID": FilterType.rev_id.value, - "USEROS_IOS": FilterType.user_os_ios.value, - "USERDEVICE_IOS": FilterType.user_device_ios.value, - "USERCOUNTRY_IOS": FilterType.user_country_ios.value, - "USERID_IOS": FilterType.user_id_ios.value, - "USERANONYMOUSID_IOS": FilterType.user_anonymous_id_ios.value, - "REVID_IOS": FilterType.rev_id_ios.value, - "DURATION": FilterType.duration.value, - "PLATFORM": FilterType.platform.value, - "METADATA": FilterType.metadata.value, - "ISSUE": FilterType.issue.value, - "EVENTS_COUNT": FilterType.events_count.value, - "UTM_SOURCE": FilterType.utm_source.value, - "UTM_MEDIUM": FilterType.utm_medium.value, - "UTM_CAMPAIGN": FilterType.utm_campaign.value - }.get(values["type"], values["type"]) - return values + transform = root_validator(pre=True, allow_reuse=True)(transform_old_FilterType) @root_validator def filter_validator(cls, values): @@ -1194,6 +1196,8 @@ class LiveSessionSearchFilterSchema(BaseModel): operator: Literal[SearchEventOperator._is, \ SearchEventOperator._contains] = Field(default=SearchEventOperator._contains) + transform = root_validator(pre=True, allow_reuse=True)(transform_old_FilterType) + @root_validator def validator(cls, values): if values.get("type") is not None and values["type"] == LiveFilterType.metadata: diff --git a/ee/api/chalicelib/core/assist_records.py b/ee/api/chalicelib/core/assist_records.py index 18449dc14..be63e1524 100644 --- a/ee/api/chalicelib/core/assist_records.py +++ b/ee/api/chalicelib/core/assist_records.py @@ -41,9 +41,13 @@ def save_record(project_id, data: schemas_ee.AssistRecordSavePayloadSchema, cont def search_records(project_id, data: schemas_ee.AssistRecordSearchPayloadSchema, context: schemas_ee.CurrentContext): conditions = ["projects.tenant_id=%(tenant_id)s", "projects.deleted_at ISNULL", - "assist_records.created_at>=%(startDate)s", - "assist_records.created_at<=%(endDate)s", + "projects.project_id=%(project_id)s", "assist_records.deleted_at ISNULL"] + if data.startDate: + conditions.append("assist_records.created_at>=%(startDate)s") + if data.endDate: + conditions.append("assist_records.created_at<=%(endDate)s") + params = {"tenant_id": context.tenant_id, "project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, "p_start": (data.page - 1) * data.limit, "p_limit": data.limit, diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 70ecdd674..9b47720ee 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -15,9 +15,9 @@ def app_connection_string(name, port, path): HEALTH_ENDPOINTS = { - "alerts": app_connection_string("alerts-openreplay", 8888, "metrics"), + "alerts": app_connection_string("alerts-openreplay", 8888, "health"), "assets": app_connection_string("assets-openreplay", 8888, "metrics"), - "assist": app_connection_string("assist-openreplay", 8888, "metrics"), + "assist": app_connection_string("assist-openreplay", 8888, "health"), "chalice": app_connection_string("chalice-openreplay", 8888, "metrics"), "db": app_connection_string("db-openreplay", 8888, "metrics"), "ender": app_connection_string("ender-openreplay", 8888, "metrics"), diff --git a/ee/api/chalicelib/core/sessions_devtool.py b/ee/api/chalicelib/core/sessions_devtool.py index 9961df360..198466f65 100644 --- a/ee/api/chalicelib/core/sessions_devtool.py +++ b/ee/api/chalicelib/core/sessions_devtool.py @@ -31,3 +31,9 @@ def get_urls(session_id, project_id, context: schemas_ee.CurrentContext, check_e ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) )) return results + + +def delete_mobs(project_id, session_ids): + for session_id in session_ids: + for k in __get_devtools_keys(project_id=project_id, session_id=session_id): + s3.schedule_for_deletion(config("sessions_bucket"), k) diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index 888800681..b7539c3c2 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -1396,47 +1396,6 @@ def get_session_user(project_id, user_id): return helper.dict_to_camel_case(data) -def get_session_ids_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient() as cur: - query = cur.mogrify( - """\ - SELECT session_id FROM public.sessions - WHERE - project_id = %(project_id)s AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)} - ) - ids = cur.execute(query=query) - return ids - - -def delete_sessions_by_session_ids(session_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """\ - DELETE FROM public.sessions - WHERE - session_id IN %(session_ids)s;""", - {"session_ids": tuple(session_ids)} - ) - cur.execute(query=query) - - return True - - -def delete_sessions_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """\ - DELETE FROM public.sessions - WHERE - project_id = %(project_id)s AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)} - ) - cur.execute(query=query) - - return True - - def count_all(): with ch_client.ClickHouseClient() as cur: row = cur.execute(query=f"SELECT COUNT(session_id) AS count FROM {exp_ch_helper.get_main_sessions_table()}") diff --git a/ee/api/chalicelib/core/sessions_replay.py b/ee/api/chalicelib/core/sessions_replay.py index fcbce92eb..c3ecbca9d 100644 --- a/ee/api/chalicelib/core/sessions_replay.py +++ b/ee/api/chalicelib/core/sessions_replay.py @@ -72,10 +72,11 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu if e['source'] == "js_exception"][:500] data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, session_id=session_id) - data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) - data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False) data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, - context=context) + context=context, check_existence=False) data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, start_ts=data["startTs"], duration=data["duration"]) @@ -132,10 +133,11 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat if data["platform"] == 'ios': data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) else: - data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) - data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id, + check_existence=False) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False) data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, - context=context) + context=context, check_existence=False) data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, diff --git a/ee/api/env.default b/ee/api/env.default index 1947e9847..a35f6f451 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -73,4 +73,5 @@ PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= KAFKA_SERVERS=kafka.db.svc.cluster.local:9092 -KAFKA_USE_SSL=false \ No newline at end of file +KAFKA_USE_SSL=false +SCH_DELETE_DAYS=30 \ No newline at end of file diff --git a/ee/api/routers/crons/core_dynamic_crons.py b/ee/api/routers/crons/core_dynamic_crons.py index 5d13c90d1..89846366b 100644 --- a/ee/api/routers/crons/core_dynamic_crons.py +++ b/ee/api/routers/crons/core_dynamic_crons.py @@ -1,5 +1,4 @@ from apscheduler.triggers.cron import CronTrigger -from apscheduler.triggers.interval import IntervalTrigger from decouple import config from chalicelib.core import jobs @@ -29,13 +28,14 @@ cron_jobs = [ {"func": unlock_cron, "trigger": CronTrigger(day="*")}, ] -SINGLE_CRONS = [{"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"), - "misfire_grace_time": 60 * 60, "max_instances": 1}, - {"func": run_scheduled_jobs, "trigger": IntervalTrigger(minutes=60), - "misfire_grace_time": 20, "max_instances": 1}, - {"func": weekly_report, "trigger": CronTrigger(day_of_week="mon", hour=5), - "misfire_grace_time": 60 * 60, "max_instances": 1} - ] +SINGLE_CRONS = [ + {"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"), + "misfire_grace_time": 60 * 60, "max_instances": 1}, + {"func": run_scheduled_jobs, "trigger": CronTrigger(day_of_week="*", hour=0, minute=15), + "misfire_grace_time": 20, "max_instances": 1}, + {"func": weekly_report, "trigger": CronTrigger(day_of_week="mon", hour=5), + "misfire_grace_time": 60 * 60, "max_instances": 1} +] if config("LOCAL_CRONS", default=False, cast=bool): cron_jobs += SINGLE_CRONS diff --git a/ee/api/schemas_ee.py b/ee/api/schemas_ee.py index f5a3b8de0..50136ac02 100644 --- a/ee/api/schemas_ee.py +++ b/ee/api/schemas_ee.py @@ -137,8 +137,8 @@ class AssistRecordSavePayloadSchema(AssistRecordPayloadSchema): class AssistRecordSearchPayloadSchema(schemas._PaginatedSchema): limit: int = Field(default=200, gt=0) - startDate: int = Field(default=TimeUTC.now(-7)) - endDate: int = Field(default=TimeUTC.now(1)) + startDate: Optional[int] = Field(default=None) + endDate: Optional[int] = Field(default=None) user_id: Optional[int] = Field(default=None) query: Optional[str] = Field(default=None) order: Literal["asc", "desc"] = Field(default="desc") diff --git a/frontend/build.sh b/frontend/build.sh index 0eb9cc725..e95aa35ff 100644 --- a/frontend/build.sh +++ b/frontend/build.sh @@ -20,7 +20,9 @@ check_prereq() { chart=frontend [[ $1 == ee ]] && ee=true [[ $PATCH -eq 1 ]] && { - image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + __app_version="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/${chart} | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}' | cut -d 'v' -f2)" + sed -i "s/^VERSION = .*/VERSION = $__app_version/g" .env.sample + image_tag="v${__app_version}" [[ $ee == "true" ]] && { image_tag="${image_tag}-ee" } @@ -31,8 +33,9 @@ update_helm_release() { # Update the chart version sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml # Update image tags - sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + sed -i "s#ppVersion.*#ppVersion: \"v${__app_version}\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml # Commit the changes + git add .env.sample git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml git commit -m "chore(helm): Updating $chart image release" } diff --git a/scripts/helmcharts/databases/charts/clickhouse/values.yaml b/scripts/helmcharts/databases/charts/clickhouse/values.yaml index f3f39aefd..05a10c879 100644 --- a/scripts/helmcharts/databases/charts/clickhouse/values.yaml +++ b/scripts/helmcharts/databases/charts/clickhouse/values.yaml @@ -84,4 +84,4 @@ nodeSelector: {} tolerations: [] affinity: {} -storageSize: 100G +storageSize: 100Gi diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index addb7ec3a..82caf9dfa 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -190,6 +190,12 @@ function or_helm_upgrade() { function upgrade_old() { old_vars_path="$1" + [[ -f $old_vars_path ]] || log err "No configuration file ${BWHITE}$old_vars_path${RED}. + If you're updating from version older than ${BWHITE}v1.10.0${RED}, for example ${BWHITE}v1.9.0${RED}: + ${BWHITE}openreplay --deprecated-upgrade ~/openreplay_v1.9.0/scripts/helmcharts/vars.yaml${RED}. + If you're having a custom installation, + ${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED}. + " or_version=$(busybox awk '/fromVersion/{print $2}' < "${old_vars_path}") sudo cp "${old_vars_path}" ${OR_DIR}/vars.yaml.backup."${or_version//\"}"_"$(date +%Y%m%d-%H%M%S)" || log err "Not able to copy old vars.yaml" sudo cp "${old_vars_path}" ${OR_DIR}/vars.yaml || log err "Not able to copy old vars.yaml" @@ -271,6 +277,12 @@ function upgrade() { # 3. How to update package. Because openreplay -u will be done from old update script # 4. Update from Version exists git || log err "Git not found. Please install" + [[ -f ${OR_DIR}/vars.yaml ]] || log err "No configuration file ${BWHITE}${OR_DIR}/vars.yaml${RED}. + If you're updating from version older than ${BWHITE}v1.10.0${RED}, for example ${BWHITE}v1.9.0${RED}: + ${BWHITE}openreplay --deprecated-upgrade ~/openreplay_v1.9.0/scripts/helmcharts/vars.yaml${RED}. + If you're having a custom installation, + ${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED}. + " or_version=$(busybox awk '/fromVersion/{print $2}' < "${OR_DIR}/vars.yaml") || { log err "${BWHITE}${OR_DIR}/vars.yaml${RED} not found. Please do ${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED} diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index 1efa5bccf..e75ef91b2 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -43,10 +43,9 @@ spec: {{- .Values.healthCheck | toYaml | nindent 10}} {{- end}} env: + {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} - name: KAFKA_SERVERS value: "{{ .Values.global.kafka.kafkaHost }}" - - name: REDIS_STRING - value: "{{ .Values.global.redis.redisHost }}" - name: ch_username value: "{{ .Values.global.clickhouse.username }}" - name: ch_password @@ -114,6 +113,8 @@ spec: value: '{{ .Values.global.s3.region }}' - name: sessions_region value: '{{ .Values.global.s3.region }}' + - name: ASSIST_RECORDS_BUCKET + value: {{ .Values.global.s3.assistRecordsBucket }} - name: sessions_bucket value: {{ .Values.global.s3.recordingsBucket }} - name: sourcemaps_bucket diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 4c48b54bd..46bad0f00 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -50,7 +50,7 @@ kafka: &kafka # value: "3000000" redis: &redis - # enabled: false + enabled: true redisHost: "redis-master.db.svc.cluster.local" redisPort: "6379" @@ -117,6 +117,7 @@ global: assetsBucket: "sessions-assets" recordingsBucket: "mobs" sourcemapsBucket: "sourcemaps" + assistRecordsBucket: "records" vaultBucket: "vault-data" # This is only for enterpriseEdition quickwitBucket: "quickwit" diff --git a/static/openreplay-git-hero.svg b/static/openreplay-git-hero.svg index bf294a353..079ac5efc 100644 --- a/static/openreplay-git-hero.svg +++ b/static/openreplay-git-hero.svg @@ -1,421 +1,390 @@ - + - - + + - + - + - - - + + + - + - - - - - - - - - + + + + + + + + + - + - - - + + + - - - + + + - + - - + + - + - + - - - + + + - + - - + + - + - - - - + + + + - + - - - + + + - + - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - + + - - - - - - - - - - - - - - + + + - - + + - - - + + - - + + - - - + + - - + + - - + + + - - + - - - + + - + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - + + + - + - - - + + + - + + - - - + + + - - + - - - + + + - + - - - + + - + - - + + + + - + - - - - + + + - + - - - + + + + + + + + + + + + + + - - - - - - - - - - - + + + - - - - - - - - - - + + + + + - + - + - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - + + + + + + + + + + + + + - - - - - - - - + + + - - - - - - + - + + + + - + - + - + - + - + + + + + + + + + + + + + + + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + + + + diff --git a/static/replay-thumbnail.svg b/static/replay-thumbnail.svg new file mode 100644 index 000000000..9f550f423 --- /dev/null +++ b/static/replay-thumbnail.svg @@ -0,0 +1,148 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +