jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1221 from rjshrjndrn/rebased_dev

Browse source 
Took all delta commits(after v1.11.0)  from main to dev
This commit is contained in:
Rajesh Rajendran 2023-05-02 12:47:20 +02:00 • committed by GitHub
commit 5c8e82558a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 600 additions and 535 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/api-ee.yaml vendored
View file

@ -10,6 +10,7 @@ on:
branches:
- dev
- api-*
- v1.11.0-patch
paths:
- "ee/api/**"
- "api/**"

1
.github/workflows/api.yaml vendored
View file

@ -10,6 +10,7 @@ on:
branches:
- dev
- api-*
- v1.11.0-patch
paths:
- "api/**"
- "!api/.gitignore"

1
.github/workflows/crons-ee.yaml vendored
View file

@ -10,6 +10,7 @@ on:
branches:
- dev
- api-*
- v1.11.0-patch
paths:
- "ee/api/**"
- "api/**"

4
api/chalicelib/core/health.py
View file

@ -14,9 +14,9 @@ def app_connection_string(name, port, path):
HEALTH_ENDPOINTS = {
"alerts": app_connection_string("alerts-openreplay", 8888, "metrics"),
"alerts": app_connection_string("alerts-openreplay", 8888, "health"),
"assets": app_connection_string("assets-openreplay", 8888, "metrics"),
"assist": app_connection_string("assist-openreplay", 8888, "metrics"),
"assist": app_connection_string("assist-openreplay", 8888, "health"),
"chalice": app_connection_string("chalice-openreplay", 8888, "metrics"),
"db": app_connection_string("db-openreplay", 8888, "metrics"),
"ender": app_connection_string("ender-openreplay", 8888, "metrics"),

140
api/chalicelib/core/jobs.py
View file

@ -1,6 +1,6 @@
from chalicelib.utils import pg_client, helper
from chalicelib.utils.TimeUTC import TimeUTC
from chalicelib.core import sessions, sessions_mobs
from chalicelib.core import sessions_mobs, sessions_devtool
class Actions:
@ -17,11 +17,9 @@ class JobStatus:
def get(job_id):
with pg_client.PostgresClient() as cur:
query = cur.mogrify(
"""\
SELECT
*
FROM public.jobs
WHERE job_id = %(job_id)s;""",
"""SELECT *
FROM public.jobs
WHERE job_id = %(job_id)s;""",
{"job_id": job_id}
)
cur.execute(query=query)
@ -37,11 +35,9 @@ def get(job_id):
def get_all(project_id):
with pg_client.PostgresClient() as cur:
query = cur.mogrify(
"""\
SELECT
*
FROM public.jobs
WHERE project_id = %(project_id)s;""",
"""SELECT *
FROM public.jobs
WHERE project_id = %(project_id)s;""",
{"project_id": project_id}
)
cur.execute(query=query)
@ -51,23 +47,19 @@ def get_all(project_id):
return helper.list_to_camel_case(data)
def create(project_id, data):
def create(project_id, user_id):
with pg_client.PostgresClient() as cur:
job = {
"status": "scheduled",
"project_id": project_id,
**data
}
job = {"status": "scheduled",
"project_id": project_id,
"action": Actions.DELETE_USER_DATA,
"reference_id": user_id,
"description": f"Delete user sessions of userId = {user_id}",
"start_at": TimeUTC.to_human_readable(TimeUTC.midnight(1))}
query = cur.mogrify("""\
INSERT INTO public.jobs(
project_id, description, status, action,
reference_id, start_at
)
VALUES (
%(project_id)s, %(description)s, %(status)s, %(action)s,
%(reference_id)s, %(start_at)s
) RETURNING *;""", job)
query = cur.mogrify(
"""INSERT INTO public.jobs(project_id, description, status, action,reference_id, start_at)
VALUES (%(project_id)s, %(description)s, %(status)s, %(action)s,%(reference_id)s, %(start_at)s)
RETURNING *;""", job)
cur.execute(query=query)
@ -90,14 +82,13 @@ def update(job_id, job):
**job
}
query = cur.mogrify("""\
UPDATE public.jobs
SET
updated_at = timezone('utc'::text, now()),
status = %(status)s,
errors = %(errors)s
WHERE
job_id = %(job_id)s RETURNING *;""", job_data)
query = cur.mogrify(
"""UPDATE public.jobs
SET updated_at = timezone('utc'::text, now()),
status = %(status)s,
errors = %(errors)s
WHERE job_id = %(job_id)s
RETURNING *;""", job_data)
cur.execute(query=query)
@ -113,61 +104,64 @@ def format_datetime(r):
r["start_at"] = TimeUTC.datetime_to_timestamp(r["start_at"])
def __get_session_ids_by_user_ids(project_id, user_ids):
with pg_client.PostgresClient() as cur:
query = cur.mogrify(
"""SELECT session_id
FROM public.sessions
WHERE project_id = %(project_id)s
AND user_id IN %(userId)s
LIMIT 1000;""",
{"project_id": project_id, "userId": tuple(user_ids)})
cur.execute(query=query)
ids = cur.fetchall()
return [s["session_id"] for s in ids]
def __delete_sessions_by_session_ids(session_ids):
with pg_client.PostgresClient(unlimited_query=True) as cur:
query = cur.mogrify(
"""DELETE FROM public.sessions
WHERE session_id IN %(session_ids)s""",
{"session_ids": tuple(session_ids)}
)
cur.execute(query=query)
def get_scheduled_jobs():
with pg_client.PostgresClient() as cur:
query = cur.mogrify(
"""\
SELECT * FROM public.jobs
WHERE status = %(status)s AND start_at <= (now() at time zone 'utc');""",
{"status": JobStatus.SCHEDULED}
)
"""SELECT *
FROM public.jobs
WHERE status = %(status)s
AND start_at <= (now() at time zone 'utc');""",
{"status": JobStatus.SCHEDULED})
cur.execute(query=query)
data = cur.fetchall()
for record in data:
format_datetime(record)
return helper.list_to_camel_case(data)
def execute_jobs():
jobs = get_scheduled_jobs()
if len(jobs) == 0:
# No jobs to execute
return
for job in jobs:
print(f"job can be executed {job['id']}")
print(f"Executing jobId:{job['jobId']}")
try:
if job["action"] == Actions.DELETE_USER_DATA:
session_ids = sessions.get_session_ids_by_user_ids(
project_id=job["projectId"], user_ids=job["referenceId"]
)
sessions.delete_sessions_by_session_ids(session_ids)
sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"])
session_ids = __get_session_ids_by_user_ids(project_id=job["projectId"],
user_ids=[job["referenceId"]])
if len(session_ids) > 0:
print(f"Deleting {len(session_ids)} sessions")
__delete_sessions_by_session_ids(session_ids)
sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"])
sessions_devtool.delete_mobs(session_ids=session_ids, project_id=job["projectId"])
else:
raise Exception(f"The action {job['action']} not supported.")
raise Exception(f"The action '{job['action']}' not supported.")
job["status"] = JobStatus.COMPLETED
print(f"job completed {job['id']}")
print(f"Job completed {job['jobId']}")
except Exception as e:
job["status"] = JobStatus.FAILED
job["error"] = str(e)
print(f"job failed {job['id']}")
job["errors"] = str(e)
print(f"Job failed {job['jobId']}")
update(job["job_id"], job)
def group_user_ids_by_project_id(jobs, now):
project_id_user_ids = {}
for job in jobs:
if job["startAt"] > now:
continue
project_id = job["projectId"]
if project_id not in project_id_user_ids:
project_id_user_ids[project_id] = []
project_id_user_ids[project_id].append(job)
return project_id_user_ids
update(job["jobId"], job)

41
api/chalicelib/core/sessions.py
View file

@ -1065,47 +1065,6 @@ def get_session_user(project_id, user_id):
return helper.dict_to_camel_case(data)
def get_session_ids_by_user_ids(project_id, user_ids):
with pg_client.PostgresClient() as cur:
query = cur.mogrify(
"""\
SELECT session_id FROM public.sessions
WHERE
project_id = %(project_id)s AND user_id IN %(userId)s;""",
{"project_id": project_id, "userId": tuple(user_ids)}
)
ids = cur.execute(query=query)
return ids
def delete_sessions_by_session_ids(session_ids):
with pg_client.PostgresClient(unlimited_query=True) as cur:
query = cur.mogrify(
"""\
DELETE FROM public.sessions
WHERE
session_id IN %(session_ids)s;""",
{"session_ids": tuple(session_ids)}
)
cur.execute(query=query)
return True
def delete_sessions_by_user_ids(project_id, user_ids):
with pg_client.PostgresClient(unlimited_query=True) as cur:
query = cur.mogrify(
"""\
DELETE FROM public.sessions
WHERE
project_id = %(project_id)s AND user_id IN %(userId)s;""",
{"project_id": project_id, "userId": tuple(user_ids)}
)
cur.execute(query=query)
return True
def count_all():
with pg_client.PostgresClient(unlimited_query=True) as cur:
cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions")

6
api/chalicelib/core/sessions_devtool.py
View file

@ -24,3 +24,9 @@ def get_urls(session_id, project_id, check_existence: bool = True):
ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900)
))
return results
def delete_mobs(project_id, session_ids):
for session_id in session_ids:
for k in __get_devtools_keys(project_id=project_id, session_id=session_id):
s3.schedule_for_deletion(config("sessions_bucket"), k)

3
api/chalicelib/core/sessions_mobs.py
View file

@ -57,5 +57,6 @@ def get_ios(session_id):
def delete_mobs(project_id, session_ids):
for session_id in session_ids:
for k in __get_mob_keys(project_id=project_id, session_id=session_id):
for k in __get_mob_keys(project_id=project_id, session_id=session_id) \
+ __get_mob_keys_deprecated(session_id=session_id):
s3.schedule_for_deletion(config("sessions_bucket"), k)

16
api/chalicelib/core/sessions_replay.py
View file

@ -69,9 +69,11 @@ def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_
if e['source'] == "js_exception"][:500]
data['userEvents'] = events.get_customs_by_session_id(project_id=project_id,
session_id=session_id)
data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id)
data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id)
data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id)
data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id,
check_existence=False)
data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False)
data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id,
check_existence=False)
data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id,
start_ts=data["startTs"], duration=data["duration"])
@ -126,9 +128,11 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat
if data["platform"] == 'ios':
data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id)
else:
data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id)
data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id)
data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id)
data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id,
check_existence=False)
data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False)
data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id,
check_existence=False)
data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data)
data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id,

5
api/chalicelib/utils/s3.py
View file

@ -110,11 +110,14 @@ def rename(source_bucket, source_key, target_bucket, target_key):
def schedule_for_deletion(bucket, key):
if not exists(bucket, key):
return False
s3 = __get_s3_resource()
s3_object = s3.Object(bucket, key)
s3_object.copy_from(CopySource={'Bucket': bucket, 'Key': key},
Expires=datetime.now() + timedelta(days=7),
Expires=datetime.utcnow() + timedelta(days=config("SCH_DELETE_DAYS", cast=int, default=30)),
MetadataDirective='REPLACE')
return True
def generate_file_key(project_id, key):

3
api/env.default
View file

@ -53,4 +53,5 @@ PRESIGNED_URL_EXPIRATION=3600
ASSIST_JWT_EXPIRATION=144000
ASSIST_JWT_SECRET=
PYTHONUNBUFFERED=1
REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379
REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379
SCH_DELETE_DAYS=30

3
api/routers/crons/core_dynamic_crons.py
View file

@ -1,5 +1,4 @@
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.interval import IntervalTrigger
from chalicelib.core import telemetry
from chalicelib.core import weekly_report, jobs
@ -20,7 +19,7 @@ async def telemetry_cron() -> None:
cron_jobs = [
{"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"),
"misfire_grace_time": 60 * 60, "max_instances": 1},
{"func": run_scheduled_jobs, "trigger": IntervalTrigger(minutes=1),
{"func": run_scheduled_jobs, "trigger": CronTrigger(day_of_week="*", hour=0, minute=15),
"misfire_grace_time": 20, "max_instances": 1},
{"func": weekly_report2, "trigger": CronTrigger(day_of_week="mon", hour=5),
"misfire_grace_time": 60 * 60, "max_instances": 1}

37
api/routers/subs/v1_api.py
View file

@ -2,7 +2,6 @@ from fastapi import Depends, Body
import schemas
from chalicelib.core import sessions, events, jobs, projects
from chalicelib.utils.TimeUTC import TimeUTC
from or_dependencies import OR_context
from routers.base import get_routers
@ -15,7 +14,7 @@ async def get_user_sessions(projectKey: str, userId: str, start_date: int = None
if projectId is None:
return {"errors": ["invalid projectKey"]}
return {
'data': sessions.get_user_sessions(
"data": sessions.get_user_sessions(
project_id=projectId,
user_id=userId,
start_date=start_date,
@ -30,7 +29,7 @@ async def get_session_events(projectKey: str, sessionId: int):
if projectId is None:
return {"errors": ["invalid projectKey"]}
return {
'data': events.get_by_session_id(
"data": events.get_by_session_id(
project_id=projectId,
session_id=sessionId
)
@ -43,7 +42,7 @@ async def get_user_details(projectKey: str, userId: str):
if projectId is None:
return {"errors": ["invalid projectKey"]}
return {
'data': sessions.get_session_user(
"data": sessions.get_session_user(
project_id=projectId,
user_id=userId
)
@ -55,14 +54,8 @@ async def schedule_to_delete_user_data(projectKey: str, userId: str):
projectId = projects.get_internal_project_id(projectKey)
if projectId is None:
return {"errors": ["invalid projectKey"]}
data = {"action": "delete_user_data",
"reference_id": userId,
"description": f"Delete user sessions of userId = {userId}",
"start_at": TimeUTC.to_human_readable(TimeUTC.midnight(1))}
record = jobs.create(project_id=projectId, data=data)
return {
'data': record
}
record = jobs.create(project_id=projectId, user_id=userId)
return {"data": record}
@app_apikey.get('/v1/{projectKey}/jobs', tags=["api"])
@ -70,16 +63,12 @@ async def get_jobs(projectKey: str):
projectId = projects.get_internal_project_id(projectKey)
if projectId is None:
return {"errors": ["invalid projectKey"]}
return {
'data': jobs.get_all(project_id=projectId)
}
return {"data": jobs.get_all(project_id=projectId)}
@app_apikey.get('/v1/{projectKey}/jobs/{jobId}', tags=["api"])
async def get_job(projectKey: str, jobId: int):
return {
'data': jobs.get(job_id=jobId)
}
return {"data": jobs.get(job_id=jobId)}
@app_apikey.delete('/v1/{projectKey}/jobs/{jobId}', tags=["api"])
@ -93,9 +82,7 @@ async def cancel_job(projectKey: str, jobId: int):
return {"errors": ["The request job has already been canceled/completed."]}
job["status"] = "cancelled"
return {
'data': jobs.update(job_id=jobId, job=job)
}
return {"data": jobs.update(job_id=jobId, job=job)}
@app_apikey.get('/v1/projects', tags=["api"])
@ -104,15 +91,13 @@ async def get_projects(context: schemas.CurrentContext = Depends(OR_context)):
for record in records:
del record['projectId']
return {
'data': records
}
return {"data": records}
@app_apikey.get('/v1/projects/{projectKey}', tags=["api"])
async def get_project(projectKey: str, context: schemas.CurrentContext = Depends(OR_context)):
return {
'data': projects.get_project_by_key(tenant_id=context.tenant_id, project_key=projectKey)
"data": projects.get_project_by_key(tenant_id=context.tenant_id, project_key=projectKey)
}
@ -125,5 +110,5 @@ async def create_project(data: schemas.CreateProjectSchema = Body(...),
data=data,
skip_authorization=True
)
del record['data']['projectId']
del record["data"]['projectId']
return record

62
api/schemas.py
View file

@ -649,6 +649,36 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw):
value: Union[List[Union[_SessionSearchEventRaw, str]], str] = Field(...)
def transform_old_FilterType(cls, values):
if values.get("type") is None:
return values
values["type"] = {
"USEROS": FilterType.user_os.value,
"USERBROWSER": FilterType.user_browser.value,
"USERDEVICE": FilterType.user_device.value,
"USERCOUNTRY": FilterType.user_country.value,
"USERID": FilterType.user_id.value,
"USERANONYMOUSID": FilterType.user_anonymous_id.value,
"REFERRER": FilterType.referrer.value,
"REVID": FilterType.rev_id.value,
"USEROS_IOS": FilterType.user_os_ios.value,
"USERDEVICE_IOS": FilterType.user_device_ios.value,
"USERCOUNTRY_IOS": FilterType.user_country_ios.value,
"USERID_IOS": FilterType.user_id_ios.value,
"USERANONYMOUSID_IOS": FilterType.user_anonymous_id_ios.value,
"REVID_IOS": FilterType.rev_id_ios.value,
"DURATION": FilterType.duration.value,
"PLATFORM": FilterType.platform.value,
"METADATA": FilterType.metadata.value,
"ISSUE": FilterType.issue.value,
"EVENTS_COUNT": FilterType.events_count.value,
"UTM_SOURCE": FilterType.utm_source.value,
"UTM_MEDIUM": FilterType.utm_medium.value,
"UTM_CAMPAIGN": FilterType.utm_campaign.value
}.get(values["type"], values["type"])
return values
class SessionSearchFilterSchema(__MixedSearchFilter):
is_event: bool = Field(False, const=False)
# TODO: remove this if there nothing broken from the UI
@ -660,35 +690,7 @@ class SessionSearchFilterSchema(__MixedSearchFilter):
source: Optional[Union[ErrorSource, str]] = Field(default=None)
filters: List[IssueFilterSchema] = Field(default=[])
@root_validator(pre=True)
def transform(cls, values):
if values.get("type") is None:
return values
values["type"] = {
"USEROS": FilterType.user_os.value,
"USERBROWSER": FilterType.user_browser.value,
"USERDEVICE": FilterType.user_device.value,
"USERCOUNTRY": FilterType.user_country.value,
"USERID": FilterType.user_id.value,
"USERANONYMOUSID": FilterType.user_anonymous_id.value,
"REFERRER": FilterType.referrer.value,
"REVID": FilterType.rev_id.value,
"USEROS_IOS": FilterType.user_os_ios.value,
"USERDEVICE_IOS": FilterType.user_device_ios.value,
"USERCOUNTRY_IOS": FilterType.user_country_ios.value,
"USERID_IOS": FilterType.user_id_ios.value,
"USERANONYMOUSID_IOS": FilterType.user_anonymous_id_ios.value,
"REVID_IOS": FilterType.rev_id_ios.value,
"DURATION": FilterType.duration.value,
"PLATFORM": FilterType.platform.value,
"METADATA": FilterType.metadata.value,
"ISSUE": FilterType.issue.value,
"EVENTS_COUNT": FilterType.events_count.value,
"UTM_SOURCE": FilterType.utm_source.value,
"UTM_MEDIUM": FilterType.utm_medium.value,
"UTM_CAMPAIGN": FilterType.utm_campaign.value
}.get(values["type"], values["type"])
return values
transform = root_validator(pre=True, allow_reuse=True)(transform_old_FilterType)
@root_validator
def filter_validator(cls, values):
@ -1194,6 +1196,8 @@ class LiveSessionSearchFilterSchema(BaseModel):
operator: Literal[SearchEventOperator._is, \
SearchEventOperator._contains] = Field(default=SearchEventOperator._contains)
transform = root_validator(pre=True, allow_reuse=True)(transform_old_FilterType)
@root_validator
def validator(cls, values):
if values.get("type") is not None and values["type"] == LiveFilterType.metadata:

8
ee/api/chalicelib/core/assist_records.py
View file

@ -41,9 +41,13 @@ def save_record(project_id, data: schemas_ee.AssistRecordSavePayloadSchema, cont
def search_records(project_id, data: schemas_ee.AssistRecordSearchPayloadSchema, context: schemas_ee.CurrentContext):
conditions = ["projects.tenant_id=%(tenant_id)s",
"projects.deleted_at ISNULL",
"assist_records.created_at>=%(startDate)s",
"assist_records.created_at<=%(endDate)s",
"projects.project_id=%(project_id)s",
"assist_records.deleted_at ISNULL"]
if data.startDate:
conditions.append("assist_records.created_at>=%(startDate)s")
if data.endDate:
conditions.append("assist_records.created_at<=%(endDate)s")
params = {"tenant_id": context.tenant_id, "project_id": project_id,
"startDate": data.startDate, "endDate": data.endDate,
"p_start": (data.page - 1) * data.limit, "p_limit": data.limit,

4
ee/api/chalicelib/core/health.py
View file

@ -15,9 +15,9 @@ def app_connection_string(name, port, path):
HEALTH_ENDPOINTS = {
"alerts": app_connection_string("alerts-openreplay", 8888, "metrics"),
"alerts": app_connection_string("alerts-openreplay", 8888, "health"),
"assets": app_connection_string("assets-openreplay", 8888, "metrics"),
"assist": app_connection_string("assist-openreplay", 8888, "metrics"),
"assist": app_connection_string("assist-openreplay", 8888, "health"),
"chalice": app_connection_string("chalice-openreplay", 8888, "metrics"),
"db": app_connection_string("db-openreplay", 8888, "metrics"),
"ender": app_connection_string("ender-openreplay", 8888, "metrics"),

6
ee/api/chalicelib/core/sessions_devtool.py
View file

@ -31,3 +31,9 @@ def get_urls(session_id, project_id, context: schemas_ee.CurrentContext, check_e
ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900)
))
return results
def delete_mobs(project_id, session_ids):
for session_id in session_ids:
for k in __get_devtools_keys(project_id=project_id, session_id=session_id):
s3.schedule_for_deletion(config("sessions_bucket"), k)

41
ee/api/chalicelib/core/sessions_exp.py
View file

@ -1396,47 +1396,6 @@ def get_session_user(project_id, user_id):
return helper.dict_to_camel_case(data)
def get_session_ids_by_user_ids(project_id, user_ids):
with pg_client.PostgresClient() as cur:
query = cur.mogrify(
"""\
SELECT session_id FROM public.sessions
WHERE
project_id = %(project_id)s AND user_id IN %(userId)s;""",
{"project_id": project_id, "userId": tuple(user_ids)}
)
ids = cur.execute(query=query)
return ids
def delete_sessions_by_session_ids(session_ids):
with pg_client.PostgresClient(unlimited_query=True) as cur:
query = cur.mogrify(
"""\
DELETE FROM public.sessions
WHERE
session_id IN %(session_ids)s;""",
{"session_ids": tuple(session_ids)}
)
cur.execute(query=query)
return True
def delete_sessions_by_user_ids(project_id, user_ids):
with pg_client.PostgresClient(unlimited_query=True) as cur:
query = cur.mogrify(
"""\
DELETE FROM public.sessions
WHERE
project_id = %(project_id)s AND user_id IN %(userId)s;""",
{"project_id": project_id, "userId": tuple(user_ids)}
)
cur.execute(query=query)
return True
def count_all():
with ch_client.ClickHouseClient() as cur:
row = cur.execute(query=f"SELECT COUNT(session_id) AS count FROM {exp_ch_helper.get_main_sessions_table()}")

14
ee/api/chalicelib/core/sessions_replay.py
View file

@ -72,10 +72,11 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu
if e['source'] == "js_exception"][:500]
data['userEvents'] = events.get_customs_by_session_id(project_id=project_id,
session_id=session_id)
data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id)
data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id)
data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id,
check_existence=False)
data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False)
data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id,
context=context)
context=context, check_existence=False)
data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id,
start_ts=data["startTs"], duration=data["duration"])
@ -132,10 +133,11 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat
if data["platform"] == 'ios':
data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id)
else:
data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id)
data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id)
data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id,
check_existence=False)
data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id, check_existence=False)
data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id,
context=context)
context=context, check_existence=False)
data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data)
data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id,

3
ee/api/env.default
View file

@ -73,4 +73,5 @@ PRESIGNED_URL_EXPIRATION=3600
ASSIST_JWT_EXPIRATION=144000
ASSIST_JWT_SECRET=
KAFKA_SERVERS=kafka.db.svc.cluster.local:9092
KAFKA_USE_SSL=false
KAFKA_USE_SSL=false
SCH_DELETE_DAYS=30

16
ee/api/routers/crons/core_dynamic_crons.py
View file

@ -1,5 +1,4 @@
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.interval import IntervalTrigger
from decouple import config
from chalicelib.core import jobs
@ -29,13 +28,14 @@ cron_jobs = [
{"func": unlock_cron, "trigger": CronTrigger(day="*")},
]
SINGLE_CRONS = [{"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"),
"misfire_grace_time": 60 * 60, "max_instances": 1},
{"func": run_scheduled_jobs, "trigger": IntervalTrigger(minutes=60),
"misfire_grace_time": 20, "max_instances": 1},
{"func": weekly_report, "trigger": CronTrigger(day_of_week="mon", hour=5),
"misfire_grace_time": 60 * 60, "max_instances": 1}
]
SINGLE_CRONS = [
{"func": telemetry_cron, "trigger": CronTrigger(day_of_week="*"),
"misfire_grace_time": 60 * 60, "max_instances": 1},
{"func": run_scheduled_jobs, "trigger": CronTrigger(day_of_week="*", hour=0, minute=15),
"misfire_grace_time": 20, "max_instances": 1},
{"func": weekly_report, "trigger": CronTrigger(day_of_week="mon", hour=5),
"misfire_grace_time": 60 * 60, "max_instances": 1}
]
if config("LOCAL_CRONS", default=False, cast=bool):
cron_jobs += SINGLE_CRONS

4
ee/api/schemas_ee.py
View file

@ -137,8 +137,8 @@ class AssistRecordSavePayloadSchema(AssistRecordPayloadSchema):
class AssistRecordSearchPayloadSchema(schemas._PaginatedSchema):
limit: int = Field(default=200, gt=0)
startDate: int = Field(default=TimeUTC.now(-7))
endDate: int = Field(default=TimeUTC.now(1))
startDate: Optional[int] = Field(default=None)
endDate: Optional[int] = Field(default=None)
user_id: Optional[int] = Field(default=None)
query: Optional[str] = Field(default=None)
order: Literal["asc", "desc"] = Field(default="desc")

7
frontend/build.sh
View file

@ -20,7 +20,9 @@ check_prereq() {
chart=frontend
[[ $1 == ee ]] && ee=true
[[ $PATCH -eq 1 ]] && {
image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')"
__app_version="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/${chart} | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}' | cut -d 'v' -f2)"
sed -i "s/^VERSION = .*/VERSION = $__app_version/g" .env.sample
image_tag="v${__app_version}"
[[ $ee == "true" ]] && {
image_tag="${image_tag}-ee"
}
@ -31,8 +33,9 @@ update_helm_release() {
# Update the chart version
sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml
# Update image tags
sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml
sed -i "s#ppVersion.*#ppVersion: \"v${__app_version}\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml
# Commit the changes
git add .env.sample
git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml
git commit -m "chore(helm): Updating $chart image release"
}

2
scripts/helmcharts/databases/charts/clickhouse/values.yaml
View file

@ -84,4 +84,4 @@ nodeSelector: {}
tolerations: []
affinity: {}
storageSize: 100G
storageSize: 100Gi

12
scripts/helmcharts/openreplay-cli
View file

@ -190,6 +190,12 @@ function or_helm_upgrade() {
function upgrade_old() {
old_vars_path="$1"
[[ -f $old_vars_path ]] || log err "No configuration file ${BWHITE}$old_vars_path${RED}.
If you're updating from version older than ${BWHITE}v1.10.0${RED}, for example ${BWHITE}v1.9.0${RED}:
${BWHITE}openreplay --deprecated-upgrade ~/openreplay_v1.9.0/scripts/helmcharts/vars.yaml${RED}.
If you're having a custom installation,
${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED}.
"
or_version=$(busybox awk '/fromVersion/{print $2}' < "${old_vars_path}")
sudo cp "${old_vars_path}" ${OR_DIR}/vars.yaml.backup."${or_version//\"}"_"$(date +%Y%m%d-%H%M%S)" || log err "Not able to copy old vars.yaml"
sudo cp "${old_vars_path}" ${OR_DIR}/vars.yaml || log err "Not able to copy old vars.yaml"
@ -271,6 +277,12 @@ function upgrade() {
# 3. How to update package. Because openreplay -u will be done from old update script
# 4. Update from Version
exists git || log err "Git not found. Please install"
[[ -f ${OR_DIR}/vars.yaml ]] || log err "No configuration file ${BWHITE}${OR_DIR}/vars.yaml${RED}.
If you're updating from version older than ${BWHITE}v1.10.0${RED}, for example ${BWHITE}v1.9.0${RED}:
${BWHITE}openreplay --deprecated-upgrade ~/openreplay_v1.9.0/scripts/helmcharts/vars.yaml${RED}.
If you're having a custom installation,
${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED}.
"
or_version=$(busybox awk '/fromVersion/{print $2}' < "${OR_DIR}/vars.yaml") || {
log err "${BWHITE}${OR_DIR}/vars.yaml${RED} not found.
Please do ${BWHITE}openreplay --deprecated-upgrade /path/to/vars.yaml${RED}

5
scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml
View file

@ -43,10 +43,9 @@ spec:
{{- .Values.healthCheck | toYaml | nindent 10}}
{{- end}}
env:
{{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }}
- name: KAFKA_SERVERS
value: "{{ .Values.global.kafka.kafkaHost }}"
- name: REDIS_STRING
value: "{{ .Values.global.redis.redisHost }}"
- name: ch_username
value: "{{ .Values.global.clickhouse.username }}"
- name: ch_password
@ -114,6 +113,8 @@ spec:
value: '{{ .Values.global.s3.region }}'
- name: sessions_region
value: '{{ .Values.global.s3.region }}'
- name: ASSIST_RECORDS_BUCKET
value: {{ .Values.global.s3.assistRecordsBucket }}
- name: sessions_bucket
value: {{ .Values.global.s3.recordingsBucket }}
- name: sourcemaps_bucket

3
scripts/helmcharts/vars.yaml
View file

@ -50,7 +50,7 @@ kafka: &kafka
# value: "3000000"
redis: &redis
# enabled: false
enabled: true
redisHost: "redis-master.db.svc.cluster.local"
redisPort: "6379"
@ -117,6 +117,7 @@ global:
assetsBucket: "sessions-assets"
recordingsBucket: "mobs"
sourcemapsBucket: "sourcemaps"
assistRecordsBucket: "records"
vaultBucket: "vault-data"
# This is only for enterpriseEdition
quickwitBucket: "quickwit"

539
static/openreplay-git-hero.svg
View file

File diff suppressed because one or more lines are too long
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 570 KiB

After

Width:  |  Height:  |  Size: 567 KiB

148
static/replay-thumbnail.svg Normal file
View file

File diff suppressed because one or more lines are too long
Side by side

After

Width:  |  Height:  |  Size: 99 KiB