From 54f5bb6ede35eaced2bc43c6fc50d9c4f0f72ec3 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 15 Nov 2022 18:03:57 +0100 Subject: [PATCH] feat(chalice): search recordings feat(chalice): get record feat(minio): new bucket --- api/chalicelib/core/sessions_mobs.py | 9 ++- ee/api/chalicelib/core/assist_records.py | 56 ++++++++++++++++++- ee/api/env.default | 2 +- ee/api/routers/ee.py | 11 ++++ ee/api/schemas_ee.py | 12 ++++ .../db/init_dbs/postgresql/1.9.5/1.9.5.sql | 2 +- .../db/init_dbs/postgresql/init_schema.sql | 2 +- .../charts/http/templates/ingress.yaml | 2 +- scripts/helmcharts/openreplay/files/minio.sh | 2 +- 9 files changed, 87 insertions(+), 11 deletions(-) diff --git a/api/chalicelib/core/sessions_mobs.py b/api/chalicelib/core/sessions_mobs.py index 3d966a47c..ce3a57573 100644 --- a/api/chalicelib/core/sessions_mobs.py +++ b/api/chalicelib/core/sessions_mobs.py @@ -1,7 +1,6 @@ from decouple import config from chalicelib.utils import s3 -from chalicelib.utils.s3 import client def __get_mob_keys(project_id, session_id): @@ -18,7 +17,7 @@ def __get_mob_keys(project_id, session_id): def get_urls(project_id, session_id): results = [] for k in __get_mob_keys(project_id=project_id, session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) @@ -28,7 +27,7 @@ def get_urls(project_id, session_id): def get_urls_depercated(session_id): return [ - client.generate_presigned_url( + s3.client.generate_presigned_url( 'get_object', Params={ 'Bucket': config("sessions_bucket"), @@ -36,7 +35,7 @@ def get_urls_depercated(session_id): }, ExpiresIn=100000 ), - client.generate_presigned_url( + s3.client.generate_presigned_url( 'get_object', Params={ 'Bucket': config("sessions_bucket"), @@ -47,7 +46,7 @@ def get_urls_depercated(session_id): def get_ios(session_id): - return client.generate_presigned_url( + return s3.client.generate_presigned_url( 'get_object', Params={ 'Bucket': config("ios_bucket"), diff --git a/ee/api/chalicelib/core/assist_records.py b/ee/api/chalicelib/core/assist_records.py index 7683f5645..18c166445 100644 --- a/ee/api/chalicelib/core/assist_records.py +++ b/ee/api/chalicelib/core/assist_records.py @@ -1,7 +1,8 @@ from decouple import config +import schemas import schemas_ee -from chalicelib.utils import s3, pg_client +from chalicelib.utils import s3, pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC @@ -19,3 +20,56 @@ def presign_records(project_id, data: schemas_ee.AssistRecordPayloadSchema, cont params) cur.execute(query) return presigned_url + + +def search_records(project_id, data: schemas_ee.AssistRecordSearchPayloadSchema, context: schemas_ee.CurrentContext): + conditions = ["projects.tenant_id=%(tenant_id)s", + "projects.deleted_at ISNULL", + "assist_records.created_at>=%(startDate)s", + "assist_records.created_at<=%(endDate)s"] + params = {"tenant_id": context.tenant_id, "project_id": project_id, + "startDate": data.startDate, "endDate": data.endDate, + "p_start": (data.page - 1) * data.limit, "p_end": data.page * data.limit, + **data.dict()} + if data.user_id is not None: + conditions.append("assist_records.user_id=%(user_id)s") + if data.query is not None and len(data.query) > 0: + conditions.append("(users.name ILIKE %(query)s OR assist_records.name ILIKE %(query)s)") + params["query"] = helper.values_for_operator(value=data.query, + op=schemas.SearchEventOperator._contains) + with pg_client.PostgresClient() as cur: + query = cur.mogrify(f"""SELECT record_id, user_id, session_id, assist_records.created_at, + assist_records.name, duration, users.name AS created_by + FROM assist_records + INNER JOIN projects USING (project_id) + LEFT JOIN users USING (user_id) + WHERE {" AND ".join(conditions)};""", + params) + cur.execute(query) + results = helper.list_to_camel_case(cur.fetchall()) + return results + + +def get_record(project_id, record_id, context: schemas_ee.CurrentContext): + conditions = ["projects.tenant_id=%(tenant_id)s", + "projects.deleted_at ISNULL", + "assist_records.record_id=%(record_id)s"] + params = {"tenant_id": context.tenant_id, "project_id": project_id, "record_id": record_id} + with pg_client.PostgresClient() as cur: + query = cur.mogrify(f"""SELECT record_id, user_id, session_id, assist_records.created_at, + assist_records.name, duration, users.name AS created_by, + file_key + FROM assist_records + INNER JOIN projects USING (project_id) + LEFT JOIN users USING (user_id) + WHERE {" AND ".join(conditions)} + LIMIT 1;""", params) + cur.execute(query) + result = helper.dict_to_camel_case(cur.fetchone()) + if result: + result["URL"] = s3.client.generate_presigned_url( + 'get_object', + Params={'Bucket': config("ASSIST_RECORDS_BUCKET"), 'Key': result.pop("fileKey")}, + ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) + ) + return result diff --git a/ee/api/env.default b/ee/api/env.default index f980c9763..0aab758ce 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -45,7 +45,7 @@ PG_MAXCONN=50 PG_RETRY_MAX=50 PG_RETRY_INTERVAL=2 PG_POOL=true -ASSIST_RECORDS_BUCKET=mobs +ASSIST_RECORDS_BUCKET=records sessions_bucket=mobs sessions_region=us-east-1 sourcemaps_bucket=sourcemaps diff --git a/ee/api/routers/ee.py b/ee/api/routers/ee.py index e04795ae0..2d4ec8750 100644 --- a/ee/api/routers/ee.py +++ b/ee/api/routers/ee.py @@ -81,3 +81,14 @@ def sign_record_for_upload(projectId: int, data: schemas_ee.AssistRecordPayloadS if not sessions.session_exists(project_id=projectId, session_id=data.session_id): return {"errors": ["Session not found"]} return {"data": {"URL": assist_records.presign_records(project_id=projectId, data=data, context=context)}} + + +@app.post('/{projectId}/assist/records', tags=["assist"]) +def search_records(projectId: int, data: schemas_ee.AssistRecordSearchPayloadSchema = Body(...), + context: schemas_ee.CurrentContext = Depends(OR_context)): + return {"data": assist_records.search_records(project_id=projectId, data=data, context=context)} + + +@app.get('/{projectId}/assist/records/{recordId}', tags=["assist"]) +def search_records(projectId: int, recordId: int, context: schemas_ee.CurrentContext = Depends(OR_context)): + return {"data": assist_records.get_record(project_id=projectId, record_id=recordId, context=context)} diff --git a/ee/api/schemas_ee.py b/ee/api/schemas_ee.py index 193fdc2a5..7f105c8ef 100644 --- a/ee/api/schemas_ee.py +++ b/ee/api/schemas_ee.py @@ -90,3 +90,15 @@ class AssistRecordPayloadSchema(BaseModel): class Config: alias_generator = schemas.attribute_to_camel_case + + +class AssistRecordSearchPayloadSchema(schemas._PaginatedSchema): + limit: int = Field(default=200, gt=0) + startDate: int = Field(default=TimeUTC.now(-7)) + endDate: int = Field(default=TimeUTC.now(1)) + user_id: Optional[int] = Field(default=None) + query: Optional[str] = Field(default=None) + order: Literal["asc", "desc"] = Field(default="desc") + + class Config: + alias_generator = schemas.attribute_to_camel_case diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql index 3d9a9e4d0..df6fdba52 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS assist_records project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE SET NULL, - created_at timestamp without time zone NOT NULL default (now() at time zone 'utc'), + created_at bigint NOT NULL DEFAULT (EXTRACT(EPOCH FROM now() at time zone 'utc') * 1000)::bigint, deleted_at timestamp without time zone NULL DEFAULT NULL, name text NOT NULL, file_key text NOT NULL, diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index c473e8fe5..ae247b832 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -1252,7 +1252,7 @@ $$ project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE SET NULL, - created_at timestamp without time zone NOT NULL default (now() at time zone 'utc'), + created_at bigint NOT NULL DEFAULT (EXTRACT(EPOCH FROM now() at time zone 'utc') * 1000)::bigint, deleted_at timestamp without time zone NULL DEFAULT NULL, name text NOT NULL, file_key text NOT NULL, diff --git a/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml index 2097fb1cd..bb16e4713 100644 --- a/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml @@ -58,7 +58,7 @@ spec: name: minio port: number: 9000 - path: /(minio|mobs|sessions-assets|frontend|static|sourcemaps|ios-images)/ + path: /(minio|mobs|sessions-assets|frontend|static|sourcemaps|ios-images|records)/ tls: - hosts: - {{ .Values.global.domainName }} diff --git a/scripts/helmcharts/openreplay/files/minio.sh b/scripts/helmcharts/openreplay/files/minio.sh index 0e7b4c506..18ca3b105 100644 --- a/scripts/helmcharts/openreplay/files/minio.sh +++ b/scripts/helmcharts/openreplay/files/minio.sh @@ -5,7 +5,7 @@ set -e cd /tmp -buckets=("mobs" "sessions-assets" "static" "sourcemaps" "sessions-mobile-assets" "quickwit" "vault-data") +buckets=("mobs" "sessions-assets" "static" "sourcemaps" "sessions-mobile-assets" "quickwit" "vault-data" "records") mc alias set minio http://minio.db.svc.cluster.local:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY