diff --git a/api/.env.default b/api/.env.default index 3b05ce3d2..8f4ddc01c 100644 --- a/api/.env.default +++ b/api/.env.default @@ -35,7 +35,7 @@ pg_password=asayerPostgres pg_port=5432 pg_user=postgres pg_timeout=30 -pg_minconn=50 +pg_minconn=45 put_S3_TTL=20 sentryURL= sessions_bucket=mobs diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index ffd911fea..62bddfbb3 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -56,8 +56,7 @@ def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): query ) r = cur.fetchone() - r = helper.dict_to_camel_case(r) - return {"data": r} + return {"data": get(metric_id=r["metric_id"], project_id=project_id, user_id=user_id)} def __get_series_id(metric_id): @@ -135,8 +134,7 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche query ) r = cur.fetchone() - r = helper.dict_to_camel_case(r) - return r + return get(metric_id=metric_id, project_id=project_id, user_id=user_id) def get_all(project_id, user_id): diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index d73204c77..835a655f4 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -1,7 +1,8 @@ import json import chalicelib.utils.helper -from chalicelib.core import events, significance, sessions +import schemas +from chalicelib.core import significance, sessions from chalicelib.utils import dev from chalicelib.utils import helper, pg_client from chalicelib.utils.TimeUTC import TimeUTC @@ -11,23 +12,24 @@ REMOVE_KEYS = ["key", "_key", "startDate", "endDate"] ALLOW_UPDATE_FOR = ["name", "filter"] -def filter_stages(stages): - ALLOW_TYPES = [events.event_type.CLICK.ui_type, events.event_type.INPUT.ui_type, - events.event_type.LOCATION.ui_type, events.event_type.CUSTOM.ui_type, - events.event_type.CLICK_IOS.ui_type, events.event_type.INPUT_IOS.ui_type, - events.event_type.VIEW_IOS.ui_type, events.event_type.CUSTOM_IOS.ui_type, ] - return [s for s in stages if s["type"] in ALLOW_TYPES and s.get("value") is not None] +# def filter_stages(stages): +# ALLOW_TYPES = [events.event_type.CLICK.ui_type, events.event_type.INPUT.ui_type, +# events.event_type.LOCATION.ui_type, events.event_type.CUSTOM.ui_type, +# events.event_type.CLICK_IOS.ui_type, events.event_type.INPUT_IOS.ui_type, +# events.event_type.VIEW_IOS.ui_type, events.event_type.CUSTOM_IOS.ui_type, ] +# return [s for s in stages if s["type"] in ALLOW_TYPES and s.get("value") is not None] -def create(project_id, user_id, name, filter, is_public): +def create(project_id, user_id, name, filter: schemas.FunnelSearchPayloadSchema, is_public): helper.delete_keys_from_dict(filter, REMOVE_KEYS) - filter["events"] = filter_stages(stages=filter.get("events", [])) + # filter.events = filter_stages(stages=filter.events) with pg_client.PostgresClient() as cur: query = cur.mogrify("""\ INSERT INTO public.funnels (project_id, user_id, name, filter,is_public) VALUES (%(project_id)s, %(user_id)s, %(name)s, %(filter)s::jsonb,%(is_public)s) RETURNING *;""", - {"user_id": user_id, "project_id": project_id, "name": name, "filter": json.dumps(filter), + {"user_id": user_id, "project_id": project_id, "name": name, + "filter": json.dumps(filter.dict()), "is_public": is_public}) cur.execute( @@ -59,7 +61,8 @@ def update(funnel_id, user_id, project_id, name=None, filter=None, is_public=Non AND project_id = %(project_id)s AND (user_id = %(user_id)s OR is_public) RETURNING *;""", {"user_id": user_id, "funnel_id": funnel_id, "name": name, - "filter": json.dumps(filter) if filter is not None else None, "is_public": is_public}) + "filter": json.dumps(filter) if filter is not None else None, "is_public": is_public, + "project_id": project_id}) # print("--------------------") # print(query) # print("--------------------") @@ -93,7 +96,7 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date for row in rows: row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"]) if details: - row["filter"]["events"] = filter_stages(row["filter"]["events"]) + # row["filter"]["events"] = filter_stages(row["filter"]["events"]) get_start_end_time(filter_d=row["filter"], range_value=range_value, start_date=start_date, end_date=end_date) counts = sessions.search2_pg(data=row["filter"], project_id=project_id, user_id=None, count_only=True) @@ -144,28 +147,28 @@ def delete(project_id, funnel_id, user_id): def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) return sessions.search2_pg(data=f["filter"], project_id=project_id, user_id=user_id) -def get_sessions_on_the_fly(funnel_id, project_id, user_id, data): - data["events"] = filter_stages(data.get("events", [])) - if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) +def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.FunnelSearchPayloadSchema): + # data.events = filter_stages(data.events) + if len(data.events) == 0: + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), - start_date=data.get('startDate', None), - end_date=data.get('endDate', None)) - data = f["filter"] - return sessions.search2_pg(data=data, project_id=project_id, user_id=user_id) + get_start_end_time(filter_d=f["filter"], range_value=data.range_value, + start_date=data.startDate, end_date=data.endDate) + data = schemas.FunnelSearchPayloadSchema.parse_obj(f["filter"]) + return sessions.search2_pg(data=data, project_id=project_id, + user_id=user_id) -def get_top_insights(project_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) +def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -175,10 +178,10 @@ def get_top_insights(project_id, funnel_id, range_value=None, start_date=None, e "totalDropDueToIssues": total_drop_due_to_issues}} -def get_top_insights_on_the_fly(funnel_id, project_id, data): - data["events"] = filter_stages(data.get("events", [])) +def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data): + # data["events"] = filter_stages(data.get("events", [])) if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), @@ -192,8 +195,8 @@ def get_top_insights_on_the_fly(funnel_id, project_id, data): "totalDropDueToIssues": total_drop_due_to_issues}} -def get_issues(project_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) +def get_issues(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -203,12 +206,12 @@ def get_issues(project_id, funnel_id, range_value=None, start_date=None, end_dat @dev.timed -def get_issues_on_the_fly(funnel_id, project_id, data): +def get_issues_on_the_fly(funnel_id, user_id, project_id, data): first_stage = data.get("firstStage") last_stage = data.get("lastStage") - data["events"] = filter_stages(data.get("events", [])) + # data["events"] = filter_stages(data.get("events", [])) if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), @@ -242,22 +245,25 @@ def get(funnel_id, project_id, user_id): return None f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) - f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) + # f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) return f @dev.timed -def search_by_issue(user_id, project_id, funnel_id, issue_id, data, range_value=None, start_date=None, end_date=None): - if len(data.get("events", [])) == 0: +def search_by_issue(user_id, project_id, funnel_id, issue_id, data: schemas.FunnelSearchPayloadSchema, range_value=None, + start_date=None, end_date=None): + if len(data.events) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=data.get('startDate', start_date), - end_date=data.get('endDate', end_date)) - data = f["filter"] + data.startDate = data.startDate if data.startDate is not None else start_date + data.endDate = data.endDate if data.endDate is not None else end_date + get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=data.startDate, + end_date=data.endDate) + data = schemas.FunnelSearchPayloadSchema.parse_obj(f["filter"]) - # insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data, project_id=project_id) - issues = get_issues_on_the_fly(funnel_id=funnel_id, project_id=project_id, data=data).get("issues", {}) + issues = get_issues_on_the_fly(funnel_id=funnel_id, user_id=user_id, project_id=project_id, data=data.dict()) \ + .get("issues", {}) issues = issues.get("significant", []) + issues.get("insignificant", []) issue = None for i in issues: diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index 9fc55065f..d81378ddb 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -223,9 +223,9 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: params = {"project_id": project_id, "startTimestamp": filter_d["startDate"], "endTimestamp": filter_d["endDate"], "issueTypes": tuple(filter_issues), **values} with pg_client.PostgresClient() as cur: - print("---------------------------------------------------") - print(cur.mogrify(n_stages_query, params)) - print("---------------------------------------------------") + # print("---------------------------------------------------") + # print(cur.mogrify(n_stages_query, params)) + # print("---------------------------------------------------") cur.execute(cur.mogrify(n_stages_query, params)) rows = cur.fetchall() return rows diff --git a/api/routers/core.py b/api/routers/core.py index 792e4c739..9c5bec86e 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -645,7 +645,7 @@ def add_funnel(projectId: int, data: schemas.FunnelSchema = Body(...), return funnels.create(project_id=projectId, user_id=context.user_id, name=data.name, - filter=data.filter.dict(), + filter=data.filter, is_public=data.is_public) @@ -678,32 +678,31 @@ def get_possible_issue_types(projectId: int, context: schemas.CurrentContext = D @app.get('/{projectId}/funnels/{funnelId}/insights', tags=["funnels"]) def get_funnel_insights(projectId: int, funnelId: int, rangeValue: str = None, startDate: int = None, endDate: int = None, context: schemas.CurrentContext = Depends(OR_context)): - return funnels.get_top_insights(funnel_id=funnelId, project_id=projectId, - range_value=rangeValue, - start_date=startDate, - end_date=endDate) + return funnels.get_top_insights(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + range_value=rangeValue, start_date=startDate, end_date=endDate) @app.post('/{projectId}/funnels/{funnelId}/insights', tags=["funnels"]) @app.put('/{projectId}/funnels/{funnelId}/insights', tags=["funnels"]) def get_funnel_insights_on_the_fly(projectId: int, funnelId: int, data: schemas.FunnelInsightsPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return funnels.get_top_insights_on_the_fly(funnel_id=funnelId, project_id=projectId, data=data.dict()) + return funnels.get_top_insights_on_the_fly(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + data=data.dict()) @app.get('/{projectId}/funnels/{funnelId}/issues', tags=["funnels"]) def get_funnel_issues(projectId: int, funnelId, rangeValue: str = None, startDate: int = None, endDate: int = None, context: schemas.CurrentContext = Depends(OR_context)): - return funnels.get_issues(funnel_id=funnelId, project_id=projectId, - range_value=rangeValue, - start_date=startDate, end_date=endDate) + return funnels.get_issues(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + range_value=rangeValue, start_date=startDate, end_date=endDate) @app.post('/{projectId}/funnels/{funnelId}/issues', tags=["funnels"]) @app.put('/{projectId}/funnels/{funnelId}/issues', tags=["funnels"]) def get_funnel_issues_on_the_fly(projectId: int, funnelId: int, data: schemas.FunnelSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return {"data": funnels.get_issues_on_the_fly(funnel_id=funnelId, project_id=projectId, data=data.dict())} + return {"data": funnels.get_issues_on_the_fly(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + data=data.dict())} @app.get('/{projectId}/funnels/{funnelId}/sessions', tags=["funnels"]) @@ -720,7 +719,7 @@ def get_funnel_sessions(projectId: int, funnelId: int, rangeValue: str = None, s def get_funnel_sessions_on_the_fly(projectId: int, funnelId: int, data: schemas.FunnelSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): return {"data": funnels.get_sessions_on_the_fly(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, - data=data.dict())} + data=data)} @app.get('/{projectId}/funnels/issues/{issueId}/sessions', tags=["funnels"]) @@ -740,7 +739,7 @@ def get_funnel_issue_sessions(projectId: int, funnelId: int, issueId: str, data: schemas.FunnelSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): data = funnels.search_by_issue(project_id=projectId, user_id=context.user_id, issue_id=issueId, - funnel_id=funnelId, data=data.dict()) + funnel_id=funnelId, data=data) if "errors" in data: return data if data.get("issue") is None: diff --git a/ee/api/.env.default b/ee/api/.env.default index 6fff1793c..cec7e59a4 100644 --- a/ee/api/.env.default +++ b/ee/api/.env.default @@ -38,11 +38,13 @@ jwt_exp_delta_seconds=2592000 jwt_issuer=openreplay-default-ee jwt_secret="SET A RANDOM STRING HERE" peers=http://utilities-openreplay.app.svc.cluster.local:9000/assist/%s/peers -pg_dbname=app -pg_host=127.0.0.1 -pg_password= -pg_port=9202 -pg_user= +pg_dbname=postgres +pg_host=postgresql.db.svc.cluster.local +pg_password=asayerPostgres +pg_port=5432 +pg_user=postgres +pg_timeout=30 +pg_minconn=45 put_S3_TTL=20 sentryURL= sessions_bucket=mobs