From eb065deb3ddb4e90d36bfdf923be2013119c7fa8 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 17 Jan 2022 16:20:33 +0100 Subject: [PATCH 1/7] feat(api): custom metrics return full-metric on create and update --- api/chalicelib/core/custom_metrics.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index ffd911fea..62bddfbb3 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -56,8 +56,7 @@ def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): query ) r = cur.fetchone() - r = helper.dict_to_camel_case(r) - return {"data": r} + return {"data": get(metric_id=r["metric_id"], project_id=project_id, user_id=user_id)} def __get_series_id(metric_id): @@ -135,8 +134,7 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche query ) r = cur.fetchone() - r = helper.dict_to_camel_case(r) - return r + return get(metric_id=metric_id, project_id=project_id, user_id=user_id) def get_all(project_id, user_id): From fef0af2a122ccbfd5722930cba106a072fe3bc8d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 17 Jan 2022 18:22:01 +0100 Subject: [PATCH 2/7] feat(api): funnels fixes --- api/chalicelib/core/funnels.py | 40 ++++++++++++++++++---------------- api/routers/core.py | 19 ++++++++-------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index d73204c77..cb544a62f 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -1,6 +1,7 @@ import json import chalicelib.utils.helper +import schemas from chalicelib.core import events, significance, sessions from chalicelib.utils import dev from chalicelib.utils import helper, pg_client @@ -144,28 +145,28 @@ def delete(project_id, funnel_id, user_id): def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) return sessions.search2_pg(data=f["filter"], project_id=project_id, user_id=user_id) -def get_sessions_on_the_fly(funnel_id, project_id, user_id, data): - data["events"] = filter_stages(data.get("events", [])) - if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) +def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.FunnelSearchPayloadSchema): + data.events = filter_stages(data.events) + if len(data.events) == 0: + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), - start_date=data.get('startDate', None), - end_date=data.get('endDate', None)) - data = f["filter"] - return sessions.search2_pg(data=data, project_id=project_id, user_id=user_id) + get_start_end_time(filter_d=f["filter"], range_value=data.range_value, + start_date=data.startDate, end_date=data.endDate) + data = schemas.FunnelSearchPayloadSchema.parse_obj(f["filter"]) + return sessions.search2_pg(data=data, project_id=project_id, + user_id=user_id) -def get_top_insights(project_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) +def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -175,10 +176,10 @@ def get_top_insights(project_id, funnel_id, range_value=None, start_date=None, e "totalDropDueToIssues": total_drop_due_to_issues}} -def get_top_insights_on_the_fly(funnel_id, project_id, data): +def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data): data["events"] = filter_stages(data.get("events", [])) if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), @@ -192,8 +193,8 @@ def get_top_insights_on_the_fly(funnel_id, project_id, data): "totalDropDueToIssues": total_drop_due_to_issues}} -def get_issues(project_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) +def get_issues(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -203,12 +204,12 @@ def get_issues(project_id, funnel_id, range_value=None, start_date=None, end_dat @dev.timed -def get_issues_on_the_fly(funnel_id, project_id, data): +def get_issues_on_the_fly(funnel_id, user_id, project_id, data): first_stage = data.get("firstStage") last_stage = data.get("lastStage") data["events"] = filter_stages(data.get("events", [])) if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), @@ -257,7 +258,8 @@ def search_by_issue(user_id, project_id, funnel_id, issue_id, data, range_value= data = f["filter"] # insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data, project_id=project_id) - issues = get_issues_on_the_fly(funnel_id=funnel_id, project_id=project_id, data=data).get("issues", {}) + issues = get_issues_on_the_fly(funnel_id=funnel_id, user_id=user_id, project_id=project_id, data=data) \ + .get("issues", {}) issues = issues.get("significant", []) + issues.get("insignificant", []) issue = None for i in issues: diff --git a/api/routers/core.py b/api/routers/core.py index 792e4c739..857df1fa6 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -678,32 +678,31 @@ def get_possible_issue_types(projectId: int, context: schemas.CurrentContext = D @app.get('/{projectId}/funnels/{funnelId}/insights', tags=["funnels"]) def get_funnel_insights(projectId: int, funnelId: int, rangeValue: str = None, startDate: int = None, endDate: int = None, context: schemas.CurrentContext = Depends(OR_context)): - return funnels.get_top_insights(funnel_id=funnelId, project_id=projectId, - range_value=rangeValue, - start_date=startDate, - end_date=endDate) + return funnels.get_top_insights(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + range_value=rangeValue, start_date=startDate, end_date=endDate) @app.post('/{projectId}/funnels/{funnelId}/insights', tags=["funnels"]) @app.put('/{projectId}/funnels/{funnelId}/insights', tags=["funnels"]) def get_funnel_insights_on_the_fly(projectId: int, funnelId: int, data: schemas.FunnelInsightsPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return funnels.get_top_insights_on_the_fly(funnel_id=funnelId, project_id=projectId, data=data.dict()) + return funnels.get_top_insights_on_the_fly(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + data=data.dict()) @app.get('/{projectId}/funnels/{funnelId}/issues', tags=["funnels"]) def get_funnel_issues(projectId: int, funnelId, rangeValue: str = None, startDate: int = None, endDate: int = None, context: schemas.CurrentContext = Depends(OR_context)): - return funnels.get_issues(funnel_id=funnelId, project_id=projectId, - range_value=rangeValue, - start_date=startDate, end_date=endDate) + return funnels.get_issues(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + range_value=rangeValue, start_date=startDate, end_date=endDate) @app.post('/{projectId}/funnels/{funnelId}/issues', tags=["funnels"]) @app.put('/{projectId}/funnels/{funnelId}/issues', tags=["funnels"]) def get_funnel_issues_on_the_fly(projectId: int, funnelId: int, data: schemas.FunnelSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return {"data": funnels.get_issues_on_the_fly(funnel_id=funnelId, project_id=projectId, data=data.dict())} + return {"data": funnels.get_issues_on_the_fly(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + data=data.dict())} @app.get('/{projectId}/funnels/{funnelId}/sessions', tags=["funnels"]) @@ -720,7 +719,7 @@ def get_funnel_sessions(projectId: int, funnelId: int, rangeValue: str = None, s def get_funnel_sessions_on_the_fly(projectId: int, funnelId: int, data: schemas.FunnelSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): return {"data": funnels.get_sessions_on_the_fly(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, - data=data.dict())} + data=data)} @app.get('/{projectId}/funnels/issues/{issueId}/sessions', tags=["funnels"]) From 11d94a179999ab05104e401ddaff0aec21b34a0a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 17 Jan 2022 18:43:20 +0100 Subject: [PATCH 3/7] feat(api): funnels changes --- api/chalicelib/core/funnels.py | 29 +++++++++++++++-------------- api/routers/core.py | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index cb544a62f..281aeb422 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -12,23 +12,24 @@ REMOVE_KEYS = ["key", "_key", "startDate", "endDate"] ALLOW_UPDATE_FOR = ["name", "filter"] -def filter_stages(stages): - ALLOW_TYPES = [events.event_type.CLICK.ui_type, events.event_type.INPUT.ui_type, - events.event_type.LOCATION.ui_type, events.event_type.CUSTOM.ui_type, - events.event_type.CLICK_IOS.ui_type, events.event_type.INPUT_IOS.ui_type, - events.event_type.VIEW_IOS.ui_type, events.event_type.CUSTOM_IOS.ui_type, ] - return [s for s in stages if s["type"] in ALLOW_TYPES and s.get("value") is not None] +# def filter_stages(stages): +# ALLOW_TYPES = [events.event_type.CLICK.ui_type, events.event_type.INPUT.ui_type, +# events.event_type.LOCATION.ui_type, events.event_type.CUSTOM.ui_type, +# events.event_type.CLICK_IOS.ui_type, events.event_type.INPUT_IOS.ui_type, +# events.event_type.VIEW_IOS.ui_type, events.event_type.CUSTOM_IOS.ui_type, ] +# return [s for s in stages if s["type"] in ALLOW_TYPES and s.get("value") is not None] -def create(project_id, user_id, name, filter, is_public): +def create(project_id, user_id, name, filter: schemas.FunnelSearchPayloadSchema, is_public): helper.delete_keys_from_dict(filter, REMOVE_KEYS) - filter["events"] = filter_stages(stages=filter.get("events", [])) + # filter.events = filter_stages(stages=filter.events) with pg_client.PostgresClient() as cur: query = cur.mogrify("""\ INSERT INTO public.funnels (project_id, user_id, name, filter,is_public) VALUES (%(project_id)s, %(user_id)s, %(name)s, %(filter)s::jsonb,%(is_public)s) RETURNING *;""", - {"user_id": user_id, "project_id": project_id, "name": name, "filter": json.dumps(filter), + {"user_id": user_id, "project_id": project_id, "name": name, + "filter": json.dumps(filter.dict()), "is_public": is_public}) cur.execute( @@ -94,7 +95,7 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date for row in rows: row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"]) if details: - row["filter"]["events"] = filter_stages(row["filter"]["events"]) + # row["filter"]["events"] = filter_stages(row["filter"]["events"]) get_start_end_time(filter_d=row["filter"], range_value=range_value, start_date=start_date, end_date=end_date) counts = sessions.search2_pg(data=row["filter"], project_id=project_id, user_id=None, count_only=True) @@ -153,7 +154,7 @@ def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=No def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.FunnelSearchPayloadSchema): - data.events = filter_stages(data.events) + # data.events = filter_stages(data.events) if len(data.events) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: @@ -177,7 +178,7 @@ def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_dat def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data): - data["events"] = filter_stages(data.get("events", [])) + # data["events"] = filter_stages(data.get("events", [])) if len(data["events"]) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: @@ -207,7 +208,7 @@ def get_issues(project_id, user_id, funnel_id, range_value=None, start_date=None def get_issues_on_the_fly(funnel_id, user_id, project_id, data): first_stage = data.get("firstStage") last_stage = data.get("lastStage") - data["events"] = filter_stages(data.get("events", [])) + # data["events"] = filter_stages(data.get("events", [])) if len(data["events"]) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: @@ -243,7 +244,7 @@ def get(funnel_id, project_id, user_id): return None f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) - f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) + # f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) return f diff --git a/api/routers/core.py b/api/routers/core.py index 857df1fa6..abaa60dea 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -645,7 +645,7 @@ def add_funnel(projectId: int, data: schemas.FunnelSchema = Body(...), return funnels.create(project_id=projectId, user_id=context.user_id, name=data.name, - filter=data.filter.dict(), + filter=data.filter, is_public=data.is_public) From b3a78aeca887a43c6673153b79021ed433e285c2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 18 Jan 2022 12:39:41 +0100 Subject: [PATCH 4/7] feat(api): fixed update funnels --- api/chalicelib/core/funnels.py | 5 +++-- api/chalicelib/core/significance.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 281aeb422..22ab73cba 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -2,7 +2,7 @@ import json import chalicelib.utils.helper import schemas -from chalicelib.core import events, significance, sessions +from chalicelib.core import significance, sessions from chalicelib.utils import dev from chalicelib.utils import helper, pg_client from chalicelib.utils.TimeUTC import TimeUTC @@ -61,7 +61,8 @@ def update(funnel_id, user_id, project_id, name=None, filter=None, is_public=Non AND project_id = %(project_id)s AND (user_id = %(user_id)s OR is_public) RETURNING *;""", {"user_id": user_id, "funnel_id": funnel_id, "name": name, - "filter": json.dumps(filter) if filter is not None else None, "is_public": is_public}) + "filter": json.dumps(filter) if filter is not None else None, "is_public": is_public, + "project_id": project_id}) # print("--------------------") # print(query) # print("--------------------") diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index 9fc55065f..d81378ddb 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -223,9 +223,9 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: params = {"project_id": project_id, "startTimestamp": filter_d["startDate"], "endTimestamp": filter_d["endDate"], "issueTypes": tuple(filter_issues), **values} with pg_client.PostgresClient() as cur: - print("---------------------------------------------------") - print(cur.mogrify(n_stages_query, params)) - print("---------------------------------------------------") + # print("---------------------------------------------------") + # print(cur.mogrify(n_stages_query, params)) + # print("---------------------------------------------------") cur.execute(cur.mogrify(n_stages_query, params)) rows = cur.fetchall() return rows From be2b1708b719936db648d52a56bb328b42b9fe27 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 18 Jan 2022 14:08:58 +0100 Subject: [PATCH 5/7] feat(api): fixed get funnel-sessions-issues --- api/chalicelib/core/funnels.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 22ab73cba..835a655f4 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -250,17 +250,19 @@ def get(funnel_id, project_id, user_id): @dev.timed -def search_by_issue(user_id, project_id, funnel_id, issue_id, data, range_value=None, start_date=None, end_date=None): - if len(data.get("events", [])) == 0: +def search_by_issue(user_id, project_id, funnel_id, issue_id, data: schemas.FunnelSearchPayloadSchema, range_value=None, + start_date=None, end_date=None): + if len(data.events) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=data.get('startDate', start_date), - end_date=data.get('endDate', end_date)) - data = f["filter"] + data.startDate = data.startDate if data.startDate is not None else start_date + data.endDate = data.endDate if data.endDate is not None else end_date + get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=data.startDate, + end_date=data.endDate) + data = schemas.FunnelSearchPayloadSchema.parse_obj(f["filter"]) - # insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data, project_id=project_id) - issues = get_issues_on_the_fly(funnel_id=funnel_id, user_id=user_id, project_id=project_id, data=data) \ + issues = get_issues_on_the_fly(funnel_id=funnel_id, user_id=user_id, project_id=project_id, data=data.dict()) \ .get("issues", {}) issues = issues.get("significant", []) + issues.get("insignificant", []) issue = None From e742955fd01e75ef2d73a78b6bd211fb490fc05d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 18 Jan 2022 14:19:16 +0100 Subject: [PATCH 6/7] feat(api): funnel changes feat(api): reduce DB pool initial size --- api/.env.default | 2 +- api/routers/core.py | 2 +- ee/api/.env.default | 12 +++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/api/.env.default b/api/.env.default index 3b05ce3d2..8f4ddc01c 100644 --- a/api/.env.default +++ b/api/.env.default @@ -35,7 +35,7 @@ pg_password=asayerPostgres pg_port=5432 pg_user=postgres pg_timeout=30 -pg_minconn=50 +pg_minconn=45 put_S3_TTL=20 sentryURL= sessions_bucket=mobs diff --git a/api/routers/core.py b/api/routers/core.py index abaa60dea..9c5bec86e 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -739,7 +739,7 @@ def get_funnel_issue_sessions(projectId: int, funnelId: int, issueId: str, data: schemas.FunnelSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): data = funnels.search_by_issue(project_id=projectId, user_id=context.user_id, issue_id=issueId, - funnel_id=funnelId, data=data.dict()) + funnel_id=funnelId, data=data) if "errors" in data: return data if data.get("issue") is None: diff --git a/ee/api/.env.default b/ee/api/.env.default index 6fff1793c..cec7e59a4 100644 --- a/ee/api/.env.default +++ b/ee/api/.env.default @@ -38,11 +38,13 @@ jwt_exp_delta_seconds=2592000 jwt_issuer=openreplay-default-ee jwt_secret="SET A RANDOM STRING HERE" peers=http://utilities-openreplay.app.svc.cluster.local:9000/assist/%s/peers -pg_dbname=app -pg_host=127.0.0.1 -pg_password= -pg_port=9202 -pg_user= +pg_dbname=postgres +pg_host=postgresql.db.svc.cluster.local +pg_password=asayerPostgres +pg_port=5432 +pg_user=postgres +pg_timeout=30 +pg_minconn=45 put_S3_TTL=20 sentryURL= sessions_bucket=mobs From 8037879e4347dcc4650dba1ddeae031ac355512f Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 19 Jan 2022 00:10:31 +0530 Subject: [PATCH 7/7] fix(install): kafka ssl topic creation Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/openreplay/files/kafka.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/files/kafka.sh b/scripts/helmcharts/openreplay/files/kafka.sh index 9105f3b00..1c811eb5d 100644 --- a/scripts/helmcharts/openreplay/files/kafka.sh +++ b/scripts/helmcharts/openreplay/files/kafka.sh @@ -13,7 +13,7 @@ topics=( touch /tmp/config.txt if [[ $KAFKA_SSL == "true" ]]; then - echo 'securty.protocol=SSL' > /tmp/config.txt + echo 'security.protocol=SSL' > /tmp/config.txt fi function init() {