Api v1.15.0 (#1510)

* feat(chalice): upgraded dependencies

* feat(chalice): changed path analysis schema

* feat(DB): click coordinate support

* feat(chalice): changed path analysis issues schema
feat(chalice): upgraded dependencies

* fix(chalice): fixed pydantic issue

* refactor(chalice): refresh token validator

* feat(chalice): role restrictions

* feat(chalice): EE path analysis changes

* refactor(DB): changed creation queries
refactor(DB): changed delte queries
feat(DB): support new path analysis payload

* feat(chalice): save path analysis card

* feat(chalice): restrict access

* feat(chalice): restrict access

* feat(chalice): EE save new path analysis card

* refactor(chalice): path analysis

* feat(chalice): path analysis new query

* fix(chalice): configurable CH config

* fix(chalice): assist autocomplete

* refactor(chalice): refactored permissions

* refactor(chalice): changed log level

* refactor(chalice): upgraded dependencies

* refactor(chalice): changed path analysis query

* refactor(chalice): changed path analysis query

* refactor(chalice): upgraded dependencies
refactor(alerts): upgraded dependencies
refactor(crons): upgraded dependencies

* feat(chalice): path analysis ignore start point

* feat(chalice): path analysis in progress

* refactor(chalice): path analysis changed link sort

* refactor(chalice): path analysis changed link sort

* refactor(chalice): path analysis changed link sort

* refactor(chalice): path analysis new query
refactor(chalice): authorizers

* refactor(chalice): refactored authorizer
This commit is contained in:
Kraiem Taha Yassine 2023-10-10 15:10:11 +02:00 committed by GitHub
parent 15b55c837b
commit d7909f5c8b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 165 additions and 211 deletions

View file

@ -1,4 +1,5 @@
import datetime
import logging
from typing import Optional
from fastapi import Request
@ -9,11 +10,13 @@ from starlette.exceptions import HTTPException
import schemas
from chalicelib.core import authorizers, users
logger = logging.getLogger(__name__)
def _get_current_auth_context(request: Request, jwt_payload: dict) -> schemas.CurrentContext:
user = users.get(user_id=jwt_payload.get("userId", -1), tenant_id=jwt_payload.get("tenantId", -1))
if user is None:
print("JWTAuth: User not found.")
logger.warning("User not found.")
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="User not found.")
request.state.authorizer_identity = "jwt"
request.state.currentContext = schemas.CurrentContext(tenantId=jwt_payload.get("tenantId", -1),
@ -66,17 +69,17 @@ class JWTAuth(HTTPBearer):
or jwt_payload.get("iat") is None or jwt_payload.get("aud") is None \
or not auth_exists:
if jwt_payload is not None:
print(jwt_payload)
logger.debug(jwt_payload)
if jwt_payload.get("iat") is None:
print("JWTAuth: iat is None")
logger.debug("iat is None")
if jwt_payload.get("aud") is None:
print("JWTAuth: aud is None")
logger.debug("aud is None")
if not auth_exists:
print("JWTAuth: not users.auth_exists")
logger.warning("not users.auth_exists")
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid token or expired token.")
return _get_current_auth_context(request=request, jwt_payload=jwt_payload)
print("JWTAuth: Invalid authorization code.")
logger.warning("Invalid authorization code.")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid authorization code.")

View file

@ -1,9 +1,14 @@
import logging
import jwt
from chalicelib.utils import helper
from chalicelib.utils.TimeUTC import TimeUTC
from decouple import config
from chalicelib.core import tenants
from chalicelib.core import users
from chalicelib.utils import helper
from chalicelib.utils.TimeUTC import TimeUTC
logger = logging.getLogger(__name__)
def jwt_authorizer(scheme: str, token: str, leeway=0):
@ -18,11 +23,11 @@ def jwt_authorizer(scheme: str, token: str, leeway=0):
leeway=leeway
)
except jwt.ExpiredSignatureError:
print("! JWT Expired signature")
logger.debug("! JWT Expired signature")
return None
except BaseException as e:
print("! JWT Base Exception")
print(e)
logger.warning("! JWT Base Exception")
logger.debug(e)
return None
return payload
@ -38,11 +43,11 @@ def jwt_refresh_authorizer(scheme: str, token: str):
audience=[f"front:{helper.get_stage_name()}"]
)
except jwt.ExpiredSignatureError:
print("! JWT-refresh Expired signature")
logger.debug("! JWT-refresh Expired signature")
return None
except BaseException as e:
print("! JWT-refresh Base Exception")
print(e)
logger.warning("! JWT-refresh Base Exception")
logger.debug(e)
return None
return payload

View file

@ -291,7 +291,7 @@ def __get_funnel_issues(project_id: int, user_id: int, data: schemas.CardFunnel)
def __get_path_analysis_issues(project_id: int, user_id: int, data: schemas.CardPathAnalysis):
if len(data.series) == 0:
return {"data": []}
return {"data": {}}
card_table = schemas.CardTable(
startTimestamp=data.startTimestamp,
endTimestamp=data.endTimestamp,
@ -308,12 +308,12 @@ def __get_path_analysis_issues(project_id: int, user_id: int, data: schemas.Card
card_table.series[0].filter.filters.insert(0, schemas.SessionSearchEventSchema2(type=s.type,
operator=s.operator,
value=s.value))
for s in data.exclude:
for s in data.excludes:
card_table.series[0].filter.filters.append(schemas.SessionSearchEventSchema2(type=s.type,
operator=schemas.SearchEventOperator._not_on,
value=s.value))
return __get_table_of_issues(project_id=project_id, user_id=user_id, data=card_table)
result = __get_table_of_issues(project_id=project_id, user_id=user_id, data=card_table)
return result[0] if len(result) > 0 else {}
def get_issues(project_id: int, user_id: int, data: schemas.CardSchema):
@ -335,7 +335,7 @@ def get_issues(project_id: int, user_id: int, data: schemas.CardSchema):
def __get_path_analysis_card_info(data: schemas.CardPathAnalysis):
r = {"start_point": [s.model_dump() for s in data.start_point],
"start_type": data.start_type,
"exclude": [e.model_dump() for e in data.exclude]}
"exclude": [e.model_dump() for e in data.excludes]}
print(r)
return r

View file

@ -36,7 +36,36 @@ def __transform_journey2(rows, reverse_path=False):
links.append(link)
return {"nodes": nodes_values,
"links": sorted(links, key=lambda x: x["value"], reverse=True)}
"links": sorted(links, key=lambda x: (x["source"], x["target"]), reverse=False)}
def __transform_journey3(rows, reverse_path=False):
# nodes should contain duplicates for different steps otherwise the UI crashes
nodes = []
nodes_values = []
links = []
for r in rows:
source = f"{r['event_number_in_session']}_{r['event_type']}_{r['e_value']}"
if source not in nodes:
nodes.append(source)
nodes_values.append({"name": r['e_value'], "eventType": r['event_type']})
if r['next_value']:
target = f"{r['event_number_in_session'] + 1}_{r['next_type']}_{r['next_value']}"
if target not in nodes:
nodes.append(target)
nodes_values.append({"name": r['next_value'], "eventType": r['next_type']})
link = {"eventType": r['event_type'], "value": r["sessions_count"],
"avgTimeFromPervious": r["avg_time_from_previous"]}
if not reverse_path:
link["source"] = nodes.index(source)
link["target"] = nodes.index(target)
else:
link["source"] = nodes.index(target)
link["target"] = nodes.index(source)
links.append(link)
return {"nodes": nodes_values,
"links": sorted(links, key=lambda x: (x["source"], x["target"]), reverse=False)}
JOURNEY_TYPES = {
@ -48,7 +77,7 @@ JOURNEY_TYPES = {
# query: Q2, the result is correct
def path_analysis(project_id: int, data: schemas.CardPathAnalysis):
def path_analysis_deprecated(project_id: int, data: schemas.CardPathAnalysis):
sub_events = []
start_points_join = ""
start_points_conditions = []
@ -79,7 +108,7 @@ def path_analysis(project_id: int, data: schemas.CardPathAnalysis):
+ ")")
exclusions = {}
for i, ef in enumerate(data.exclude):
for i, ef in enumerate(data.excludes):
if ef.type in data.metric_value:
f_k = f"exclude_{i}"
extra_values = {**extra_values, **sh.multi_values(ef.value, value_key=f_k)}
@ -359,7 +388,7 @@ WITH sub_sessions AS ( SELECT session_id
_now = time()
cur.execute(query)
if time() - _now > 0:
if time() - _now > 2:
print(f">>>>>>>>>PathAnalysis long query ({int(time() - _now)}s)<<<<<<<<<")
print("----------------------")
print(query)
@ -369,10 +398,12 @@ WITH sub_sessions AS ( SELECT session_id
return __transform_journey2(rows=rows, reverse_path=reverse)
# the query generated by this function is retuning a wrong result
def path_analysis_deprecated(project_id: int, data: schemas.CardPathAnalysis):
# query: Q3, the result is correct,
# startPoints are computed before ranked_events to reduce the number of window functions over rows
# replaced time_to_target by time_from_previous
def path_analysis(project_id: int, data: schemas.CardPathAnalysis):
sub_events = []
start_points_join = ""
start_points_from = "pre_ranked_events"
start_points_conditions = []
sessions_conditions = ["start_ts>=%(startTimestamp)s", "start_ts<%(endTimestamp)s",
"project_id=%(project_id)s", "events_count > 1", "duration>0"]
@ -401,7 +432,7 @@ def path_analysis_deprecated(project_id: int, data: schemas.CardPathAnalysis):
+ ")")
exclusions = {}
for i, ef in enumerate(data.exclude):
for i, ef in enumerate(data.excludes):
if ef.type in data.metric_value:
f_k = f"exclude_{i}"
extra_values = {**extra_values, **sh.multi_values(ef.value, value_key=f_k)}
@ -418,6 +449,9 @@ def path_analysis_deprecated(project_id: int, data: schemas.CardPathAnalysis):
f_k = f"f_value_{i}"
extra_values = {**extra_values, **sh.multi_values(f.value, value_key=f_k)}
if not is_any and len(f.value) == 0:
continue
# ---- meta-filters
if f.type == schemas.FilterType.user_browser:
if is_any:
@ -587,85 +621,109 @@ def path_analysis_deprecated(project_id: int, data: schemas.CardPathAnalysis):
path_direction = ""
if len(start_points_conditions) == 0:
start_points_join = """INNER JOIN
(SELECT event_type, e_value
FROM ranked_events
WHERE event_number_in_session = 1
GROUP BY event_type, e_value
ORDER BY count(1) DESC
LIMIT 2
) AS top_start_events USING (event_type, e_value)"""
start_points_from = """(SELECT event_type, e_value
FROM pre_ranked_events
WHERE event_number_in_session = 1
GROUP BY event_type, e_value
ORDER BY count(1) DESC
LIMIT 1) AS top_start_events
INNER JOIN pre_ranked_events
USING (event_type, e_value)"""
else:
start_points_conditions = ["(" + " OR ".join(start_points_conditions) + ")"]
start_points_conditions.append("event_number_in_session = 1")
start_points_conditions.append("next_value IS NOT NULL")
steps_query = ["""n1 AS (SELECT event_number_in_session,
event_type,
e_value,
next_type,
next_value,
time_from_previous,
count(1) AS sessions_count
FROM ranked_events
INNER JOIN start_points USING (session_id)
WHERE event_number_in_session = 1
GROUP BY event_number_in_session, event_type, e_value, next_type, next_value, time_from_previous)"""]
projection_query = ["""(SELECT event_number_in_session,
event_type,
e_value,
next_type,
next_value,
sessions_count,
avg(time_from_previous) AS avg_time_from_previous
FROM n1
GROUP BY event_number_in_session, event_type, e_value, next_type, next_value, sessions_count
ORDER BY event_number_in_session, event_type, e_value, next_type, next_value)"""]
for i in range(2, data.density):
steps_query.append(f"""n{i} AS (SELECT *
FROM (SELECT re.event_number_in_session,
re.event_type,
re.e_value,
re.next_type,
re.next_value,
re.time_from_previous,
count(1) AS sessions_count
FROM ranked_events AS re
INNER JOIN n{i - 1} ON (n{i - 1}.next_value = re.e_value)
WHERE re.event_number_in_session = {i}
GROUP BY re.event_number_in_session, re.event_type, re.e_value, re.next_type, re.next_value,
re.time_from_previous) AS sub_level
ORDER BY sessions_count DESC
LIMIT %(eventThresholdNumberInGroup)s)""")
projection_query.append(f"""(SELECT event_number_in_session,
event_type,
e_value,
next_type,
next_value,
sessions_count,
avg(time_from_previous) AS avg_time_from_previous
FROM n{i}
GROUP BY event_number_in_session, event_type, e_value, next_type, next_value, sessions_count
ORDER BY event_number_in_session, event_type, e_value, next_type, next_value)""")
with pg_client.PostgresClient() as cur:
pg_query = f"""\
WITH sub_sessions AS ( SELECT session_id
FROM public.sessions
WHERE {" AND ".join(sessions_conditions)}),
WITH sub_sessions AS (SELECT session_id
FROM public.sessions
WHERE {" AND ".join(sessions_conditions)}),
sub_events AS ({events_subquery}),
ranked_events AS (SELECT *
FROM (SELECT session_id,
event_type,
e_value,
row_number() OVER (PARTITION BY session_id ORDER BY timestamp {path_direction}) AS event_number_in_session,
LEAD(e_value, 1) OVER (PARTITION BY session_id ORDER BY timestamp {path_direction}) AS next_value,
LEAD(event_type, 1) OVER (PARTITION BY session_id ORDER BY timestamp {path_direction}) AS next_type,
abs(LEAD(timestamp, 1) OVER (PARTITION BY session_id ORDER BY timestamp {path_direction}) -
timestamp) AS time_to_next
FROM sub_events
ORDER BY session_id) AS full_ranked_events
WHERE event_number_in_session < %(density)s
),
pre_ranked_events AS (SELECT *
FROM (SELECT session_id,
event_type,
e_value,
timestamp,
row_number() OVER (PARTITION BY session_id ORDER BY timestamp {path_direction}) AS event_number_in_session
FROM sub_events
ORDER BY session_id) AS full_ranked_events
WHERE event_number_in_session < %(density)s),
start_points AS (SELECT session_id
FROM ranked_events {start_points_join}
FROM {start_points_from}
WHERE {" AND ".join(start_points_conditions)}),
limited_events AS (SELECT *
FROM (SELECT *,
row_number()
OVER (PARTITION BY event_number_in_session, event_type, e_value ORDER BY sessions_count DESC ) AS _event_number_in_group
FROM (SELECT event_number_in_session,
event_type,
e_value,
next_type,
next_value,
time_to_next,
count(1) AS sessions_count
FROM ranked_events
INNER JOIN start_points USING (session_id)
GROUP BY event_number_in_session, event_type, e_value, next_type, next_value,
time_to_next) AS groupped_events) AS ranked_groupped_events
WHERE _event_number_in_group < %(eventThresholdNumberInGroup)s)
SELECT event_number_in_session,
event_type,
e_value,
next_type,
next_value,
sessions_count,
avg(time_to_next) AS avg_time_to_target
FROM limited_events
GROUP BY event_number_in_session, event_type, e_value, next_type, next_value, sessions_count
ORDER BY event_number_in_session, e_value, next_value;"""
ranked_events AS (SELECT *,
LEAD(e_value, 1) OVER (PARTITION BY session_id ORDER BY timestamp {path_direction}) AS next_value,
LEAD(event_type, 1) OVER (PARTITION BY session_id ORDER BY timestamp {path_direction}) AS next_type,
abs(LAG(timestamp, 1) OVER (PARTITION BY session_id ORDER BY timestamp {path_direction}) -
timestamp) AS time_from_previous
FROM pre_ranked_events
INNER JOIN start_points USING (session_id)),
{",".join(steps_query)}
{"UNION ALL".join(projection_query)};"""
params = {"project_id": project_id, "startTimestamp": data.startTimestamp,
"endTimestamp": data.endTimestamp, "density": data.density,
"eventThresholdNumberInGroup": 8 if data.hide_excess else 6,
# TODO: add if data=args is required
# **__get_constraint_values(args),
"eventThresholdNumberInGroup": 6 if data.hide_excess else 8,
**extra_values}
query = cur.mogrify(pg_query, params)
_now = time()
cur.execute(query)
if time() - _now > 3:
if time() - _now > 2:
print(f">>>>>>>>>PathAnalysis long query ({int(time() - _now)}s)<<<<<<<<<")
print("----------------------")
print(query)
print("----------------------")
rows = cur.fetchall()
return __transform_journey2(rows=rows, reverse_path=reverse)
return __transform_journey3(rows=rows, reverse_path=reverse)
#
# def __compute_weekly_percentage(rows):

View file

@ -1283,7 +1283,7 @@ class CardPathAnalysis(__CardSchema):
start_type: Literal["start", "end"] = Field(default="start")
start_point: List[PathAnalysisSubFilterSchema] = Field(default=[])
exclude: List[PathAnalysisSubFilterSchema] = Field(default=[])
excludes: List[PathAnalysisSubFilterSchema] = Field(default=[])
series: List[CardPathAnalysisSchema] = Field(default=[])
@ -1325,7 +1325,7 @@ class CardPathAnalysis(__CardSchema):
for f in values.start_point:
s_e_values[f.type] = s_e_values.get(f.type, []) + f.value
for f in values.exclude:
for f in values.excludes:
exclude_values[f.type] = exclude_values.get(f.type, []) + f.value
assert len(

1
ee/api/.gitignore vendored
View file

@ -272,3 +272,4 @@ Pipfile.lock
#exp /chalicelib/core/dashboards.py
/schemas/overrides.py
/schemas/schemas.py
/chalicelib/core/authorizers.py

View file

@ -1,4 +1,5 @@
import datetime
import logging
from typing import Optional
from fastapi import Request
@ -9,11 +10,13 @@ from starlette.exceptions import HTTPException
import schemas
from chalicelib.core import authorizers, users
logger = logging.getLogger(__name__)
def _get_current_auth_context(request: Request, jwt_payload: dict) -> schemas.CurrentContext:
user = users.get(user_id=jwt_payload.get("userId", -1), tenant_id=jwt_payload.get("tenantId", -1))
if user is None:
print("JWTAuth: User not found.")
logger.warning("User not found.")
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="User not found.")
request.state.authorizer_identity = "jwt"
if user["serviceAccount"]:
@ -72,16 +75,17 @@ class JWTAuth(HTTPBearer):
or jwt_payload.get("iat") is None or jwt_payload.get("aud") is None \
or not auth_exists:
if jwt_payload is not None:
print(jwt_payload)
logger.debug(jwt_payload)
if jwt_payload.get("iat") is None:
print("JWTAuth: iat is None")
logger.debug("iat is None")
if jwt_payload.get("aud") is None:
print("JWTAuth: aud is None")
logger.debug("aud is None")
if not auth_exists:
print("JWTAuth: not users.auth_exists")
logger.warning("not users.auth_exists")
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid token or expired token.")
return _get_current_auth_context(request=request, jwt_payload=jwt_payload)
print("JWTAuth: Invalid authorization code.")
logger.warning("Invalid authorization code.")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid authorization code.")

View file

@ -1,104 +0,0 @@
import jwt
from decouple import config
from chalicelib.core import tenants
from chalicelib.core import users
from chalicelib.utils import helper
from chalicelib.utils.TimeUTC import TimeUTC
def jwt_authorizer(scheme: str, token: str, leeway=0):
if scheme.lower() != "bearer":
return None
try:
payload = jwt.decode(
token,
config("jwt_secret"),
algorithms=config("jwt_algorithm"),
audience=[f"front:{helper.get_stage_name()}"],
leeway=leeway
)
except jwt.ExpiredSignatureError:
print("! JWT Expired signature")
return None
except BaseException as e:
print("! JWT Base Exception")
print(e)
return None
return payload
def jwt_refresh_authorizer(scheme: str, token: str):
if scheme.lower() != "bearer":
return None
try:
payload = jwt.decode(
token,
config("JWT_REFRESH_SECRET"),
algorithms=config("jwt_algorithm"),
audience=[f"front:{helper.get_stage_name()}"]
)
except jwt.ExpiredSignatureError:
print("! JWT-refresh Expired signature")
return None
except BaseException as e:
print("! JWT-refresh Base Exception")
print(e)
return None
return payload
def jwt_context(context):
user = users.get(user_id=context["userId"], tenant_id=context["tenantId"])
if user is None:
return None
return {
"tenantId": context["tenantId"],
"userId": context["userId"],
**user
}
def get_jwt_exp(iat):
return iat // 1000 + config("JWT_EXPIRATION", cast=int) + TimeUTC.get_utc_offset() // 1000
def generate_jwt(user_id, tenant_id, iat, aud, exp=None):
token = jwt.encode(
payload={
"userId": user_id,
"tenantId": tenant_id,
"exp": exp + TimeUTC.get_utc_offset() // 1000 if exp is not None else iat + config("JWT_EXPIRATION",
cast=int),
"iss": config("JWT_ISSUER"),
"iat": iat,
"aud": aud
},
key=config("jwt_secret"),
algorithm=config("jwt_algorithm")
)
return token
def generate_jwt_refresh(user_id, tenant_id, iat, aud, jwt_jti):
token = jwt.encode(
payload={
"userId": user_id,
"tenantId": tenant_id,
"exp": iat + config("JWT_REFRESH_EXPIRATION", cast=int),
"iss": config("JWT_ISSUER"),
"iat": iat,
"aud": aud,
"jti": jwt_jti
},
key=config("JWT_REFRESH_SECRET"),
algorithm=config("jwt_algorithm")
)
return token
def api_key_authorizer(token):
t = tenants.get_by_api_key(token)
if t is not None:
t["createdAt"] = TimeUTC.datetime_to_timestamp(t["createdAt"])
return t

View file

@ -11,20 +11,6 @@ from chalicelib.core import metadata
from time import time
def __transform_journey(rows):
nodes = []
links = []
for r in rows:
source = r["source_event"][r["source_event"].index("_") + 1:]
target = r["target_event"][r["target_event"].index("_") + 1:]
if source not in nodes:
nodes.append(source)
if target not in nodes:
nodes.append(target)
links.append({"source": nodes.index(source), "target": nodes.index(target), "value": r["value"]})
return {"nodes": nodes, "links": sorted(links, key=lambda x: x["value"], reverse=True)}
def __transform_journey2(rows, reverse_path=False):
# nodes should contain duplicates for different steps otherwise the UI crashes
nodes = []
@ -52,7 +38,7 @@ def __transform_journey2(rows, reverse_path=False):
links.append(link)
return {"nodes": nodes_values,
"links": sorted(links, key=lambda x: x["value"], reverse=True)}
"links": sorted(links, key=lambda x: (x["source"], x["target"]), reverse=False)}
JOURNEY_TYPES = {

View file

@ -4,6 +4,7 @@ rm -rf ./chalicelib/core/alerts.py
#exp rm -rf ./chalicelib/core/alerts_processor.py
rm -rf ./chalicelib/core/announcements.py
rm -rf ./chalicelib/core/autocomplete.py
rm -rf ./chalicelib/core/authorizers.py
rm -rf ./chalicelib/core/click_maps.py
rm -rf ./chalicelib/core/collaboration_base.py
rm -rf ./chalicelib/core/collaboration_msteams.py