jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(chalice): refactored autocomplete

Browse source 
feat(chalice): ch autocomplete
feat(chalice): changed project-auth
This commit is contained in:
Taha Yassine Kraiem 2022-08-02 13:09:13 +02:00
parent 80b4af501f
commit e9453d7a40
16 changed files with 284 additions and 204 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
api/auth/auth_project.py
View file

@ -15,10 +15,12 @@ class ProjectAuthorizer:
if len(request.path_params.keys()) == 0 or request.path_params.get(self.project_identifier) is None:
return
current_user: schemas.CurrentContext = await OR_context(request)
project_identifier = request.path_params[self.project_identifier]
value = request.path_params[self.project_identifier]
if (self.project_identifier == "projectId" \
and projects.get_project(project_id=project_identifier, tenant_id=current_user.tenant_id) is None) \
or (self.project_identifier.lower() == "projectKey" \
and projects.get_internal_project_id(project_key=project_identifier) is None):
and not (isinstance(value, int) or isinstance(value, str) and value.isnumeric())
and projects.get_project(project_id=value, tenant_id=current_user.tenant_id) is None) \
or (self.project_identifier == "projectKey" \
and projects.get_internal_project_id(project_key=value) is None):
print("project not found")
print(value)
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="project not found.")

86
api/chalicelib/core/autocomplete.py Normal file
View file

@ -0,0 +1,86 @@
from chalicelib.utils import helper
from chalicelib.utils import pg_client
import schemas
from chalicelib.utils.event_filter_definition import SupportedFilter, Event
def __get_autocomplete_table(value, project_id):
autocomplete_events = [schemas.FilterType.rev_id,
schemas.EventType.click,
schemas.FilterType.user_device,
schemas.FilterType.user_id,
schemas.FilterType.user_browser,
schemas.FilterType.user_os,
schemas.EventType.custom,
schemas.FilterType.user_country,
schemas.EventType.location,
schemas.EventType.input]
autocomplete_events.sort()
sub_queries = []
for e in autocomplete_events:
sub_queries.append(f"""(SELECT type, value
FROM public.autocomplete
WHERE project_id = %(project_id)s
AND type= '{e}'
AND value ILIKE %(svalue)s
LIMIT 5)""")
if len(value) > 2:
sub_queries.append(f"""(SELECT type, value
FROM public.autocomplete
WHERE project_id = %(project_id)s
AND type= '{e}'
AND value ILIKE %(value)s
LIMIT 5)""")
with pg_client.PostgresClient() as cur:
query = cur.mogrify(" UNION DISTINCT ".join(sub_queries) + ";",
{"project_id": project_id, "value": helper.string_to_sql_like(value),
"svalue": helper.string_to_sql_like("^" + value)})
try:
cur.execute(query)
except Exception as err:
print("--------- AUTOCOMPLETE SEARCH QUERY EXCEPTION -----------")
print(query.decode('UTF-8'))
print("--------- VALUE -----------")
print(value)
print("--------------------")
raise err
results = helper.list_to_camel_case(cur.fetchall())
return results
def __generic_query(typename, value_length=None):
if value_length is None or value_length > 2:
return f"""(SELECT DISTINCT value, type
FROM public.autocomplete
WHERE
project_id = %(project_id)s
AND type='{typename}'
AND value ILIKE %(svalue)s
LIMIT 5)
UNION DISTINCT
(SELECT DISTINCT value, type
FROM public.autocomplete
WHERE
project_id = %(project_id)s
AND type='{typename}'
AND value ILIKE %(value)s
LIMIT 5);"""
return f"""SELECT DISTINCT value, type
FROM public.autocomplete
WHERE
project_id = %(project_id)s
AND type='{typename}'
AND value ILIKE %(svalue)s
LIMIT 10;"""
def __generic_autocomplete(event: Event):
def f(project_id, value, key=None, source=None):
with pg_client.PostgresClient() as cur:
query = __generic_query(event.ui_type, value_length=len(value))
params = {"project_id": project_id, "value": helper.string_to_sql_like(value),
"svalue": helper.string_to_sql_like("^" + value)}
cur.execute(cur.mogrify(query, params))
return helper.list_to_camel_case(cur.fetchall())
return f

209
api/chalicelib/core/events.py
View file

@ -1,10 +1,14 @@
import schemas
from chalicelib.core import issues
from chalicelib.core import sessions_metas, metadata
from chalicelib.core import metadata
from chalicelib.core import sessions_metas
from chalicelib.utils import pg_client, helper
from chalicelib.utils.TimeUTC import TimeUTC
from chalicelib.utils.event_filter_definition import SupportedFilter, Event
from chalicelib.core import autocomplete
def get_customs_by_sessionId2_pg(session_id, project_id):
with pg_client.PostgresClient() as cur:
@ -92,11 +96,6 @@ def get_by_sessionId2_pg(session_id, project_id, group_clickrage=False):
return rows
def __get_data_for_extend(data):
if "errors" not in data:
return data["data"]
def __pg_errors_query(source=None, value_length=None):
if value_length is None or value_length > 2:
return f"""((SELECT DISTINCT ON(lg.message)
@ -110,7 +109,7 @@ def __pg_errors_query(source=None, value_length=None):
AND lg.project_id = %(project_id)s
{"AND source = %(source)s" if source is not None else ""}
LIMIT 5)
UNION ALL
UNION DISTINCT
(SELECT DISTINCT ON(lg.name)
lg.name AS value,
source,
@ -122,7 +121,7 @@ def __pg_errors_query(source=None, value_length=None):
AND lg.project_id = %(project_id)s
{"AND source = %(source)s" if source is not None else ""}
LIMIT 5)
UNION
UNION DISTINCT
(SELECT DISTINCT ON(lg.message)
lg.message AS value,
source,
@ -134,7 +133,7 @@ def __pg_errors_query(source=None, value_length=None):
AND lg.project_id = %(project_id)s
{"AND source = %(source)s" if source is not None else ""}
LIMIT 5)
UNION ALL
UNION DISTINCT
(SELECT DISTINCT ON(lg.name)
lg.name AS value,
source,
@ -157,7 +156,7 @@ def __pg_errors_query(source=None, value_length=None):
AND lg.project_id = %(project_id)s
{"AND source = %(source)s" if source is not None else ""}
LIMIT 5)
UNION ALL
UNION DISTINCT
(SELECT DISTINCT ON(lg.name)
lg.name AS value,
source,
@ -177,8 +176,7 @@ def __search_pg_errors(project_id, value, key=None, source=None):
with pg_client.PostgresClient() as cur:
cur.execute(
cur.mogrify(__pg_errors_query(source,
value_length=len(value) \
if SUPPORTED_TYPES[event_type.ERROR.ui_type].change_by_length else None),
value_length=len(value)),
{"project_id": project_id, "value": helper.string_to_sql_like(value),
"svalue": helper.string_to_sql_like("^" + value),
"source": source}))
@ -189,7 +187,7 @@ def __search_pg_errors(project_id, value, key=None, source=None):
def __search_pg_errors_ios(project_id, value, key=None, source=None):
now = TimeUTC.now()
if SUPPORTED_TYPES[event_type.ERROR_IOS.ui_type].change_by_length is False or len(value) > 2:
if len(value) > 2:
query = f"""(SELECT DISTINCT ON(lg.reason)
lg.reason AS value,
'{event_type.ERROR_IOS.ui_type}' AS type
@ -268,7 +266,7 @@ def __search_pg_metadata(project_id, value, key=None, source=None):
for k in meta_keys.keys():
colname = metadata.index_to_colname(meta_keys[k])
if SUPPORTED_TYPES[event_type.METADATA.ui_type].change_by_length is False or len(value) > 2:
if len(value) > 2:
sub_from.append(f"""((SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key
FROM public.sessions
WHERE project_id = %(project_id)s
@ -294,48 +292,6 @@ def __search_pg_metadata(project_id, value, key=None, source=None):
return results
def __generic_query(typename, value_length=None):
if value_length is None or value_length > 2:
return f"""(SELECT DISTINCT value, type
FROM public.autocomplete
WHERE
project_id = %(project_id)s
AND type='{typename}'
AND value ILIKE %(svalue)s
LIMIT 5)
UNION
(SELECT DISTINCT value, type
FROM public.autocomplete
WHERE
project_id = %(project_id)s
AND type='{typename}'
AND value ILIKE %(value)s
LIMIT 5);"""
return f"""SELECT DISTINCT value, type
FROM public.autocomplete
WHERE
project_id = %(project_id)s
AND type='{typename}'
AND value ILIKE %(svalue)s
LIMIT 10;"""
def __generic_autocomplete(event: Event):
def f(project_id, value, key=None, source=None):
with pg_client.PostgresClient() as cur:
cur.execute(
cur.mogrify(
__generic_query(event.ui_type,
value_length=len(value) \
if SUPPORTED_TYPES[event.ui_type].change_by_length \
else None),
{"project_id": project_id, "value": helper.string_to_sql_like(value),
"svalue": helper.string_to_sql_like("^" + value)}))
return helper.list_to_camel_case(cur.fetchall())
return f
class event_type:
CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label")
INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label")
@ -358,99 +314,65 @@ class event_type:
SUPPORTED_TYPES = {
event_type.CLICK.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CLICK),
query=__generic_query(typename=event_type.CLICK.ui_type),
change_by_length=True),
event_type.INPUT.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.INPUT),
query=__generic_query(typename=event_type.INPUT.ui_type),
change_by_length=True),
event_type.LOCATION.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.LOCATION),
query=__generic_query(typename=event_type.LOCATION.ui_type),
change_by_length=True),
event_type.CUSTOM.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CUSTOM),
query=__generic_query(typename=event_type.CUSTOM.ui_type),
change_by_length=True),
event_type.REQUEST.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.REQUEST),
query=__generic_query(typename=event_type.REQUEST.ui_type),
change_by_length=True),
event_type.GRAPHQL.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.GRAPHQL),
query=__generic_query(typename=event_type.GRAPHQL.ui_type),
change_by_length=True),
event_type.STATEACTION.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.STATEACTION),
query=__generic_query(typename=event_type.STATEACTION.ui_type),
change_by_length=True),
event_type.CLICK.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.CLICK),
query=autocomplete.__generic_query(typename=event_type.CLICK.ui_type)),
event_type.INPUT.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.INPUT),
query=autocomplete.__generic_query(typename=event_type.INPUT.ui_type)),
event_type.LOCATION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.LOCATION),
query=autocomplete.__generic_query(
typename=event_type.LOCATION.ui_type)),
event_type.CUSTOM.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.CUSTOM),
query=autocomplete.__generic_query(typename=event_type.CUSTOM.ui_type)),
event_type.REQUEST.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.REQUEST),
query=autocomplete.__generic_query(
typename=event_type.REQUEST.ui_type)),
event_type.GRAPHQL.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.GRAPHQL),
query=autocomplete.__generic_query(
typename=event_type.GRAPHQL.ui_type)),
event_type.STATEACTION.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.STATEACTION),
query=autocomplete.__generic_query(
typename=event_type.STATEACTION.ui_type)),
event_type.ERROR.ui_type: SupportedFilter(get=__search_pg_errors,
query=None, change_by_length=True),
query=None),
event_type.METADATA.ui_type: SupportedFilter(get=__search_pg_metadata,
query=None, change_by_length=True),
query=None),
# IOS
event_type.CLICK_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CLICK_IOS),
query=__generic_query(typename=event_type.CLICK_IOS.ui_type),
change_by_length=True),
event_type.INPUT_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.INPUT_IOS),
query=__generic_query(typename=event_type.INPUT_IOS.ui_type),
change_by_length=True),
event_type.VIEW_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.VIEW_IOS),
query=__generic_query(typename=event_type.VIEW_IOS.ui_type),
change_by_length=True),
event_type.CUSTOM_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CUSTOM_IOS),
query=__generic_query(typename=event_type.CUSTOM_IOS.ui_type),
change_by_length=True),
event_type.REQUEST_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.REQUEST_IOS),
query=__generic_query(typename=event_type.REQUEST_IOS.ui_type),
change_by_length=True),
event_type.CLICK_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.CLICK_IOS),
query=autocomplete.__generic_query(
typename=event_type.CLICK_IOS.ui_type)),
event_type.INPUT_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.INPUT_IOS),
query=autocomplete.__generic_query(
typename=event_type.INPUT_IOS.ui_type)),
event_type.VIEW_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.VIEW_IOS),
query=autocomplete.__generic_query(
typename=event_type.VIEW_IOS.ui_type)),
event_type.CUSTOM_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.CUSTOM_IOS),
query=autocomplete.__generic_query(
typename=event_type.CUSTOM_IOS.ui_type)),
event_type.REQUEST_IOS.ui_type: SupportedFilter(get=autocomplete.__generic_autocomplete(event_type.REQUEST_IOS),
query=autocomplete.__generic_query(
typename=event_type.REQUEST_IOS.ui_type)),
event_type.ERROR_IOS.ui_type: SupportedFilter(get=__search_pg_errors_ios,
query=None, change_by_length=True),
query=None),
}
def __get_autocomplete_table(value, project_id):
autocomplete_events = [schemas.FilterType.rev_id,
schemas.EventType.click,
schemas.FilterType.user_device,
schemas.FilterType.user_id,
schemas.FilterType.user_browser,
schemas.FilterType.user_os,
schemas.EventType.custom,
schemas.FilterType.user_country,
schemas.EventType.location,
schemas.EventType.input]
autocomplete_events.sort()
sub_queries = []
for e in autocomplete_events:
sub_queries.append(f"""(SELECT type, value
FROM public.autocomplete
WHERE project_id = %(project_id)s
AND type= '{e}'
AND value ILIKE %(svalue)s
LIMIT 5)""")
if len(value) > 2:
sub_queries.append(f"""(SELECT type, value
FROM public.autocomplete
WHERE project_id = %(project_id)s
AND type= '{e}'
AND value ILIKE %(value)s
LIMIT 5)""")
def get_errors_by_session_id(session_id, project_id):
with pg_client.PostgresClient() as cur:
query = cur.mogrify(" UNION ".join(sub_queries) + ";",
{"project_id": project_id, "value": helper.string_to_sql_like(value),
"svalue": helper.string_to_sql_like("^" + value)})
try:
cur.execute(query)
except Exception as err:
print("--------- AUTOCOMPLETE SEARCH QUERY EXCEPTION -----------")
print(query.decode('UTF-8'))
print("--------- VALUE -----------")
print(value)
print("--------------------")
raise err
results = helper.list_to_camel_case(cur.fetchall())
return results
cur.execute(cur.mogrify(f"""\
SELECT er.*,ur.*, er.timestamp - s.start_ts AS time
FROM {event_type.ERROR.table} AS er INNER JOIN public.errors AS ur USING (error_id) INNER JOIN public.sessions AS s USING (session_id)
WHERE er.session_id = %(session_id)s AND s.project_id=%(project_id)s
ORDER BY timestamp;""", {"session_id": session_id, "project_id": project_id}))
errors = cur.fetchall()
for e in errors:
e["stacktrace_parsed_at"] = TimeUTC.datetime_to_timestamp(e["stacktrace_parsed_at"])
return helper.list_to_camel_case(errors)
def search(text, event_type, project_id, source, key):
if not event_type:
return {"data": __get_autocomplete_table(text, project_id)}
return {"data": autocomplete.__get_autocomplete_table(text, project_id)}
if event_type in SUPPORTED_TYPES.keys():
rows = SUPPORTED_TYPES[event_type].get(project_id=project_id, value=text, key=key, source=source)
@ -470,16 +392,3 @@ def search(text, event_type, project_id, source, key):
return {"errors": ["unsupported event"]}
return {"data": rows}
def get_errors_by_session_id(session_id, project_id):
with pg_client.PostgresClient() as cur:
cur.execute(cur.mogrify(f"""\
SELECT er.*,ur.*, er.timestamp - s.start_ts AS time
FROM {event_type.ERROR.table} AS er INNER JOIN public.errors AS ur USING (error_id) INNER JOIN public.sessions AS s USING (session_id)
WHERE er.session_id = %(session_id)s AND s.project_id=%(project_id)s
ORDER BY timestamp;""", {"session_id": session_id, "project_id": project_id}))
errors = cur.fetchall()
for e in errors:
e["stacktrace_parsed_at"] = TimeUTC.datetime_to_timestamp(e["stacktrace_parsed_at"])
return helper.list_to_camel_case(errors)

54
api/chalicelib/core/sessions_metas.py
View file

@ -113,8 +113,7 @@ def __generic_autocomplete(typename):
def f(project_id, text):
with pg_client.PostgresClient() as cur:
query = cur.mogrify(__generic_query(typename,
value_length=len(text) \
if SUPPORTED_TYPES[typename].change_by_length else None),
value_length=len(text)),
{"project_id": project_id, "value": helper.string_to_sql_like(text),
"svalue": helper.string_to_sql_like("^" + text)})
@ -128,74 +127,57 @@ def __generic_autocomplete(typename):
SUPPORTED_TYPES = {
schemas.FilterType.user_os: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.user_os),
query=__generic_query(typename=schemas.FilterType.user_os),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.user_os)),
schemas.FilterType.user_browser: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.user_browser),
query=__generic_query(typename=schemas.FilterType.user_browser),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.user_browser)),
schemas.FilterType.user_device: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.user_device),
query=__generic_query(typename=schemas.FilterType.user_device),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.user_device)),
schemas.FilterType.user_country: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.user_country),
query=__generic_query(typename=schemas.FilterType.user_country),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.user_country)),
schemas.FilterType.user_id: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.user_id),
query=__generic_query(typename=schemas.FilterType.user_id),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.user_id)),
schemas.FilterType.user_anonymous_id: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.user_anonymous_id),
query=__generic_query(typename=schemas.FilterType.user_anonymous_id),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.user_anonymous_id)),
schemas.FilterType.rev_id: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.rev_id),
query=__generic_query(typename=schemas.FilterType.rev_id),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.rev_id)),
schemas.FilterType.referrer: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.referrer),
query=__generic_query(typename=schemas.FilterType.referrer),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.referrer)),
schemas.FilterType.utm_campaign: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.utm_campaign),
query=__generic_query(typename=schemas.FilterType.utm_campaign),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.utm_campaign)),
schemas.FilterType.utm_medium: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.utm_medium),
query=__generic_query(typename=schemas.FilterType.utm_medium),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.utm_medium)),
schemas.FilterType.utm_source: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.utm_source),
query=__generic_query(typename=schemas.FilterType.utm_source),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.utm_source)),
# IOS
schemas.FilterType.user_os_ios: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.user_os_ios),
query=__generic_query(typename=schemas.FilterType.user_os_ios),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.user_os_ios)),
schemas.FilterType.user_device_ios: SupportedFilter(
get=__generic_autocomplete(
typename=schemas.FilterType.user_device_ios),
query=__generic_query(typename=schemas.FilterType.user_device_ios),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.user_device_ios)),
schemas.FilterType.user_country_ios: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.user_country_ios),
query=__generic_query(typename=schemas.FilterType.user_country_ios),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.user_country_ios)),
schemas.FilterType.user_id_ios: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.user_id_ios),
query=__generic_query(typename=schemas.FilterType.user_id_ios),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.user_id_ios)),
schemas.FilterType.user_anonymous_id_ios: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.user_anonymous_id_ios),
query=__generic_query(typename=schemas.FilterType.user_anonymous_id_ios),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.user_anonymous_id_ios)),
schemas.FilterType.rev_id_ios: SupportedFilter(
get=__generic_autocomplete(typename=schemas.FilterType.rev_id_ios),
query=__generic_query(typename=schemas.FilterType.rev_id_ios),
change_by_length=True),
query=__generic_query(typename=schemas.FilterType.rev_id_ios)),
}

3
api/chalicelib/utils/event_filter_definition.py
View file

@ -6,7 +6,6 @@ class Event:
class SupportedFilter:
def __init__(self, get, query, change_by_length):
def __init__(self, get, query):
self.get = get
self.query = query
self.change_by_length = change_by_length

1
ee/api/.gitignore vendored
View file

@ -180,6 +180,7 @@ Pipfile
/chalicelib/core/alerts.py
/chalicelib/core/alerts_processor.py
/chalicelib/core/announcements.py
/chalicelib/core/autocomplete.py
/chalicelib/core/collaboration_slack.py
/chalicelib/core/errors_favorite_viewed.py
/chalicelib/core/events.py

12
ee/api/auth/auth_project.py
View file

@ -15,13 +15,15 @@ class ProjectAuthorizer:
if len(request.path_params.keys()) == 0 or request.path_params.get(self.project_identifier) is None:
return
current_user: schemas.CurrentContext = await OR_context(request)
project_identifier = request.path_params[self.project_identifier]
value = request.path_params[self.project_identifier]
user_id = current_user.user_id if request.state.authorizer_identity == "jwt" else None
if (self.project_identifier == "projectId" \
and not projects.is_authorized(project_id=project_identifier, tenant_id=current_user.tenant_id,
and not projects.is_authorized(project_id=value, tenant_id=current_user.tenant_id,
user_id=user_id)) \
or (self.project_identifier.lower() == "projectKey" \
and not projects.is_authorized(project_id=projects.get_internal_project_id(project_identifier),
tenant_id=current_user.tenant_id, user_id=user_id)):
or (self.project_identifier == "projectKey" \
and not projects.is_authorized(
project_id=projects.get_internal_project_id(value),
tenant_id=current_user.tenant_id, user_id=user_id)):
print("unauthorized project")
print(value)
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="unauthorized project.")

6
ee/api/chalicelib/core/__init__.py
View file

@ -0,0 +1,6 @@
from decouple import config
if config("LEGACY_SEARCH", cast=bool, default=True):
from . import autocomplete as autocomplete
else:
from . import autocomplete_ee as autocomplete

88
ee/api/chalicelib/core/autocomplete_ee.py Normal file
View file

@ -0,0 +1,88 @@
import schemas
from chalicelib.utils import ch_client
from chalicelib.utils import helper
from chalicelib.utils.event_filter_definition import Event
def __get_autocomplete_table(value, project_id):
autocomplete_events = [schemas.FilterType.rev_id,
schemas.EventType.click,
schemas.FilterType.user_device,
schemas.FilterType.user_id,
schemas.FilterType.user_browser,
schemas.FilterType.user_os,
schemas.EventType.custom,
schemas.FilterType.user_country,
schemas.EventType.location,
schemas.EventType.input]
autocomplete_events.sort()
sub_queries = []
for e in autocomplete_events:
sub_queries.append(f"""(SELECT type, value
FROM final.autocomplete
WHERE project_id = %(project_id)s
AND type= '{e}'
AND value ILIKE %(svalue)s
LIMIT 5)""")
if len(value) > 2:
sub_queries.append(f"""(SELECT type, value
FROM final.autocomplete
WHERE project_id = %(project_id)s
AND type= '{e}'
AND value ILIKE %(value)s
LIMIT 5)""")
with ch_client.ClickHouseClient() as cur:
query = " UNION DISTINCT ".join(sub_queries) + ";"
params = {"project_id": project_id, "value": helper.string_to_sql_like(value),
"svalue": helper.string_to_sql_like("^" + value)}
results = []
try:
results = cur.execute(query=query, params=params)
except Exception as err:
print("--------- CH AUTOCOMPLETE SEARCH QUERY EXCEPTION -----------")
print(cur.format(query=query, params=params))
print("--------- PARAMS -----------")
print(params)
print("--------- VALUE -----------")
print(value)
print("--------------------")
raise err
return results
def __generic_query(typename, value_length=None):
if value_length is None or value_length > 2:
return f"""(SELECT DISTINCT value, type
FROM final.autocomplete
WHERE
project_id = %(project_id)s
AND type='{typename}'
AND value ILIKE %(svalue)s
LIMIT 5)
UNION DISTINCT
(SELECT DISTINCT value, type
FROM final.autocomplete
WHERE
project_id = %(project_id)s
AND type='{typename}'
AND value ILIKE %(value)s
LIMIT 5);"""
return f"""SELECT DISTINCT value, type
FROM final.autocomplete
WHERE
project_id = %(project_id)s
AND type='{typename}'
AND value ILIKE %(svalue)s
LIMIT 10;"""
def __generic_autocomplete(event: Event):
def f(project_id, value, key=None, source=None):
with ch_client.ClickHouseClient() as cur:
query = __generic_query(event.ui_type, value_length=len(value))
params = {"project_id": project_id, "value": helper.string_to_sql_like(value),
"svalue": helper.string_to_sql_like("^" + value)}
results = cur.execute(query=query, params=params)
return helper.list_to_camel_case(results)
return f

10
ee/api/chalicelib/core/sessions.py
View file

@ -1217,10 +1217,10 @@ def __get_event_type(event_type: Union[schemas.EventType, schemas.PerformanceEve
defs = {
schemas.EventType.click: "CLICK",
schemas.EventType.input: "INPUT",
schemas.EventType.location: "PAGE",
schemas.PerformanceEventType.location_dom_complete: "PAGE",
schemas.PerformanceEventType.location_largest_contentful_paint_time: "PAGE",
schemas.PerformanceEventType.location_ttfb: "PAGE",
schemas.EventType.location: "LOCATION",
schemas.PerformanceEventType.location_dom_complete: "LOCATION",
schemas.PerformanceEventType.location_largest_contentful_paint_time: "LOCATION",
schemas.PerformanceEventType.location_ttfb: "LOCATION",
schemas.EventType.custom: "CUSTOM",
schemas.EventType.request: "REQUEST",
schemas.EventType.request_details: "REQUEST",
@ -1385,7 +1385,7 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue,
referrer_constraint = f"""(SELECT DISTINCT session_id
FROM {MAIN_EVENTS_TABLE} AS r
WHERE {" AND ".join([f"r.{b}" for b in __events_where_basic])}
AND event_type='PAGE'
AND event_type='{__get_event_type(schemas.EventType.location)}'
AND {referrer_constraint})"""
# events_conditions_where.append(f"""main.session_id IN {referrer_constraint}""")
# extra_constraints.append(f"""s.session_id IN {referrer_constraint}""")

1
ee/api/clean.sh
View file

@ -3,6 +3,7 @@
rm -rf ./chalicelib/core/alerts.py
rm -rf ./chalicelib/core/alerts_processor.py
rm -rf ./chalicelib/core/announcements.py
rm -rf ./chalicelib/core/autocomplete.py
rm -rf ./chalicelib/core/collaboration_slack.py
rm -rf ./chalicelib/core/errors_favorite_viewed.py
rm -rf ./chalicelib/core/events.py

1
ee/api/env.default
View file

@ -57,3 +57,4 @@ sourcemaps_bucket=sourcemaps
sourcemaps_reader=http://127.0.0.1:9000/sourcemaps
stage=default-ee
version_number=1.0.0
LEGACY_SEARCH=false

2
ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql
View file

@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS events_s
(
session_id UInt64,
project_id UInt16,
event_type Enum8('CLICK'=0, 'INPUT'=1, 'PAGE'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8, 'GRAPHQL'=9, 'STATEACTION'=10),
event_type Enum8('CLICK'=0, 'INPUT'=1, 'LOCATION'=2,'RESOURCE'=3,'REQUEST'=4,'PERFORMANCE'=5,'LONGTASK'=6,'ERROR'=7,'CUSTOM'=8, 'GRAPHQL'=9, 'STATEACTION'=10),
datetime DateTime,
label Nullable(String),
hesitation_time Nullable(UInt32),

1
ee/utilities/.gitignore vendored
View file

@ -14,3 +14,4 @@ servers/sourcemaps-server.js
/utils/HeapSnapshot.js
/utils/helper.js
/utils/assistHelper.js
.local

1
utilities/.dockerignore
View file

@ -4,3 +4,4 @@
**/build.sh
**/build_*.sh
**/*deploy.sh
.local

3
utilities/.gitignore vendored
View file

@ -2,4 +2,5 @@
node_modules
npm-debug.log
.cache
test.html
test.html
.local