jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #370 from openreplay/dev

Browse source 
(fix) Optimized few queries
This commit is contained in:
Mehdi Osman 2022-03-09 18:43:17 +01:00 • committed by GitHub
commit 06d4733b35
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 20 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
api/chalicelib/core/events.py
View file

@ -106,6 +106,7 @@ def __pg_errors_query(source=None):
WHERE
s.project_id = %(project_id)s
AND lg.message ILIKE %(svalue)s
AND lg.project_id = %(project_id)s
{"AND source = %(source)s" if source is not None else ""}
LIMIT 5)
UNION ALL
@ -117,6 +118,7 @@ def __pg_errors_query(source=None):
WHERE
s.project_id = %(project_id)s
AND lg.name ILIKE %(svalue)s
AND lg.project_id = %(project_id)s
{"AND source = %(source)s" if source is not None else ""}
LIMIT 5)
UNION
@ -128,6 +130,7 @@ def __pg_errors_query(source=None):
WHERE
s.project_id = %(project_id)s
AND lg.message ILIKE %(value)s
AND lg.project_id = %(project_id)s
{"AND source = %(source)s" if source is not None else ""}
LIMIT 5)
UNION ALL
@ -139,6 +142,7 @@ def __pg_errors_query(source=None):
WHERE
s.project_id = %(project_id)s
AND lg.name ILIKE %(value)s
AND lg.project_id = %(project_id)s
{"AND source = %(source)s" if source is not None else ""}
LIMIT 5));"""

22
api/chalicelib/core/sessions.py
View file

@ -172,6 +172,13 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f
error_status="ALL", count_only=False, issue=None):
full_args, query_part, sort = search_query_parts(data, error_status, errors_only, favorite_only, issue, project_id,
user_id)
if data.limit is not None and data.page is not None:
full_args["sessions_limit_s"] = (data.page - 1) * data.limit
full_args["sessions_limit_e"] = data.page * data.limit
else:
full_args["sessions_limit_s"] = 1
full_args["sessions_limit_e"] = 200
meta_keys = []
with pg_client.PostgresClient() as cur:
if errors_only:
@ -192,7 +199,9 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f
{query_part};""", full_args)
elif data.group_by_user:
meta_keys = metadata.get(project_id=project_id)
main_query = cur.mogrify(f"""SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS sessions
main_query = cur.mogrify(f"""SELECT COUNT(*) AS count,
COALESCE(JSONB_AGG(users_sessions)
FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions
FROM (SELECT user_id,
count(full_sessions) AS user_sessions_count,
jsonb_agg(full_sessions) FILTER (WHERE rn <= 1) AS last_session,
@ -209,7 +218,9 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f
full_args)
else:
meta_keys = metadata.get(project_id=project_id)
main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, COALESCE(JSONB_AGG(full_sessions) FILTER (WHERE rn <= 200), '[]'::JSONB) AS sessions
main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count,
COALESCE(JSONB_AGG(full_sessions)
FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions
FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY favorite DESC, issue_score DESC, session_id desc, start_ts desc) AS rn
FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS}
{"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])}
@ -218,13 +229,6 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f
ORDER BY favorite DESC, issue_score DESC, {sort} {data.order}) AS full_sessions;""",
full_args)
# main_query = cur.mogrify(f"""SELECT * FROM
# (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS}
# {query_part}
# ORDER BY s.session_id desc) AS filtred_sessions
# ORDER BY favorite DESC, issue_score DESC, {sort} {order};""",
# full_args)
# print("--------------------")
# print(main_query)
# print("--------------------")

2
api/schemas.py
View file

@ -612,6 +612,8 @@ class SessionsSearchPayloadSchema(BaseModel):
order: str = Field(default="DESC")
events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then)
group_by_user: bool = Field(default=False)
limit: int = Field(default=20, gt=0, le=200)
page: int = Field(default=1, gt=0)
class Config:
alias_generator = attribute_to_camel_case

2
backend/pkg/db/postgres/messages_web.go
View file

@ -219,7 +219,7 @@ func (conn *Conn) InsertWebFetchEvent(sessionID uint64, savePayload bool, e *Fet
) VALUES (
$1, $2,
$3, $4, $5, $6,
$7, $8, $9::smallint, NULLIF($10, '')::events_common.http_method
$7, $8, $9::smallint, NULLIF($10, '')::http_method
) ON CONFLICT DO NOTHING`,
sessionID, e.Timestamp,
getSqIdx(e.MessageID), e.URL, e.Duration, e.Status < 400,