diff --git a/.gitignore b/.gitignore index b92743393..ee79ca544 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ public node_modules *DS_Store *.env +.idea \ No newline at end of file diff --git a/api/.chalice/config.json b/api/.chalice/config.json new file mode 100644 index 000000000..d6c3aa2f1 --- /dev/null +++ b/api/.chalice/config.json @@ -0,0 +1,72 @@ +{ + "version": "2.0", + "app_name": "parrot", + "environment_variables": { + "version": "0.0.0" + }, + "stages": { + "default-foss": { + "api_gateway_stage": "default-fos", + "manage_iam_role": false, + "iam_role_arn": "", + "autogen_policy": true, + "environment_variables": { + "isFOS": "true", + "isEE": "false", + "stage": "default-fos", + "jwt_issuer": "asayer-default-fos", + "allowCron": "true", + "sentry": "false", + "sentryURL": "", + "pg_host": "", + "pg_port": "5432", + "pg_dbname": "", + "pg_user": "", + "pg_password": "", + "alert_ntf": "http://127.0.0.1:8000/async/alerts/notifications/%s", + "email_signup": "http://127.0.0.1:8000/async/email_signup/%s", + "email_funnel": "http://127.0.0.1:8000/async/funnel/%s", + "email_basic": "http://127.0.0.1:8000/async/basic/%s", + "assign_link": "http://127.0.0.1:8000/async/email_assignment", + "verification_link": "http://127.0.0.1:8000/email/validate/", + "verification_success_redirect": "", + "verification_fail_redirect": "", + "captcha_server": "", + "captcha_key": "", + "jira_api": "http://127.0.0.1:3000/dev", + "github_api": "http://127.0.0.1:4000/dev", + "sessions_bucket": "", + "sessions_region": "", + "stack_reader": "", + "put_S3_TTL": "20", + "sourcemaps_bucket": "", + "sourcemaps_bucket_key": "", + "sourcemaps_bucket_secret": "", + "sourcemaps_bucket_region": "", + "js_cache_bucket": "", + "web_mobs": "https://mobs-staging.asayer.io", + "async_Token": "", + "EMAIL_HOST": "", + "EMAIL_PORT": "587", + "EMAIL_USER": "", + "EMAIL_PASSWORD": "", + "EMAIL_USE_TLS": "true", + "EMAIL_USE_SSL": "false", + "EMAIL_SSL_KEY": "", + "EMAIL_SSL_CERT": "", + "EMAIL_FROM": "OpenReplay", + "SITE_URL": "", + "announcement_bucket": "", + "jwt_secret": "", + "jwt_algorithm": "HS512", + "jwt_exp_delta_seconds": "2592000" + }, + "lambda_timeout": 150, + "lambda_memory_size": 400, + "subnet_ids": [ + ], + "security_group_ids": [ + ] + } + } +} diff --git a/api/.gitignore b/api/.gitignore new file mode 100644 index 000000000..d9688e343 --- /dev/null +++ b/api/.gitignore @@ -0,0 +1,176 @@ +# Created by .ignore support plugin (hsz.mobi) +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties +### Example user template template +### Example user template + +# IntelliJ project files +.idea +*.iml +out +gen### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +.static_storage/ +.media/ +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +/.chalice/deployments/ + +vendor + + +logs*.txt +*.csv + +*.p +*.js +SUBNETS.json + +./chalicelib/.configs diff --git a/api/Dockerfile b/api/Dockerfile new file mode 100644 index 000000000..4d25a923b --- /dev/null +++ b/api/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.6-slim +LABEL Maintainer="Rajesh Rajendran" +WORKDIR /work +COPY . . +RUN pip install -r requirements.txt -t ./vendor --upgrade +RUN pip install chalice==1.22.2 + +# Add Tini +# Startup daemon +ENV TINI_VERSION v0.19.0 +ARG envarg +ENV ENTERPRISE_BUILD ${envarg} +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] +CMD chalice local --no-autoreload --host 0.0.0.0 --stage ${ENTERPRISE_BUILD} \ No newline at end of file diff --git a/api/app.py b/api/app.py new file mode 100644 index 000000000..9edf3453b --- /dev/null +++ b/api/app.py @@ -0,0 +1,96 @@ +import sentry_sdk +from chalice import Chalice, Response +from sentry_sdk import configure_scope + +from chalicelib import _overrides +from chalicelib.blueprints import bp_authorizers +from chalicelib.blueprints import bp_core, bp_core_crons +from chalicelib.blueprints import bp_core_dynamic, bp_core_dynamic_crons +from chalicelib.blueprints.subs import bp_dashboard +from chalicelib.utils import helper +from chalicelib.utils import pg_client +from chalicelib.utils.helper import environ + +app = Chalice(app_name='parrot') +app.debug = not helper.is_production() or helper.is_local() + +sentry_sdk.init(environ["sentryURL"]) + +# Monkey-patch print for DataDog hack +import sys +import traceback + +old_tb = traceback.print_exception +old_f = sys.stdout +old_e = sys.stderr +ASAYER_SESSION_ID = None + + +class F: + def write(self, x): + if ASAYER_SESSION_ID is not None and x != '\n' and not helper.is_local(): + old_f.write(f"[asayer_session_id={ASAYER_SESSION_ID}] {x}") + else: + old_f.write(x) + + def flush(self): + pass + + +def tb_print_exception(etype, value, tb, limit=None, file=None, chain=True): + if ASAYER_SESSION_ID is not None and not helper.is_local(): + # bugsnag.notify(Exception(str(value)), meta_data={"special_info": {"asayerSessionId": ASAYER_SESSION_ID}}) + value = type(value)(f"[asayer_session_id={ASAYER_SESSION_ID}] " + str(value)) + + old_tb(etype, value, tb, limit, file, chain) + + +if helper.is_production(): + traceback.print_exception = tb_print_exception + +sys.stdout = F() +sys.stderr = F() +# ---End Monkey-patch + + +_overrides.chalice_app(app) + +# v0505 +@app.middleware('http') +def asayer_middleware(event, get_response): + global ASAYER_SESSION_ID + ASAYER_SESSION_ID = app.current_request.headers.get('vnd.asayer.io.sid') + if "authorizer" in event.context and event.context["authorizer"] is None: + print("Deleted user!!") + pg_client.close() + return Response(body={"errors": ["Deleted user"]}, status_code=403) + + try: + if helper.TRACK_TIME: + import time + now = int(time.time() * 1000) + response = get_response(event) + if helper.TRACK_TIME: + print(f"Execution time: {int(time.time() * 1000) - now} ms") + except Exception as e: + print("middleware exception handling") + print(e) + pg_client.close() + if helper.allow_sentry() and ASAYER_SESSION_ID is not None and not helper.is_local(): + with configure_scope() as scope: + scope.set_tag('stage', environ["stage"]) + scope.set_tag('openReplaySessionToken', ASAYER_SESSION_ID) + scope.set_extra("context", event.context) + sentry_sdk.capture_exception(e) + raise e + pg_client.close() + return response + + +# Open source +app.register_blueprint(bp_authorizers.app) +app.register_blueprint(bp_core.app) +app.register_blueprint(bp_core_crons.app) +app.register_blueprint(bp_core_dynamic.app) +app.register_blueprint(bp_core_dynamic_crons.app) +app.register_blueprint(bp_dashboard.app) diff --git a/api/build.sh b/api/build.sh new file mode 100644 index 000000000..a3f129d0e --- /dev/null +++ b/api/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Script to build api module +# flags to accept: +# envarg: build for enterprize edition. +# Default will be OSS build. + +# Usage: bash build.sh + +git_sha1=$(git rev-parse HEAD) +envarg="default-foss" +check_prereq() { + which docker || { + echo "Docker not installed, please install docker." + exit=1 + } + [[ exit -eq 1 ]] && exit 1 +} + +function build_api(){ + # Copy enterprize code + [[ $1 == "ee" ]] && { + cp -rf ../ee/api/* ./ + cp -rf ../ee/api/.chalice/* ./.chalice/ + envarg="default-ee" + } + docker build -f ./Dockerfile --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/chalice:${git_sha1} . +} + +check_prereq +build_api $1 diff --git a/api/chalicelib/__init__.py b/api/chalicelib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/chalicelib/_overrides.py b/api/chalicelib/_overrides.py new file mode 100644 index 000000000..5033b0324 --- /dev/null +++ b/api/chalicelib/_overrides.py @@ -0,0 +1,104 @@ +from chalice import Chalice, CORSConfig +from chalicelib.blueprints import bp_authorizers +from chalicelib.core import authorizers + +import sched +import threading +import time +from datetime import datetime +import pytz +from croniter import croniter + +base_time = datetime.now(pytz.utc) + +cors_config = CORSConfig( + allow_origin='*', + allow_headers=['vnd.asayer.io.sid'], + # max_age=600, + # expose_headers=['X-Special-Header'], + allow_credentials=True +) + + +def chalice_app(app): + def app_route(self, path, **kwargs): + kwargs.setdefault('cors', cors_config) + kwargs.setdefault('authorizer', bp_authorizers.jwt_authorizer) + handler_type = 'route' + name = kwargs.pop('name', None) + registration_kwargs = {'path': path, 'kwargs': kwargs, 'authorizer': kwargs.get("authorizer")} + + def _register_handler(user_handler): + handler_name = name + if handler_name is None: + handler_name = user_handler.__name__ + if registration_kwargs is not None: + kwargs = registration_kwargs + else: + kwargs = {} + + if kwargs['authorizer'] == bp_authorizers.jwt_authorizer \ + or kwargs['authorizer'] == bp_authorizers.api_key_authorizer: + def _user_handler(context=None, **args): + if context is not None: + args['context'] = context + else: + authorizer_context = app.current_request.context['authorizer'] + if kwargs['authorizer'] == bp_authorizers.jwt_authorizer: + args['context'] = authorizers.jwt_context(authorizer_context) + else: + args['context'] = authorizer_context + return user_handler(**args) + + wrapped = self._wrap_handler(handler_type, handler_name, _user_handler) + self._register_handler(handler_type, handler_name, _user_handler, wrapped, kwargs) + else: + wrapped = self._wrap_handler(handler_type, handler_name, user_handler) + self._register_handler(handler_type, handler_name, user_handler, wrapped, kwargs) + return wrapped + + return _register_handler + + app.route = app_route.__get__(app, Chalice) + + def app_schedule(self, expression, name=None, description=''): + handler_type = 'schedule' + registration_kwargs = {'expression': expression, + 'description': description} + + def _register_handler(user_handler): + handler_name = name + if handler_name is None: + handler_name = user_handler.__name__ + kwargs = registration_kwargs + cron_expression = kwargs["expression"].to_string()[len("cron("):-1] + if len(cron_expression.split(" ")) > 5: + cron_expression = " ".join(cron_expression.split(" ")[:-1]) + cron_expression = cron_expression.replace("?", "*") + cron_shell(user_handler, cron_expression) + + wrapped = self._wrap_handler(handler_type, handler_name, user_handler) + self._register_handler(handler_type, handler_name, user_handler, wrapped, kwargs) + return wrapped + + return _register_handler + + app.schedule = app_schedule.__get__(app, Chalice) + + def spawn(function, args): + th = threading.Thread(target=function, kwargs=args) + th.setDaemon(True) + th.start() + + def cron_shell(function, cron_expression): + def to_start(): + scheduler = sched.scheduler(time.time, time.sleep) + citer = croniter(cron_expression, base_time) + while True: + next_execution = citer.get_next(datetime) + print(f"{function.__name__} next execution: {next_execution}") + scheduler.enterabs(next_execution.timestamp(), 1, function, argument=(None,)) + scheduler.run() + print(f"{function.__name__} executed: {next_execution}") + + spawn(to_start, None) diff --git a/api/chalicelib/blueprints/__init__.py b/api/chalicelib/blueprints/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/chalicelib/blueprints/bp_authorizers.py b/api/chalicelib/blueprints/bp_authorizers.py new file mode 100644 index 000000000..414394e5a --- /dev/null +++ b/api/chalicelib/blueprints/bp_authorizers.py @@ -0,0 +1,37 @@ +from chalice import Blueprint, AuthResponse +from chalicelib.core import authorizers + + +from chalicelib.core import users +app = Blueprint(__name__) + + +@app.authorizer() +def api_key_authorizer(auth_request): + r = authorizers.api_key_authorizer(auth_request.token) + if r is None: + return AuthResponse(routes=[], principal_id=None) + + return AuthResponse( + routes=['*'], + principal_id=r['tenantId'], + context=r + ) + + +@app.authorizer(ttl_seconds=60) +def jwt_authorizer(auth_request): + print("---- Auth") + jwt_payload = authorizers.jwt_authorizer(auth_request.token) + print(jwt_payload) + if jwt_payload is None \ + or jwt_payload.get("iat") is None or jwt_payload.get("aud") is None \ + or not users.auth_exists(user_id=jwt_payload["userId"], tenant_id=jwt_payload["tenantId"], + jwt_iat=jwt_payload["iat"], jwt_aud=jwt_payload["aud"]): + return AuthResponse(routes=[], principal_id=None) + + return AuthResponse( + routes=['*'], + principal_id=jwt_payload['userId'], + context=jwt_payload + ) diff --git a/api/chalicelib/blueprints/bp_core.py b/api/chalicelib/blueprints/bp_core.py new file mode 100644 index 000000000..f2ca10746 --- /dev/null +++ b/api/chalicelib/blueprints/bp_core.py @@ -0,0 +1,879 @@ +from chalicelib.utils.helper import environ + +from chalice import Blueprint +from chalice import Response + +from chalicelib import _overrides +from chalicelib.blueprints import bp_authorizers +from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assignments, projects, \ + sessions_metas, alerts, funnels, issues, integrations_manager, errors_favorite_viewed, metadata, \ + log_tool_elasticsearch, log_tool_datadog, \ + log_tool_stackdriver, reset_password, sessions_favorite_viewed, \ + log_tool_cloudwatch, log_tool_sentry, log_tool_sumologic, log_tools, errors, sessions, \ + log_tool_newrelic, announcements, log_tool_bugsnag, weekly_report, integration_jira_cloud, integration_github +from chalicelib.core.collaboration_slack import Slack +from chalicelib.utils import email_helper + +app = Blueprint(__name__) +_overrides.chalice_app(app) + + +@app.route('/{projectId}/sessions2/favorite', methods=['GET']) +def get_favorite_sessions2(projectId, context): + params = app.current_request.query_params + + return { + 'data': sessions.get_favorite_sessions(project_id=projectId, user_id=context["userId"], include_viewed=True) + } + + +@app.route('/{projectId}/sessions2/{sessionId}', methods=['GET']) +def get_session2(projectId, sessionId, context): + data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, user_id=context["userId"], + include_fav_viewed=True, group_metadata=True) + if data is not None: + sessions_favorite_viewed.view_session(project_id=projectId, user_id=context['userId'], session_id=sessionId) + return { + 'data': data + } + + +@app.route('/{projectId}/sessions2/{sessionId}/favorite', methods=['GET']) +def add_remove_favorite_session2(projectId, sessionId, context): + return { + "data": sessions_favorite_viewed.favorite_session(project_id=projectId, user_id=context['userId'], + session_id=sessionId)} + + +@app.route('/{projectId}/sessions2/{sessionId}/assign', methods=['GET']) +def assign_session(projectId, sessionId, context): + data = sessions_assignments.get_by_session(project_id=projectId, session_id=sessionId, + tenant_id=context['tenantId'], + user_id=context["userId"]) + if "errors" in data: + return data + return { + 'data': data + } + + +@app.route('/{projectId}/sessions2/{sessionId}/errors/{errorId}/sourcemaps', methods=['GET']) +def get_error_trace(projectId, sessionId, errorId, context): + data = errors.get_trace(project_id=projectId, error_id=errorId) + if "errors" in data: + return data + return { + 'data': data + } + + +@app.route('/{projectId}/sessions2/{sessionId}/assign/{issueId}', methods=['GET']) +def assign_session(projectId, sessionId, issueId, context): + data = sessions_assignments.get(project_id=projectId, session_id=sessionId, assignment_id=issueId, + tenant_id=context['tenantId'], user_id=context["userId"]) + if "errors" in data: + return data + return { + 'data': data + } + + +@app.route('/{projectId}/sessions2/{sessionId}/assign/{issueId}/comment', methods=['POST', 'PUT']) +def comment_assignment(projectId, sessionId, issueId, context): + data = app.current_request.json_body + data = sessions_assignments.comment(tenant_id=context['tenantId'], project_id=projectId, + session_id=sessionId, assignment_id=issueId, + user_id=context["userId"], message=data["message"]) + if "errors" in data.keys(): + return data + return { + 'data': data + } + + +@app.route('/{projectId}/events/search', methods=['GET']) +def events_search(projectId, context): + params = app.current_request.query_params + if params is None: + return {"data": []} + + q = params.get('q', '') + if len(q) == 0: + return {"data": []} + result = events.search_pg2(q, params.get('type', ''), project_id=projectId, source=params.get('source'), + key=params.get("key")) + return result + + +@app.route('/{projectId}/sessions/search2', methods=['POST']) +def sessions_search2(projectId, context): + data = app.current_request.json_body + + data = sessions.search2_pg(data, projectId, user_id=context["userId"]) + return {'data': data} + + +@app.route('/{projectId}/sessions/filters', methods=['GET']) +def session_filter_values(projectId, context): + return {'data': sessions_metas.get_key_values(projectId)} + + +@app.route('/{projectId}/sessions/filters/top', methods=['GET']) +def session_top_filter_values(projectId, context): + return {'data': sessions_metas.get_top_key_values(projectId)} + + +@app.route('/{projectId}/sessions/filters/search', methods=['GET']) +def get_session_filters_meta(projectId, context): + params = app.current_request.query_params + if params is None: + return {"data": []} + + meta_type = params.get('type', '') + if len(meta_type) == 0: + return {"data": []} + q = params.get('q', '') + if len(q) == 0: + return {"data": []} + return sessions_metas.search(project_id=projectId, meta_type=meta_type, text=q) + + +@app.route('/{projectId}/integrations/{integration}/notify/{integrationId}/{source}/{sourceId}', + methods=['POST', 'PUT']) +def integration_notify(projectId, integration, integrationId, source, sourceId, context): + data = app.current_request.json_body + comment = None + if "comment" in data: + comment = data["comment"] + if integration == "slack": + args = {"tenant_id": context["tenantId"], + "user": context['email'], "comment": comment, "project_id": projectId, + "integration_id": integrationId} + if source == "sessions": + return Slack.share_session(session_id=sourceId, **args) + elif source == "errors": + return Slack.share_error(error_id=sourceId, **args) + return {"data": None} + + +@app.route('/integrations/sentry', methods=['GET']) +def get_all_sentry(context): + return {"data": log_tool_sentry.get_all(tenant_id=context["tenantId"])} + + +@app.route('/{projectId}/integrations/sentry', methods=['GET']) +def get_sentry(projectId, context): + return {"data": log_tool_sentry.get(project_id=projectId)} + + +@app.route('/{projectId}/integrations/sentry', methods=['POST', 'PUT']) +def add_edit_sentry(projectId, context): + data = app.current_request.json_body + + return {"data": log_tool_sentry.add_edit(tenant_id=context["tenantId"], project_id=projectId, data=data)} + + +@app.route('/{projectId}/integrations/sentry', methods=['DELETE']) +def delete_sentry(projectId, context): + return {"data": log_tool_sentry.delete(tenant_id=context["tenantId"], project_id=projectId)} + + +@app.route('/{projectId}/integrations/sentry/events/{eventId}', methods=['GET']) +def proxy_sentry(projectId, eventId, context): + return {"data": log_tool_sentry.proxy_get(tenant_id=context["tenantId"], project_id=projectId, event_id=eventId)} + + +@app.route('/integrations/datadog', methods=['GET']) +def get_all_datadog(context): + return {"data": log_tool_datadog.get_all(tenant_id=context["tenantId"])} + + +@app.route('/{projectId}/integrations/datadog', methods=['GET']) +def get_datadog(projectId, context): + return {"data": log_tool_datadog.get(project_id=projectId)} + + +@app.route('/{projectId}/integrations/datadog', methods=['POST', 'PUT']) +def add_edit_datadog(projectId, context): + data = app.current_request.json_body + + return {"data": log_tool_datadog.add_edit(tenant_id=context["tenantId"], project_id=projectId, data=data)} + + +@app.route('/{projectId}/integrations/datadog', methods=['DELETE']) +def delete_datadog(projectId, context): + return {"data": log_tool_datadog.delete(tenant_id=context["tenantId"], project_id=projectId)} + + +@app.route('/integrations/stackdriver', methods=['GET']) +def get_all_stackdriver(context): + return {"data": log_tool_stackdriver.get_all(tenant_id=context["tenantId"])} + + +@app.route('/{projectId}/integrations/stackdriver', methods=['GET']) +def get_stackdriver(projectId, context): + return {"data": log_tool_stackdriver.get(project_id=projectId)} + + +@app.route('/{projectId}/integrations/stackdriver', methods=['POST', 'PUT']) +def add_edit_stackdriver(projectId, context): + data = app.current_request.json_body + + return {"data": log_tool_stackdriver.add_edit(tenant_id=context["tenantId"], project_id=projectId, data=data)} + + +@app.route('/{projectId}/integrations/stackdriver', methods=['DELETE']) +def delete_stackdriver(projectId, context): + return {"data": log_tool_stackdriver.delete(tenant_id=context["tenantId"], project_id=projectId)} + + +@app.route('/integrations/newrelic', methods=['GET']) +def get_all_newrelic(context): + return {"data": log_tool_newrelic.get_all(tenant_id=context["tenantId"])} + + +@app.route('/{projectId}/integrations/newrelic', methods=['GET']) +def get_newrelic(projectId, context): + return {"data": log_tool_newrelic.get(project_id=projectId)} + + +@app.route('/{projectId}/integrations/newrelic', methods=['POST', 'PUT']) +def add_edit_newrelic(projectId, context): + data = app.current_request.json_body + + return {"data": log_tool_newrelic.add_edit(tenant_id=context["tenantId"], project_id=projectId, data=data)} + + +@app.route('/{projectId}/integrations/newrelic', methods=['DELETE']) +def delete_newrelic(projectId, context): + return {"data": log_tool_newrelic.delete(tenant_id=context["tenantId"], project_id=projectId)} + + +@app.route('/integrations/rollbar', methods=['GET']) +def get_all_rollbar(context): + return {"data": log_tool_rollbar.get_all(tenant_id=context["tenantId"])} + + +@app.route('/{projectId}/integrations/rollbar', methods=['GET']) +def get_rollbar(projectId, context): + return {"data": log_tool_rollbar.get(project_id=projectId)} + + +@app.route('/{projectId}/integrations/rollbar', methods=['POST', 'PUT']) +def add_edit_rollbar(projectId, context): + data = app.current_request.json_body + + return {"data": log_tool_rollbar.add_edit(tenant_id=context["tenantId"], project_id=projectId, data=data)} + + +@app.route('/{projectId}/integrations/rollbar', methods=['DELETE']) +def delete_datadog(projectId, context): + return {"data": log_tool_rollbar.delete(tenant_id=context["tenantId"], project_id=projectId)} + + +@app.route('/integrations/bugsnag/list_projects', methods=['POST']) +def list_projects_bugsnag(context): + data = app.current_request.json_body + return {"data": log_tool_bugsnag.list_projects(auth_token=data["authorizationToken"])} + + +@app.route('/integrations/bugsnag', methods=['GET']) +def get_all_bugsnag(context): + return {"data": log_tool_bugsnag.get_all(tenant_id=context["tenantId"])} + + +@app.route('/{projectId}/integrations/bugsnag', methods=['GET']) +def get_bugsnag(projectId, context): + return {"data": log_tool_bugsnag.get(project_id=projectId)} + + +@app.route('/{projectId}/integrations/bugsnag', methods=['POST', 'PUT']) +def add_edit_bugsnag(projectId, context): + data = app.current_request.json_body + + return {"data": log_tool_bugsnag.add_edit(tenant_id=context["tenantId"], project_id=projectId, data=data)} + + +@app.route('/{projectId}/integrations/bugsnag', methods=['DELETE']) +def delete_bugsnag(projectId, context): + return {"data": log_tool_bugsnag.delete(tenant_id=context["tenantId"], project_id=projectId)} + + +@app.route('/integrations/cloudwatch/list_groups', methods=['POST']) +def list_groups_cloudwatch(context): + data = app.current_request.json_body + return {"data": log_tool_cloudwatch.list_log_groups(aws_access_key_id=data["awsAccessKeyId"], + aws_secret_access_key=data["awsSecretAccessKey"], + region=data["region"])} + + +@app.route('/integrations/cloudwatch', methods=['GET']) +def get_all_cloudwatch(context): + return {"data": log_tool_cloudwatch.get_all(tenant_id=context["tenantId"])} + + +@app.route('/{projectId}/integrations/cloudwatch', methods=['GET']) +def get_cloudwatch(projectId, context): + return {"data": log_tool_cloudwatch.get(project_id=projectId)} + + +@app.route('/{projectId}/integrations/cloudwatch', methods=['POST', 'PUT']) +def add_edit_cloudwatch(projectId, context): + data = app.current_request.json_body + + return {"data": log_tool_cloudwatch.add_edit(tenant_id=context["tenantId"], project_id=projectId, data=data)} + + +@app.route('/{projectId}/integrations/cloudwatch', methods=['DELETE']) +def delete_cloudwatch(projectId, context): + return {"data": log_tool_cloudwatch.delete(tenant_id=context["tenantId"], project_id=projectId)} + + +@app.route('/integrations/elasticsearch', methods=['GET']) +def get_all_elasticsearch(context): + return {"data": log_tool_elasticsearch.get_all(tenant_id=context["tenantId"])} + + +@app.route('/{projectId}/integrations/elasticsearch', methods=['GET']) +def get_elasticsearch(projectId, context): + return {"data": log_tool_elasticsearch.get(project_id=projectId)} + + +@app.route('/integrations/elasticsearch/test', methods=['POST']) +def test_elasticsearch_connection(context): + data = app.current_request.json_body + return {"data": log_tool_elasticsearch.ping(tenant_id=context["tenantId"], **data)} + + +@app.route('/{projectId}/integrations/elasticsearch', methods=['POST', 'PUT']) +def add_edit_elasticsearch(projectId, context): + data = app.current_request.json_body + + return {"data": log_tool_elasticsearch.add_edit(tenant_id=context["tenantId"], project_id=projectId, data=data)} + + +@app.route('/{projectId}/integrations/elasticsearch', methods=['DELETE']) +def delete_elasticsearch(projectId, context): + return {"data": log_tool_elasticsearch.delete(tenant_id=context["tenantId"], project_id=projectId)} + + +@app.route('/integrations/sumologic', methods=['GET']) +def get_all_sumologic(context): + return {"data": log_tool_sumologic.get_all(tenant_id=context["tenantId"])} + + +@app.route('/{projectId}/integrations/sumologic', methods=['GET']) +def get_sumologic(projectId, context): + return {"data": log_tool_sumologic.get(project_id=projectId)} + + +@app.route('/{projectId}/integrations/sumologic', methods=['POST', 'PUT']) +def add_edit_sumologic(projectId, context): + data = app.current_request.json_body + + return {"data": log_tool_sumologic.add_edit(tenant_id=context["tenantId"], project_id=projectId, data=data)} + + +@app.route('/{projectId}/integrations/sumologic', methods=['DELETE']) +def delete_sumologic(projectId, context): + return {"data": log_tool_sumologic.delete(tenant_id=context["tenantId"], project_id=projectId)} + + +@app.route('/integrations/issues', methods=['GET']) +def get_integration_status(context): + error, integration = integrations_manager.get_integration(tenant_id=context["tenantId"], + user_id=context["userId"]) + if error is not None: + return {"data": {}} + return {"data": integration.get_obfuscated()} + + +@app.route('/integrations/jira', methods=['POST', 'PUT']) +def add_edit_jira_cloud(context): + data = app.current_request.json_body + error, integration = integrations_manager.get_integration(tool=integration_jira_cloud.PROVIDER, + tenant_id=context["tenantId"], + user_id=context["userId"]) + if error is not None: + return error + return {"data": integration.add_edit(data=data)} + + +@app.route('/integrations/github', methods=['POST', 'PUT']) +def add_edit_github(context): + data = app.current_request.json_body + error, integration = integrations_manager.get_integration(tool=integration_github.PROVIDER, + tenant_id=context["tenantId"], + user_id=context["userId"]) + if error is not None: + return error + return {"data": integration.add_edit(data=data)} + + +@app.route('/integrations/issues', methods=['DELETE']) +def delete_default_issue_tracking_tool(context): + error, integration = integrations_manager.get_integration(tenant_id=context["tenantId"], + user_id=context["userId"]) + if error is not None: + return error + return {"data": integration.delete()} + + +@app.route('/integrations/jira', methods=['DELETE']) +def delete_jira_cloud(context): + error, integration = integrations_manager.get_integration(tool=integration_jira_cloud.PROVIDER, + tenant_id=context["tenantId"], + user_id=context["userId"]) + if error is not None: + return error + return {"data": integration.delete()} + + +@app.route('/integrations/github', methods=['DELETE']) +def delete_github(context): + error, integration = integrations_manager.get_integration(tool=integration_github.PROVIDER, + tenant_id=context["tenantId"], + user_id=context["userId"]) + if error is not None: + return error + return {"data": integration.delete()} + + +@app.route('/integrations/issues/list_projects', methods=['GET']) +def get_all_issue_tracking_projects(context): + error, integration = integrations_manager.get_integration(tenant_id=context["tenantId"], + user_id=context["userId"]) + if error is not None: + return error + data = integration.issue_handler.get_projects() + if "errors" in data: + return data + return {"data": data} + + +@app.route('/integrations/issues/{integrationProjectId}', methods=['GET']) +def get_integration_metadata(integrationProjectId, context): + error, integration = integrations_manager.get_integration(tenant_id=context["tenantId"], + user_id=context["userId"]) + if error is not None: + return error + data = integration.issue_handler.get_metas(integrationProjectId) + if "errors" in data.keys(): + return data + return {"data": data} + + +@app.route('/{projectId}/assignments', methods=['GET']) +def get_all_assignments(projectId, context): + data = sessions_assignments.get_all(project_id=projectId, user_id=context["userId"]) + return { + 'data': data + } + + +@app.route('/{projectId}/sessions2/{sessionId}/assign/projects/{integrationProjectId}', methods=['POST', 'PUT']) +def create_issue_assignment(projectId, sessionId, integrationProjectId, context): + data = app.current_request.json_body + data = sessions_assignments.create_new_assignment(tenant_id=context['tenantId'], project_id=projectId, + session_id=sessionId, + creator_id=context["userId"], assignee=data["assignee"], + description=data["description"], title=data["title"], + issue_type=data["issueType"], + integration_project_id=integrationProjectId) + if "errors" in data.keys(): + return data + return { + 'data': data + } + + +@app.route('/{projectId}/gdpr', methods=['GET']) +def get_gdpr(projectId, context): + return {"data": projects.get_gdpr(project_id=projectId)} + + +@app.route('/{projectId}/gdpr', methods=['POST', 'PUT']) +def edit_gdpr(projectId, context): + data = app.current_request.json_body + + return {"data": projects.edit_gdpr(project_id=projectId, gdpr=data)} + + +@app.route('/password/reset/{step}', methods=['PUT', 'POST'], authorizer=None) +def reset_password_handler(step): + data = app.current_request.json_body + if step == "1": + if "email" not in data or len(data["email"]) < 5: + return {"errors": ["please provide a valid email address"]} + return reset_password.step1(data) + elif step == "2": + return reset_password.step2(data) + + +@app.route('/{projectId}/metadata', methods=['GET']) +def get_metadata(projectId, context): + return {"data": metadata.get(project_id=projectId)} + + +@app.route('/{projectId}/metadata/list', methods=['POST', 'PUT']) +def add_edit_delete_metadata(projectId, context): + data = app.current_request.json_body + + return metadata.add_edit_delete(tenant_id=context["tenantId"], project_id=projectId, new_metas=data["list"]) + + +@app.route('/{projectId}/metadata', methods=['POST', 'PUT']) +def add_metadata(projectId, context): + data = app.current_request.json_body + + return metadata.add(tenant_id=context["tenantId"], project_id=projectId, new_name=data["key"]) + + +@app.route('/{projectId}/metadata/{index}', methods=['POST', 'PUT']) +def edit_metadata(projectId, index, context): + data = app.current_request.json_body + + return metadata.edit(tenant_id=context["tenantId"], project_id=projectId, index=int(index), + new_name=data["key"]) + + +@app.route('/{projectId}/metadata/{index}', methods=['DELETE']) +def delete_metadata(projectId, index, context): + return metadata.delete(tenant_id=context["tenantId"], project_id=projectId, index=index) + + +@app.route('/{projectId}/metadata/search', methods=['GET']) +def search_metadata(projectId, context): + params = app.current_request.query_params + q = params.get('q', '') + key = params.get('key', '') + if len(q) == 0 and len(key) == 0: + return {"data": []} + if len(q) == 0: + return {"errors": ["please provide a value for search"]} + if len(key) == 0: + return {"errors": ["please provide a key for search"]} + return metadata.search(tenant_id=context["tenantId"], project_id=projectId, value=q, key=key) + + +@app.route('/{projectId}/integration/sources', methods=['GET']) +def search_integrations(projectId, context): + return log_tools.search(project_id=projectId) + + +@app.route('/async/email_assignment', methods=['POST', 'PUT'], authorizer=None) +def async_send_signup_emails(): + data = app.current_request.json_body + if data.pop("auth") != environ["async_Token"]: + return {} + email_helper.send_assign_session(recipient=data["email"], link=data["link"], message=data["message"]) + + +@app.route('/async/funnel/weekly_report2', methods=['POST', 'PUT'], authorizer=None) +def async_weekly_report(): + print("=========================> Sending weekly report") + data = app.current_request.json_body + if data.pop("auth") != environ["async_Token"]: + return {} + email_helper.weekly_report2(recipients=data["email"], data=data.get("data", None)) + + +@app.route('/async/basic/{step}', methods=['POST', 'PUT'], authorizer=None) +def async_basic_emails(step): + data = app.current_request.json_body + if data.pop("auth") != environ["async_Token"]: + return {} + if step.lower() == "member_invitation": + email_helper.send_team_invitation(recipient=data["email"], user_name=data["userName"], + temp_password=data["tempPassword"], client_id=data["clientId"], + sender_name=data["senderName"]) + + +@app.route('/{projectId}/sample_rate', methods=['GET']) +def get_capture_status(projectId, context): + return {"data": projects.get_capture_status(project_id=projectId)} + + +@app.route('/{projectId}/sample_rate', methods=['POST', 'PUT']) +def update_capture_status(projectId, context): + data = app.current_request.json_body + + return {"data": projects.update_capture_status(project_id=projectId, changes=data)} + + +@app.route('/announcements', methods=['GET']) +def get_all_announcements(context): + return {"data": announcements.get_all(context["userId"])} + + +@app.route('/announcements/view', methods=['GET']) +def get_all_announcements(context): + return {"data": announcements.view(user_id=context["userId"])} + + +@app.route('/{projectId}/errors/{errorId}/{action}', methods=['GET']) +def add_remove_favorite_error(projectId, errorId, action, context): + if action == "favorite": + return errors_favorite_viewed.favorite_error(project_id=projectId, user_id=context['userId'], error_id=errorId) + elif action == "sessions": + params = app.current_request.query_params + if params is None: + params = {} + start_date = params.get("startDate") + end_date = params.get("endDate") + return { + "data": errors.get_sessions(project_id=projectId, user_id=context['userId'], error_id=errorId, + start_date=start_date, end_date=end_date)} + elif action in list(errors.ACTION_STATE.keys()): + return errors.change_state(project_id=projectId, user_id=context['userId'], error_id=errorId, action=action) + else: + return {"errors": ["undefined action"]} + + +@app.route('/{projectId}/errors/merge', methods=['POST']) +def errors_merge(projectId, context): + data = app.current_request.json_body + + data = errors.merge(error_ids=data.get("errors", [])) + return data + + +@app.route('/show_banner', methods=['GET']) +def errors_merge(context): + return {"data": False} + + +@app.route('/{projectId}/alerts', methods=['POST', 'PUT']) +def create_alert(projectId, context): + data = app.current_request.json_body + return alerts.create(projectId, data) + + +@app.route('/{projectId}/alerts', methods=['GET']) +def get_all_alerts(projectId, context): + return {"data": alerts.get_all(projectId)} + + +@app.route('/{projectId}/alerts/{alertId}', methods=['GET']) +def get_alert(projectId, alertId, context): + return {"data": alerts.get(alertId)} + + +@app.route('/{projectId}/alerts/{alertId}', methods=['POST', 'PUT']) +def update_alert(projectId, alertId, context): + data = app.current_request.json_body + return alerts.update(alertId, data) + + +@app.route('/{projectId}/alerts/{alertId}', methods=['DELETE']) +def delete_alert(projectId, alertId, context): + return alerts.delete(projectId, alertId) + + +@app.route('/alerts/notifications', methods=['POST', 'PUT'], authorizer=None) +def send_alerts_notifications(): + data = app.current_request.json_body + if data.get("token", "") != "nF46JdQqAM5v9KI9lPMpcu8o9xiJGvNNWOGL7TJP": + return {"errors": ["missing token"]} + return {"data": alerts.process_notifications(data.get("notifications", []))} + + +@app.route('/{projectId}/funnels', methods=['POST', 'PUT']) +def add_funnel(projectId, context): + data = app.current_request.json_body + return funnels.create(project_id=projectId, + user_id=context['userId'], + name=data["name"], + filter=data["filter"], + is_public=data.get("isPublic", False)) + + +@app.route('/{projectId}/funnels', methods=['GET']) +def get_funnels(projectId, context): + params = app.current_request.query_params + if params is None: + params = {} + + return {"data": funnels.get_by_user(project_id=projectId, + user_id=context['userId'], + range_value=None, + start_date=None, + end_date=None, + details=False)} + + +@app.route('/{projectId}/funnels/details', methods=['GET']) +def get_funnels_with_details(projectId, context): + params = app.current_request.query_params + if params is None: + params = {} + + return {"data": funnels.get_by_user(project_id=projectId, + user_id=context['userId'], + range_value=params.get("rangeValue", None), + start_date=params.get('startDate', None), + end_date=params.get('endDate', None), + details=True)} + + +@app.route('/{projectId}/funnels/issue_types', methods=['GET']) +def get_possible_issue_types(projectId, context): + params = app.current_request.query_params + if params is None: + params = {} + + return {"data": funnels.get_possible_issue_types(project_id=projectId)} + + +@app.route('/{projectId}/funnels/{funnelId}/insights', methods=['GET']) +def get_funnel_insights(projectId, funnelId, context): + params = app.current_request.query_params + if params is None: + params = {} + + return {"data": funnels.get_top_insights(funnel_id=funnelId, project_id=projectId, + range_value=params.get("range_value", None), + start_date=params.get('startDate', None), + end_date=params.get('endDate', None))} + + +@app.route('/{projectId}/funnels/{funnelId}/insights', methods=['POST', 'PUT']) +def get_funnel_insights_on_the_fly(projectId, funnelId, context): + params = app.current_request.query_params + if params is None: + params = {} + data = app.current_request.json_body + if data is None: + data = {} + + return { + "data": funnels.get_top_insights_on_the_fly(funnel_id=funnelId, project_id=projectId, data={**params, **data})} + + +@app.route('/{projectId}/funnels/{funnelId}/issues', methods=['GET']) +def get_funnel_issues(projectId, funnelId, context): + params = app.current_request.query_params + if params is None: + params = {} + + return funnels.get_issues(funnel_id=funnelId, project_id=projectId, + range_value=params.get("range_value", None), + start_date=params.get('startDate', None), end_date=params.get('endDate', None)) + + +@app.route('/{projectId}/funnels/{funnelId}/issues', methods=['POST', 'PUT']) +def get_funnel_issues_on_the_fly(projectId, funnelId, context): + params = app.current_request.query_params + if params is None: + params = {} + data = app.current_request.json_body + if data is None: + data = {} + + return {"data": funnels.get_issues_on_the_fly(funnel_id=funnelId, project_id=projectId, data={**params, **data})} + + +@app.route('/{projectId}/funnels/{funnelId}/sessions', methods=['GET']) +def get_funnel_sessions(projectId, funnelId, context): + params = app.current_request.query_params + if params is None: + params = {} + + return {"data": funnels.get_sessions(funnel_id=funnelId, user_id=context['userId'], project_id=projectId, + range_value=params.get("range_value", None), + start_date=params.get('startDate', None), + end_date=params.get('endDate', None))} + + +@app.route('/{projectId}/funnels/{funnelId}/sessions', methods=['POST', 'PUT']) +def get_funnel_sessions_on_the_fly(projectId, funnelId, context): + params = app.current_request.query_params + if params is None: + params = {} + data = app.current_request.json_body + if data is None: + data = {} + return {"data": funnels.get_sessions_on_the_fly(funnel_id=funnelId, user_id=context['userId'], project_id=projectId, + data={**params, **data})} + + +@app.route('/{projectId}/funnels/issues/{issueId}/sessions', methods=['GET']) +def get_issue_sessions(projectId, issueId, context): + params = app.current_request.query_params + if params is None: + params = {} + + issue = issues.get(project_id=projectId, issue_id=issueId) + return { + "data": {"sessions": sessions.search_by_issue(user_id=context["userId"], project_id=projectId, issue=issue, + start_date=params.get('startDate', None), + end_date=params.get('endDate', None)), + "issue": issue}} + + +@app.route('/{projectId}/funnels/{funnelId}/issues/{issueId}/sessions', methods=['POST', 'PUT']) +def get_funnel_issue_sessions(projectId, funnelId, issueId, context): + data = app.current_request.json_body + + data = funnels.search_by_issue(project_id=projectId, user_id=context["userId"], issue_id=issueId, + funnel_id=funnelId, data=data) + if "errors" in data: + return data + if data.get("issue") is None: + data["issue"] = issues.get(project_id=projectId, issue_id=issueId) + return { + "data": data + } + + +@app.route('/{projectId}/funnels/{funnelId}', methods=['GET']) +def get_funnel(projectId, funnelId, context): + return {"data": funnels.get(funnel_id=funnelId, + project_id=projectId)} + + +@app.route('/{projectId}/funnels/{funnelId}', methods=['POST', 'PUT']) +def edit_funnel(projectId, funnelId, context): + data = app.current_request.json_body + return funnels.update(funnel_id=funnelId, + user_id=context['userId'], + name=data.get("name"), + filter=data.get("filter"), + is_public=data.get("isPublic")) + + +@app.route('/{projectId}/funnels/{funnelId}', methods=['DELETE']) +def delete_filter(projectId, funnelId, context): + return funnels.delete(user_id=context['userId'], funnel_id=funnelId, project_id=projectId) + + +@app.route('/{projectId}/sourcemaps', methods=['PUT'], authorizer=bp_authorizers.api_key_authorizer) +def sign_sourcemap_for_upload(projectId, context): + data = app.current_request.json_body + project_id = projects.get_internal_project_id(projectId) + if project_id is None: + return Response(status_code=400, body='invalid projectId') + + return {"data": sourcemaps.presign_upload_urls(project_id=project_id, urls=data["URL"])} + + +@app.route('/config/weekly_report', methods=['GET']) +def get_weekly_report_config(context): + return {"data": weekly_report.get_config(user_id=context['userId'])} + + +@app.route('/config/weekly_report', methods=['POST', 'PUT']) +def get_weekly_report_config(context): + data = app.current_request.json_body + return {"data": weekly_report.edit_config(user_id=context['userId'], weekly_report=data.get("weeklyReport", True))} + + +@app.route('/{projectId}/issue_types', methods=['GET']) +def issue_types(projectId, context): + return {"data": issues.get_types(project_id=projectId)} + + +@app.route('/flows', methods=['GET', 'PUT', 'POST', 'DELETE']) +@app.route('/{projectId}/flows', methods=['GET', 'PUT', 'POST', 'DELETE']) +def removed_endpoints(context): + return Response(body={"errors": ["Endpoint no longer available"]}, status_code=410) diff --git a/api/chalicelib/blueprints/bp_core_crons.py b/api/chalicelib/blueprints/bp_core_crons.py new file mode 100644 index 000000000..a64d7ce10 --- /dev/null +++ b/api/chalicelib/blueprints/bp_core_crons.py @@ -0,0 +1,18 @@ +from chalice import Blueprint +from chalice import Cron +from chalicelib import _overrides +from chalicelib.core import reset_password, weekly_report + +app = Blueprint(__name__) +_overrides.chalice_app(app) + + +@app.schedule(Cron('0/60', '*', '*', '*', '?', '*')) +def clear_password_reset(event): + reset_password.cron() + + +# Run every monday. +@app.schedule(Cron('5', '0', '?', '*', 'MON', '*')) +def weekly_report2(event): + weekly_report.cron() diff --git a/api/chalicelib/blueprints/bp_core_dynamic.py b/api/chalicelib/blueprints/bp_core_dynamic.py new file mode 100644 index 000000000..8bdd5ca30 --- /dev/null +++ b/api/chalicelib/blueprints/bp_core_dynamic.py @@ -0,0 +1,385 @@ +from chalice import Blueprint, Response + +from chalicelib import _overrides +from chalicelib.core import metadata, errors_favorite_viewed, slack, alerts, sessions, integration_github, \ + integrations_manager +from chalicelib.utils import captcha +from chalicelib.utils import helper +from chalicelib.utils.helper import environ + +from chalicelib.core import tenants +from chalicelib.core import signup +from chalicelib.core import users +from chalicelib.core import projects +from chalicelib.core import errors +from chalicelib.core import notifications +from chalicelib.core import boarding +from chalicelib.core import webhook +from chalicelib.core.collaboration_slack import Slack + +app = Blueprint(__name__) +_overrides.chalice_app(app) + + +@app.route('/signedups', methods=['GET'], authorizer=None) +def signed_ups(): + return { + 'data': tenants.get_tenants() + } + + +@app.route('/login', methods=['POST'], authorizer=None) +def login(): + data = app.current_request.json_body + if helper.allow_captcha() and not captcha.is_valid(data["g-recaptcha-response"]): + return {"errors": ["Invalid captcha."]} + r = users.authenticate(data['email'], data['password'], + for_plugin= False + ) + if r is None: + return { + 'errors': ['You’ve entered invalid Email or Password.'] + } + + tenant_id = r.pop("tenantId") + + r["limits"] = { + "teamMember": -1, + "projects": -1, + "metadata": metadata.get_remaining_metadata_with_count(tenant_id)} + + c = tenants.get_by_tenant_id(tenant_id) + c.pop("createdAt") + c["projects"] = projects.get_projects(tenant_id=tenant_id, recording_state=True, recorded=True, + stack_integrations=True) + return { + 'jwt': r.pop('jwt'), + 'data': { + "user": r, + "client": c, + } + } + + +@app.route('/account', methods=['GET']) +def get_account(context): + r = users.get(tenant_id=context['tenantId'], user_id=context['userId']) + return { + 'data': { + **r, + "limits": { + "teamMember": -1, + "projects": -1, + "metadata": metadata.get_remaining_metadata_with_count(context['tenantId'])}, + } + } + + +@app.route('/projects', methods=['GET']) +def get_projects(context): + return {"data": projects.get_projects(tenant_id=context["tenantId"], recording_state=True, gdpr=True, recorded=True, + stack_integrations=True)} + + +@app.route('/projects', methods=['POST', 'PUT']) +def create_project(context): + data = app.current_request.json_body + return projects.create(tenant_id=context["tenantId"], user_id=context["userId"], data=data) + + +@app.route('/projects/{projectId}', methods=['POST', 'PUT']) +def create_edit_project(projectId, context): + data = app.current_request.json_body + + return projects.edit(tenant_id=context["tenantId"], user_id=context["userId"], data=data, project_id=projectId) + + +@app.route('/projects/{projectId}', methods=['GET']) +def get_project(projectId, context): + return {"data": projects.get_project(tenant_id=context["tenantId"], project_id=projectId, include_last_session=True, + include_gdpr=True)} + + +@app.route('/projects/{projectId}', methods=['DELETE']) +def delete_project(projectId, context): + return projects.delete(tenant_id=context["tenantId"], user_id=context["userId"], project_id=projectId) + + +@app.route('/projects/limit', methods=['GET']) +def get_projects_limit(context): + return {"data": { + "current": projects.count_by_tenant(tenant_id=context["tenantId"]), + "remaining": -1 + }} + + +@app.route('/client', methods=['GET']) +def get_client(context): + r = tenants.get_by_tenant_id(context['tenantId']) + if r is not None: + r.pop("createdAt") + r["projects"] = projects.get_projects(tenant_id=context['tenantId'], recording_state=True, recorded=True, + stack_integrations=True) + return { + 'data': r + } + + +@app.route('/client/new_api_key', methods=['GET']) +def generate_new_tenant_token(context): + return { + 'data': tenants.generate_new_api_key(context['tenantId']) + } + + +@app.route('/client', methods=['PUT', 'POST']) +def put_client(context): + data = app.current_request.json_body + return tenants.update(tenant_id=context["tenantId"], user_id=context["userId"], data=data) + + +@app.route('/signup', methods=['GET'], authorizer=None) +def get_all_signup(): + return {"data": signup.get_signed_ups()} + + +@app.route('/signup', methods=['POST', 'PUT'], authorizer=None) +def signup_handler(): + data = app.current_request.json_body + return signup.create_step1(data) + + +@app.route('/integrations/slack', methods=['POST', 'PUT']) +def add_slack_client(context): + data = app.current_request.json_body + if "url" not in data or "name" not in data: + return {"errors": ["please provide a url and a name"]} + if Slack.add_integration(tenant_id=context["tenantId"], url=data["url"], name=data["name"]): + return {"data": {"status": "success"}} + else: + return { + "errors": ["failed URL verification, if you received a message on slack, please notify our dev-team"] + } + +@app.route('/{projectId}/errors/search', methods=['POST']) +def errors_search(projectId, context): + data = app.current_request.json_body + params = app.current_request.query_params + if params is None: + params = {} + + return errors.search(data, projectId, user_id=context["userId"], status=params.get("status", "ALL"), + favorite_only="favorite" in params) + + +@app.route('/{projectId}/errors/stats', methods=['GET']) +def errors_stats(projectId, context): + params = app.current_request.query_params + if params is None: + params = {} + + return errors.stats(projectId, user_id=context["userId"], **params) + + +@app.route('/{projectId}/errors/{errorId}', methods=['GET']) +def errors_get_details(projectId, errorId, context): + params = app.current_request.query_params + if params is None: + params = {} + + data = errors.get_details(project_id=projectId, user_id=context["userId"], error_id=errorId, **params) + if data.get("data") is not None: + errors_favorite_viewed.viewed_error(project_id=projectId, user_id=context['userId'], error_id=errorId) + return data + + +@app.route('/{projectId}/errors/{errorId}/stats', methods=['GET']) +def errors_get_details_right_column(projectId, errorId, context): + params = app.current_request.query_params + if params is None: + params = {} + + data = errors.get_details_chart(project_id=projectId, user_id=context["userId"], error_id=errorId, **params) + return data + + +@app.route('/{projectId}/errors/{errorId}/sourcemaps', methods=['GET']) +def errors_get_details_sourcemaps(projectId, errorId, context): + data = errors.get_trace(project_id=projectId, error_id=errorId) + if "errors" in data: + return data + return { + 'data': data + } + + +@app.route('/async/alerts/notifications/{step}', methods=['POST', 'PUT'], authorizer=None) +def send_alerts_notification_async(step): + data = app.current_request.json_body + if data.pop("auth") != environ["async_Token"]: + return {"errors": ["missing auth"]} + if step == "slack": + slack.send_batch(notifications_list=data.get("notifications")) + elif step == "email": + alerts.send_by_email_batch(notifications_list=data.get("notifications")) + elif step == "webhook": + webhook.trigger_batch(data_list=data.get("notifications")) + + +@app.route('/notifications', methods=['GET']) +def get_notifications(context): + return {"data": notifications.get_all(tenant_id=context['tenantId'], user_id=context['userId'])} + + +@app.route('/notifications/{notificationId}/view', methods=['GET']) +def view_notifications(notificationId, context): + return {"data": notifications.view_notification(notification_ids=[notificationId], user_id=context['userId'])} + + +@app.route('/notifications/view', methods=['POST', 'PUT']) +def batch_view_notifications(context): + data = app.current_request.json_body + return {"data": notifications.view_notification(notification_ids=data.get("ids", []), + startTimestamp=data.get("startTimestamp"), + endTimestamp=data.get("endTimestamp"), + user_id=context['userId'], + tenant_id=context["tenantId"])} + + +@app.route('/notifications', methods=['POST', 'PUT'], authorizer=None) +def create_notifications(): + data = app.current_request.json_body + if data.get("token", "") != "nF46JdQqAM5v9KI9lPMpcu8o9xiJGvNNWOGL7TJP": + return {"errors": ["missing token"]} + return notifications.create(data.get("notifications", [])) + + +@app.route('/boarding', methods=['GET']) +def get_boarding_state(context): + return {"data": boarding.get_state(tenant_id=context["tenantId"])} + + +@app.route('/boarding/installing', methods=['GET']) +def get_boarding_state_installing(context): + return {"data": boarding.get_state_installing(tenant_id=context["tenantId"])} + + +@app.route('/boarding/identify-users', methods=['GET']) +def get_boarding_state_identify_users(context): + return {"data": boarding.get_state_identify_users(tenant_id=context["tenantId"])} + + +@app.route('/boarding/manage-users', methods=['GET']) +def get_boarding_state_manage_users(context): + return {"data": boarding.get_state_manage_users(tenant_id=context["tenantId"])} + + +@app.route('/boarding/integrations', methods=['GET']) +def get_boarding_state_integrations(context): + return {"data": boarding.get_state_integrations(tenant_id=context["tenantId"])} + + +# this endpoint supports both jira & github based on `provider` attribute +@app.route('/integrations/issues', methods=['POST', 'PUT']) +def add_edit_jira_cloud_github(context): + data = app.current_request.json_body + provider = data.get("provider", "").upper() + error, integration = integrations_manager.get_integration(tool=provider, tenant_id=context["tenantId"], + user_id=context["userId"]) + if error is not None: + return error + return {"data": integration.add_edit(data=data)} + + +@app.route('/integrations/slack/{integrationId}', methods=['GET']) +def get_slack_webhook(integrationId, context): + return {"data": webhook.get(tenant_id=context["tenantId"], webhook_id=integrationId)} + + +@app.route('/integrations/slack/channels', methods=['GET']) +def get_slack_integration(context): + return {"data": webhook.get_by_type(tenant_id=context["tenantId"], webhook_type='slack')} + + +@app.route('/integrations/slack/{integrationId}', methods=['DELETE']) +def delete_slack_integration(integrationId, context): + return webhook.delete(context["tenantId"], integrationId) + + +@app.route('/webhooks', methods=['POST', 'PUT']) +def add_edit_webhook(context): + data = app.current_request.json_body + return {"data": webhook.add_edit(tenant_id=context["tenantId"], data=data, replace_none=True)} + + +@app.route('/webhooks', methods=['GET']) +def get_webhooks(context): + return {"data": webhook.get_by_tenant(tenant_id=context["tenantId"], replace_none=True)} + + +@app.route('/webhooks/{webhookId}', methods=['DELETE']) +def delete_webhook(webhookId, context): + return {"data": webhook.delete(tenant_id=context["tenantId"], webhook_id=webhookId)} + + +@app.route('/client/members', methods=['GET']) +def get_members(context): + return {"data": users.get_members(tenant_id=context['tenantId'])} + + +@app.route('/client/members', methods=['PUT', 'POST']) +def add_member(context): + data = app.current_request.json_body + return users.create_member(tenant_id=context['tenantId'], user_id=context['userId'], data=data) + + +@app.route('/client/members/{memberId}', methods=['PUT', 'POST']) +def edit_member(memberId, context): + data = app.current_request.json_body + return users.edit(tenant_id=context['tenantId'], editor_id=context['userId'], changes=data, + user_id_to_update=memberId) + + +@app.route('/client/members/{memberId}', methods=['DELETE']) +def delete_member(memberId, context): + return users.delete_member(tenant_id=context["tenantId"], user_id=context['userId'], id_to_delete=memberId) + + +@app.route('/account/new_api_key', methods=['GET']) +def generate_new_user_token(context): + return {"data": users.generate_new_api_key(user_id=context['userId'])} + + +@app.route('/account', methods=['POST', 'PUT']) +def edit_account(context): + data = app.current_request.json_body + return users.edit(tenant_id=context['tenantId'], user_id_to_update=context['userId'], changes=data, + editor_id=context['userId']) + + +@app.route('/account/password', methods=['PUT', 'POST']) +def change_client_password(context): + data = app.current_request.json_body + return users.change_password(email=context['email'], old_password=data["oldPassword"], + new_password=data["newPassword"], tenant_id=context["tenantId"], + user_id=context["userId"]) + + +@app.route('/metadata/session_search', methods=['GET']) +def search_sessions_by_metadata(context): + params = app.current_request.query_params + if params is None: + return {"errors": ["please provide a key&value for search"]} + value = params.get('value', '') + key = params.get('key', '') + project_id = params.get('projectId') + if len(value) == 0 and len(key) == 0: + return {"errors": ["please provide a key&value for search"]} + if len(value) == 0: + return {"errors": ["please provide a value for search"]} + if len(key) == 0: + return {"errors": ["please provide a key for search"]} + return { + "data": sessions.search_by_metadata(tenant_id=context["tenantId"], user_id=context["userId"], m_value=value, + m_key=key, + project_id=project_id)} diff --git a/api/chalicelib/blueprints/bp_core_dynamic_crons.py b/api/chalicelib/blueprints/bp_core_dynamic_crons.py new file mode 100644 index 000000000..74d8766e5 --- /dev/null +++ b/api/chalicelib/blueprints/bp_core_dynamic_crons.py @@ -0,0 +1,13 @@ +from chalice import Blueprint, Cron +from chalicelib import _overrides + +app = Blueprint(__name__) +_overrides.chalice_app(app) + +from chalicelib.core import telemetry + + +# Run every day. +@app.schedule(Cron('0', '0', '?', '*', '*', '*')) +def telemetry_cron(event): + telemetry.compute() diff --git a/api/chalicelib/blueprints/subs/__init__.py b/api/chalicelib/blueprints/subs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/chalicelib/blueprints/subs/bp_dashboard.py b/api/chalicelib/blueprints/subs/bp_dashboard.py new file mode 100644 index 000000000..51bdce092 --- /dev/null +++ b/api/chalicelib/blueprints/subs/bp_dashboard.py @@ -0,0 +1,605 @@ +from chalice import Blueprint +from chalicelib.utils import helper +from chalicelib import _overrides + +from chalicelib.core import dashboard +from chalicelib.core import metadata + +app = Blueprint(__name__) +_overrides.chalice_app(app) + + +@app.route('/{projectId}/dashboard/metadata', methods=['GET']) +def get_metadata_map(projectId, context): + metamap = [] + for m in metadata.get(project_id=projectId): + metamap.append({"name": m["key"], "key": f"metadata{m['index']}"}) + return {"data": metamap} + + +@app.route('/{projectId}/dashboard/sessions', methods=['GET', 'POST']) +def get_dashboard_processed_sessions(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_processed_sessions(project_id=projectId, **{**data, **args})} + + +@app.route('/{projectId}/dashboard/errors', methods=['GET', 'POST']) +def get_dashboard_errors(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_errors(project_id=projectId, **{**data, **args})} + + +@app.route('/{projectId}/dashboard/errors_trend', methods=['GET', 'POST']) +def get_dashboard_errors_trend(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_errors_trend(project_id=projectId, **{**data, **args})} + + +@app.route('/{projectId}/dashboard/application_activity', methods=['GET', 'POST']) +def get_dashboard_application_activity(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_application_activity(project_id=projectId, **{**data, **args})} + + +@app.route('/{projectId}/dashboard/page_metrics', methods=['GET', 'POST']) +def get_dashboard_page_metrics(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_page_metrics(project_id=projectId, **{**data, **args})} + + +@app.route('/{projectId}/dashboard/user_activity', methods=['GET', 'POST']) +def get_dashboard_user_activity(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_user_activity(project_id=projectId, **{**data, **args})} + + +@app.route('/{projectId}/dashboard/performance', methods=['GET', 'POST']) +def get_dashboard_performance(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_performance(project_id=projectId, **{**data, **args})} + + +@app.route('/{projectId}/dashboard/slowest_images', methods=['GET', 'POST']) +def get_dashboard_slowest_images(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_slowest_images(project_id=projectId, **{**data, **args})} + + +@app.route('/{projectId}/dashboard/missing_resources', methods=['GET', 'POST']) +def get_performance_sessions(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_missing_resources_trend(project_id=projectId, **{**data, **args})} + + +@app.route('/{projectId}/dashboard/network', methods=['GET', 'POST']) +def get_network_widget(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_network(project_id=projectId, **{**data, **args})} + + +@app.route('/{projectId}/dashboard/{widget}/search', methods=['GET']) +def get_dashboard_autocomplete(projectId, widget, context): + params = app.current_request.query_params + if params is None: + return {"data": []} + + if widget in ['performance']: + data = dashboard.search(params.get('q', ''), params.get('type', ''), project_id=projectId, + platform=params.get('platform', None), performance=True) + elif widget in ['pages', 'pages_dom_buildtime', 'top_metrics', 'time_to_render', + 'impacted_sessions_by_slow_pages', 'pages_response_time']: + data = dashboard.search(params.get('q', ''), params.get('type', ''), project_id=projectId, + platform=params.get('platform', None), pages_only=True) + elif widget in ['resources_loading_time']: + data = dashboard.search(params.get('q', ''), params.get('type', ''), project_id=projectId, + platform=params.get('platform', None), performance=False) + elif widget in ['time_between_events', 'events']: + data = dashboard.search(params.get('q', ''), params.get('type', ''), project_id=projectId, + platform=params.get('platform', None), performance=False, events_only=True) + elif widget in ['metadata']: + data = dashboard.search(params.get('q', ''), None, project_id=projectId, + platform=params.get('platform', None), metadata=True, key=params.get("key")) + else: + return {"errors": [f"unsupported widget: {widget}"]} + return {'data': data} + + +# 1 +@app.route('/{projectId}/dashboard/slowest_resources', methods=['GET', 'POST']) +def get_dashboard_slowest_resources(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_slowest_resources(project_id=projectId, **{**data, **args})} + + +# 2 +@app.route('/{projectId}/dashboard/resources_loading_time', methods=['GET', 'POST']) +def get_dashboard_resources(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_resources_loading_time(project_id=projectId, **{**data, **args})} + + +# 3 +@app.route('/{projectId}/dashboard/pages_dom_buildtime', methods=['GET', 'POST']) +def get_dashboard_pages_dom(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_pages_dom_build_time(project_id=projectId, **{**data, **args})} + + +# 4 +@app.route('/{projectId}/dashboard/busiest_time_of_day', methods=['GET', 'POST']) +def get_dashboard_busiest_time_of_day(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_busiest_time_of_day(project_id=projectId, **{**data, **args})} + + +# 5 +@app.route('/{projectId}/dashboard/sessions_location', methods=['GET', 'POST']) +def get_dashboard_sessions_location(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_sessions_location(project_id=projectId, **{**data, **args})} + + +# 6 +@app.route('/{projectId}/dashboard/speed_location', methods=['GET', 'POST']) +def get_dashboard_speed_location(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_speed_index_location(project_id=projectId, **{**data, **args})} + + +# 7 +@app.route('/{projectId}/dashboard/pages_response_time', methods=['GET', 'POST']) +def get_dashboard_pages_response_time(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_pages_response_time(project_id=projectId, **{**data, **args})} + + +# 8 +@app.route('/{projectId}/dashboard/pages_response_time_distribution', methods=['GET', 'POST']) +def get_dashboard_pages_response_time_distribution(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_pages_response_time_distribution(project_id=projectId, **{**data, **args})} + + +# 9 +@app.route('/{projectId}/dashboard/top_metrics', methods=['GET', 'POST']) +def get_dashboard_top_metrics(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_top_metrics(project_id=projectId, **{**data, **args})} + + +# 10 +@app.route('/{projectId}/dashboard/time_to_render', methods=['GET', 'POST']) +def get_dashboard_time_to_render(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_time_to_render(project_id=projectId, **{**data, **args})} + + +# 11 +@app.route('/{projectId}/dashboard/impacted_sessions_by_slow_pages', methods=['GET', 'POST']) +def get_dashboard_impacted_sessions_by_slow_pages(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_impacted_sessions_by_slow_pages(project_id=projectId, **{**data, **args})} + + +# 12 +@app.route('/{projectId}/dashboard/memory_consumption', methods=['GET', 'POST']) +def get_dashboard_memory_consumption(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_memory_consumption(project_id=projectId, **{**data, **args})} + + +# 12.1 +@app.route('/{projectId}/dashboard/fps', methods=['GET', 'POST']) +def get_dashboard_avg_fps(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_avg_fps(project_id=projectId, **{**data, **args})} + + +# 12.2 +@app.route('/{projectId}/dashboard/cpu', methods=['GET', 'POST']) +def get_dashboard_avg_cpu(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_avg_cpu(project_id=projectId, **{**data, **args})} + + +# 13 +@app.route('/{projectId}/dashboard/crashes', methods=['GET', 'POST']) +def get_dashboard_impacted_sessions_by_slow_pages(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_crashes(project_id=projectId, **{**data, **args})} + + +# 14 +@app.route('/{projectId}/dashboard/domains_errors', methods=['GET', 'POST']) +def get_dashboard_domains_errors(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_domains_errors(project_id=projectId, **{**data, **args})} + + +# 14.1 +@app.route('/{projectId}/dashboard/domains_errors_4xx', methods=['GET', 'POST']) +def get_dashboard_domains_errors_4xx(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_domains_errors_4xx(project_id=projectId, **{**data, **args})} + + +# 14.2 +@app.route('/{projectId}/dashboard/domains_errors_5xx', methods=['GET', 'POST']) +def get_dashboard_domains_errors_5xx(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_domains_errors_5xx(project_id=projectId, **{**data, **args})} + + +# 15 +@app.route('/{projectId}/dashboard/slowest_domains', methods=['GET', 'POST']) +def get_dashboard_slowest_domains(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_slowest_domains(project_id=projectId, **{**data, **args})} + + +# 16 +@app.route('/{projectId}/dashboard/errors_per_domains', methods=['GET', 'POST']) +def get_dashboard_errors_per_domains(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_errors_per_domains(project_id=projectId, **{**data, **args})} + + +# 17 +@app.route('/{projectId}/dashboard/sessions_per_browser', methods=['GET', 'POST']) +def get_dashboard_sessions_per_browser(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_sessions_per_browser(project_id=projectId, **{**data, **args})} + + +# 18 +@app.route('/{projectId}/dashboard/calls_errors', methods=['GET', 'POST']) +def get_dashboard_calls_errors(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_calls_errors(project_id=projectId, **{**data, **args})} + + +# 18.1 +@app.route('/{projectId}/dashboard/calls_errors_4xx', methods=['GET', 'POST']) +def get_dashboard_calls_errors_4xx(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_calls_errors_4xx(project_id=projectId, **{**data, **args})} + + +# 18.2 +@app.route('/{projectId}/dashboard/calls_errors_5xx', methods=['GET', 'POST']) +def get_dashboard_calls_errors_5xx(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_calls_errors_5xx(project_id=projectId, **{**data, **args})} + + +# 19 +@app.route('/{projectId}/dashboard/errors_per_type', methods=['GET', 'POST']) +def get_dashboard_errors_per_type(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_errors_per_type(project_id=projectId, **{**data, **args})} + + +# 20 +@app.route('/{projectId}/dashboard/resources_by_party', methods=['GET', 'POST']) +def get_dashboard_resources_by_party(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_resources_by_party(project_id=projectId, **{**data, **args})} + + +# 21 +@app.route('/{projectId}/dashboard/resource_type_vs_response_end', methods=['GET', 'POST']) +def get_dashboard_errors_per_resource_type(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.resource_type_vs_response_end(project_id=projectId, **{**data, **args})} + + +# 22 +@app.route('/{projectId}/dashboard/resources_vs_visually_complete', methods=['GET', 'POST']) +def get_dashboard_resources_vs_visually_complete(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_resources_vs_visually_complete(project_id=projectId, **{**data, **args})} + + +# 23 +@app.route('/{projectId}/dashboard/impacted_sessions_by_js_errors', methods=['GET', 'POST']) +def get_dashboard_impacted_sessions_by_js_errors(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_impacted_sessions_by_js_errors(project_id=projectId, **{**data, **args})} + + +# 24 +@app.route('/{projectId}/dashboard/resources_count_by_type', methods=['GET', 'POST']) +def get_dashboard_resources_count_by_type(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": dashboard.get_resources_count_by_type(project_id=projectId, **{**data, **args})} + + +# 25 +@app.route('/{projectId}/dashboard/time_between_events', methods=['GET']) +def get_dashboard_resources_count_by_type(projectId, context): + return {"errors": ["please choose 2 events"]} + + +@app.route('/{projectId}/dashboard/overview', methods=['GET', 'POST']) +def get_dashboard_group(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": [ + *helper.explode_widget(key="count_sessions", + data=dashboard.get_processed_sessions(project_id=projectId, **{**data, **args})), + *helper.explode_widget(data={**dashboard.get_application_activity(project_id=projectId, **{**data, **args}), + "chart": dashboard.get_performance(project_id=projectId, **{**data, **args}) + .get("chart", [])}), + *helper.explode_widget(data=dashboard.get_page_metrics(project_id=projectId, **{**data, **args})), + *helper.explode_widget(data=dashboard.get_user_activity(project_id=projectId, **{**data, **args})), + *helper.explode_widget(data=dashboard.get_pages_dom_build_time(project_id=projectId, **{**data, **args}), + key="avg_pages_dom_buildtime"), + *helper.explode_widget(data=dashboard.get_pages_response_time(project_id=projectId, **{**data, **args}), + key="avg_pages_response_time"), + *helper.explode_widget(dashboard.get_top_metrics(project_id=projectId, **{**data, **args})), + *helper.explode_widget(data=dashboard.get_time_to_render(project_id=projectId, **{**data, **args}), + key="avg_time_to_render"), + *helper.explode_widget(dashboard.get_memory_consumption(project_id=projectId, **{**data, **args})), + *helper.explode_widget(dashboard.get_avg_cpu(project_id=projectId, **{**data, **args})), + *helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **{**data, **args})), + ]} + + +@app.route('/{projectId}/dashboard/errors_crashes', methods=['GET', 'POST']) +def get_dashboard_group(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": [ + {"key": "errors", + "data": dashboard.get_errors(project_id=projectId, **{**data, **args})}, + {"key": "errors_trend", + "data": dashboard.get_errors_trend(project_id=projectId, **{**data, **args})}, + {"key": "crashes", + "data": dashboard.get_crashes(project_id=projectId, **{**data, **args})}, + {"key": "domains_errors", + "data": dashboard.get_domains_errors(project_id=projectId, **{**data, **args})}, + {"key": "errors_per_domains", + "data": dashboard.get_errors_per_domains(project_id=projectId, **{**data, **args})}, + {"key": "calls_errors", + "data": dashboard.get_calls_errors(project_id=projectId, **{**data, **args})}, + {"key": "errors_per_type", + "data": dashboard.get_errors_per_type(project_id=projectId, **{**data, **args})}, + {"key": "impacted_sessions_by_js_errors", + "data": dashboard.get_impacted_sessions_by_js_errors(project_id=projectId, **{**data, **args})} + ]} + + +@app.route('/{projectId}/dashboard/resources', methods=['GET', 'POST']) +def get_dashboard_group(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + params = app.current_request.query_params + args = dashboard.dashboard_args(params) + + return {"data": [ + {"key": "slowest_images", + "data": dashboard.get_slowest_images(project_id=projectId, **{**data, **args})}, + {"key": "missing_resources", + "data": dashboard.get_missing_resources_trend(project_id=projectId, **{**data, **args})}, + {"key": "slowest_resources", + "data": dashboard.get_slowest_resources(project_id=projectId, type='all', **{**data, **args})}, + {"key": "resources_loading_time", + "data": dashboard.get_resources_loading_time(project_id=projectId, **{**data, **args})}, + {"key": "resources_by_party", + "data": dashboard.get_resources_by_party(project_id=projectId, **{**data, **args})}, + {"key": "resource_type_vs_response_end", + "data": dashboard.resource_type_vs_response_end(project_id=projectId, **{**data, **args})}, + {"key": "resources_vs_visually_complete", + "data": dashboard.get_resources_vs_visually_complete(project_id=projectId, **{**data, **args})}, + {"key": "resources_count_by_type", + "data": dashboard.get_resources_count_by_type(project_id=projectId, **{**data, **args})} + ]} diff --git a/api/chalicelib/core/__init__.py b/api/chalicelib/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/chalicelib/core/alerts.py b/api/chalicelib/core/alerts.py new file mode 100644 index 000000000..6ed9aff3e --- /dev/null +++ b/api/chalicelib/core/alerts.py @@ -0,0 +1,168 @@ +import time +from chalicelib.utils.helper import environ + +from chalicelib.core import notifications +from chalicelib.utils import pg_client, helper, email_helper +from chalicelib.utils.TimeUTC import TimeUTC +import json + +ALLOW_UPDATE = ["name", "description", "active", "detectionMethod", "query", "options"] + + +def get(id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + SELECT * + FROM public.alerts + WHERE alert_id =%(id)s;""", + {"id": id}) + ) + a = helper.dict_to_camel_case(cur.fetchone()) + return __process_circular(a) + + +def get_all(project_id): + with pg_client.PostgresClient() as cur: + query = cur.mogrify("""\ + SELECT * + FROM public.alerts + WHERE project_id =%(project_id)s AND deleted_at ISNULL + ORDER BY created_at;""", + {"project_id": project_id}) + cur.execute(query=query) + all = helper.list_to_camel_case(cur.fetchall()) + for a in all: + a = __process_circular(a) + return all + + +SUPPORTED_THRESHOLD = [15, 30, 60, 120, 240, 1440] + + +def __transform_structure(data): + if data.get("options") is None: + return f"Missing 'options'", None + if data["options"].get("currentPeriod") not in SUPPORTED_THRESHOLD: + return f"Unsupported currentPeriod, please provide one of these values {SUPPORTED_THRESHOLD}", None + if data["options"].get("previousPeriod", 15) not in SUPPORTED_THRESHOLD: + return f"Unsupported previousPeriod, please provide one of these values {SUPPORTED_THRESHOLD}", None + if data["options"].get("renotifyInterval") is None: + data["options"]["renotifyInterval"] = 720 + data["query"]["right"] = float(data["query"]["right"]) + data["query"] = json.dumps(data["query"]) + data["description"] = data["description"] if data.get("description") is not None and len( + data["description"]) > 0 else None + if data.get("options"): + messages = [] + for m in data["options"].get("message", []): + if m.get("value") is None: + continue + m["value"] = str(m["value"]) + messages.append(m) + data["options"]["message"] = messages + data["options"] = json.dumps(data["options"]) + return None, data + + +def __process_circular(alert): + if alert is None: + return None + alert.pop("deletedAt") + alert["createdAt"] = TimeUTC.datetime_to_timestamp(alert["createdAt"]) + return alert + + +def create(project_id, data): + err, data = __transform_structure(data) + if err is not None: + return {"errors": [err]} + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + INSERT INTO public.alerts(project_id, name, description, detection_method, query, options) + VALUES (%(project_id)s, %(name)s, %(description)s, %(detectionMethod)s, %(query)s, %(options)s::jsonb) + RETURNING *;""", + {"project_id": project_id, **data}) + ) + a = helper.dict_to_camel_case(cur.fetchone()) + return {"data": helper.dict_to_camel_case(__process_circular(a))} + + +def update(id, changes): + changes = {k: changes[k] for k in changes.keys() if k in ALLOW_UPDATE} + err, changes = __transform_structure(changes) + if err is not None: + return {"errors": [err]} + updateq = [] + for k in changes.keys(): + updateq.append(f"{helper.key_to_snake_case(k)} = %({k})s") + if len(updateq) == 0: + return {"errors": ["nothing to update"]} + with pg_client.PostgresClient() as cur: + query = cur.mogrify(f"""\ + UPDATE public.alerts + SET {", ".join(updateq)} + WHERE alert_id =%(id)s AND deleted_at ISNULL + RETURNING *;""", + {"id": id, **changes}) + cur.execute(query=query) + a = helper.dict_to_camel_case(cur.fetchone()) + return {"data": __process_circular(a)} + + +def process_notifications(data): + full = {} + for n in data: + if "message" in n["options"]: + webhook_data = {} + if "data" in n["options"]: + webhook_data = n["options"].pop("data") + for c in n["options"].pop("message"): + if c["type"] not in full: + full[c["type"]] = [] + if c["type"] in ["slack", "email"]: + full[c["type"]].append({ + "notification": n, + "destination": c["value"] + }) + elif c["type"] in ["webhook"]: + full[c["type"]].append({"data": webhook_data, "destination": c["value"]}) + notifications.create(data) + BATCH_SIZE = 200 + for t in full.keys(): + for i in range(0, len(full[t]), BATCH_SIZE): + helper.async_post(environ['alert_ntf'] % t, {"notifications": full[t][i:i + BATCH_SIZE]}) + + +def send_by_email(notification, destination): + if notification is None: + return + email_helper.alert_email(recipients=destination, + subject=f'"{notification["title"]}" has been triggered', + data={ + "message": f'"{notification["title"]}" {notification["description"]}', + "project_id": notification["options"]["projectId"]}) + + +def send_by_email_batch(notifications_list): + if notifications_list is None or len(notifications_list) == 0: + return + for n in notifications_list: + send_by_email(notification=n.get("notification"), destination=n.get("destination")) + time.sleep(1) + + +def delete(project_id, alert_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + UPDATE public.alerts + SET + deleted_at = timezone('utc'::text, now()), + active = FALSE + WHERE + alert_id = %(alert_id)s AND project_id=%(project_id)s;""", + {"alert_id": alert_id, "project_id": project_id}) + ) + return {"data": {"state": "success"}} diff --git a/api/chalicelib/core/announcements.py b/api/chalicelib/core/announcements.py new file mode 100644 index 000000000..bb719afb9 --- /dev/null +++ b/api/chalicelib/core/announcements.py @@ -0,0 +1,42 @@ +from chalicelib.utils import pg_client +from chalicelib.utils import helper +from chalicelib.utils.helper import environ +from chalicelib.utils.TimeUTC import TimeUTC + + +def get_all(user_id): + with pg_client.PostgresClient() as cur: + query = cur.mogrify(""" + SELECT a.*, u.last >= (EXTRACT(EPOCH FROM a.created_at)*1000) AS viewed + FROM public.announcements AS a, + (SELECT COALESCE(CAST(data ->> 'lastAnnouncementView' AS bigint), 0) + FROM public.users + WHERE user_id = %(userId)s + LIMIT 1) AS u(last) + ORDER BY a.created_at DESC;""", + {"userId": user_id}) + cur.execute( + query + ) + announcements = helper.list_to_camel_case(cur.fetchall()) + for a in announcements: + a["createdAt"] = TimeUTC.datetime_to_timestamp(a["createdAt"]) + if a["imageUrl"] is not None and len(a["imageUrl"]) > 0: + a["imageUrl"] = environ["announcement_bucket"] + a["imageUrl"] + return announcements + + +def view(user_id): + with pg_client.PostgresClient() as cur: + query = cur.mogrify(""" + UPDATE public.users + SET data=data || + ('{"lastAnnouncementView":' || + (EXTRACT(EPOCH FROM timezone('utc'::text, now())) * 1000)::bigint - 20 * 000 || + '}')::jsonb + WHERE user_id = %(userId)s;""", + {"userId": user_id}) + cur.execute( + query + ) + return True diff --git a/api/chalicelib/core/authorizers.py b/api/chalicelib/core/authorizers.py new file mode 100644 index 000000000..1122b0d65 --- /dev/null +++ b/api/chalicelib/core/authorizers.py @@ -0,0 +1,61 @@ +from chalicelib.utils.helper import environ +import jwt +from chalicelib.utils import helper +from chalicelib.utils.TimeUTC import TimeUTC + +from chalicelib.core import tenants +from chalicelib.core import users + + +def jwt_authorizer(token): + token = token.split(" ") + if len(token) != 2 or token[0].lower() != "bearer": + return None + try: + payload = jwt.decode( + token[1], + environ["jwt_secret"], + algorithms=environ["jwt_algorithm"], + audience=[f"plugin:{helper.get_stage_name()}", f"front:{helper.get_stage_name()}"] + ) + except jwt.ExpiredSignatureError: + print("! JWT Expired signature") + return None + except BaseException as e: + print("! JWT Base Exception") + return None + return payload + + +def jwt_context(context): + user = users.get(user_id=context["userId"], tenant_id=context["tenantId"]) + if user is None: + return None + return { + "tenantId": context["tenantId"], + "userId": context["userId"], + **user + } + + +def generate_jwt(id, tenant_id, iat, aud): + token = jwt.encode( + payload={ + "userId": id, + "tenantId": tenant_id, + "exp": iat // 1000 + int(environ["jwt_exp_delta_seconds"]) + TimeUTC.get_utc_offset() // 1000, + "iss": environ["jwt_issuer"], + "iat": iat // 1000, + "aud": aud + }, + key=environ["jwt_secret"], + algorithm=environ["jwt_algorithm"] + ) + return token.decode("utf-8") + + +def api_key_authorizer(token): + t = tenants.get_by_api_key(token) + if t is not None: + t["createdAt"] = TimeUTC.datetime_to_timestamp(t["createdAt"]) + return t diff --git a/api/chalicelib/core/boarding.py b/api/chalicelib/core/boarding.py new file mode 100644 index 000000000..e39f8ff28 --- /dev/null +++ b/api/chalicelib/core/boarding.py @@ -0,0 +1,116 @@ +from chalicelib.utils import pg_client +from chalicelib.core import projects, log_tool_datadog, log_tool_stackdriver, log_tool_sentry + +from chalicelib.core import users + + +def get_state(tenant_id): + my_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) + pids = [s["projectId"] for s in my_projects] + with pg_client.PostgresClient() as cur: + recorded = False + meta = False + + if len(pids) > 0: + cur.execute( + cur.mogrify("""\ + SELECT + COUNT(*) + FROM public.sessions AS s + where s.project_id IN %(ids)s + LIMIT 1;""", + {"ids": tuple(pids)}) + ) + recorded = cur.fetchone()["count"] > 0 + meta = False + if recorded: + cur.execute("""SELECT SUM((SELECT COUNT(t.meta) + FROM (VALUES (p.metadata_1), (p.metadata_2), (p.metadata_3), (p.metadata_4), (p.metadata_5), + (p.metadata_6), (p.metadata_7), (p.metadata_8), (p.metadata_9), (p.metadata_10), + (sessions.user_id)) AS t(meta) + WHERE t.meta NOTNULL)) + FROM public.projects AS p + LEFT JOIN LATERAL ( SELECT 'defined' + FROM public.sessions + WHERE sessions.project_id=p.project_id AND sessions.user_id IS NOT NULL + LIMIT 1) AS sessions(user_id) ON(TRUE) + WHERE p.deleted_at ISNULL;""" + ) + + meta = cur.fetchone()["sum"] > 0 + + return [ + {"task": "Install Asayer", + "done": recorded, + "URL": "https://docs.asayer.io/getting-started/quick-start"}, + {"task": "Identify Users", + "done": meta, + "URL": "https://docs.asayer.io/data-privacy-security/metadata"}, + {"task": "Invite Team Members", + "done": len(users.get_members(tenant_id=tenant_id)) > 1, + "URL": "https://app.asayer.io/client/manage-users"}, + {"task": "Integrations", + "done": len(log_tool_datadog.get_all(tenant_id=tenant_id)) > 0 \ + or len(log_tool_sentry.get_all(tenant_id=tenant_id)) > 0 \ + or len(log_tool_stackdriver.get_all(tenant_id=tenant_id)) > 0, + "URL": "https://docs.asayer.io/integrations"} + ] + + +def get_state_installing(tenant_id): + my_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) + pids = [s["projectId"] for s in my_projects] + with pg_client.PostgresClient() as cur: + recorded = False + + if len(pids) > 0: + cur.execute( + cur.mogrify("""\ + SELECT + COUNT(*) + FROM public.sessions AS s + where s.project_id IN %(ids)s + LIMIT 1;""", + {"ids": tuple(pids)}) + ) + recorded = cur.fetchone()["count"] > 0 + + return {"task": "Install Asayer", + "done": recorded, + "URL": "https://docs.asayer.io/getting-started/quick-start"} + + +def get_state_identify_users(tenant_id): + with pg_client.PostgresClient() as cur: + cur.execute( + """SELECT SUM((SELECT COUNT(t.meta) + FROM (VALUES (p.metadata_1), (p.metadata_2), (p.metadata_3), (p.metadata_4), (p.metadata_5), + (p.metadata_6), (p.metadata_7), (p.metadata_8), (p.metadata_9), (p.metadata_10), + (sessions.user_id)) AS t(meta) + WHERE t.meta NOTNULL)) + FROM public.projects AS p + LEFT JOIN LATERAL ( SELECT 'defined' + FROM public.sessions + WHERE sessions.project_id=p.project_id AND sessions.user_id IS NOT NULL + LIMIT 1) AS sessions(user_id) ON(TRUE) + WHERE p.deleted_at ISNULL;""") + + meta = cur.fetchone()["sum"] > 0 + + return {"task": "Identify Users", + "done": meta, + "URL": "https://docs.asayer.io/data-privacy-security/metadata"} + + +def get_state_manage_users(tenant_id): + return {"task": "Invite Team Members", + "done": len(users.get_members(tenant_id=tenant_id)) > 1, + "URL": "https://app.asayer.io/client/manage-users"} + + +def get_state_integrations(tenant_id): + return {"task": "Integrations", + "done": len(log_tool_datadog.get_all(tenant_id=tenant_id)) > 0 \ + or len(log_tool_sentry.get_all(tenant_id=tenant_id)) > 0 \ + or len(log_tool_stackdriver.get_all(tenant_id=tenant_id)) > 0, + "URL": "https://docs.asayer.io/integrations"} diff --git a/api/chalicelib/core/collaboration_slack.py b/api/chalicelib/core/collaboration_slack.py new file mode 100644 index 000000000..5fc80511c --- /dev/null +++ b/api/chalicelib/core/collaboration_slack.py @@ -0,0 +1,126 @@ +import requests +from chalicelib.utils.helper import environ +from datetime import datetime +from chalicelib.core import webhook + + +class Slack: + @classmethod + def add_integration(cls, tenant_id, **args): + url = args["url"] + name = args["name"] + if cls.__say_hello(url): + webhook.add(tenant_id=tenant_id, + endpoint=url, + webhook_type="slack", + name=name) + return True + return False + + @classmethod + def __say_hello(cls, url): + r = requests.post( + url=url, + json={ + "attachments": [ + { + "text": "Welcome to OpenReplay", + "ts": datetime.now().timestamp(), + } + ] + }) + if r.status_code != 200: + print("slack integration failed") + print(r.text) + return False + return True + + @classmethod + def send_text(cls, tenant_id, webhook_id, text, **args): + integration = cls.__get(tenant_id=tenant_id, integration_id=webhook_id) + if integration is None: + return {"errors": ["slack integration not found"]} + print("====> sending slack notification") + r = requests.post( + url=integration["endpoint"], + json={ + "attachments": [ + { + "text": text, + "ts": datetime.now().timestamp(), + **args + } + ] + }) + print(r) + print(r.text) + return {"data": r.text} + + @classmethod + def send_batch(cls, tenant_id, webhook_id, attachments): + integration = cls.__get(tenant_id=tenant_id, integration_id=webhook_id) + if integration is None: + return {"errors": ["slack integration not found"]} + print(f"====> sending slack batch notification: {len(attachments)}") + for i in range(0, len(attachments), 100): + r = requests.post( + url=integration["endpoint"], + json={"attachments": attachments[i:i + 100]}) + if r.status_code != 200: + print("!!!! something went wrong") + print(r) + print(r.text) + + @classmethod + def __share_to_slack(cls, tenant_id, integration_id, fallback, pretext, title, title_link, text): + integration = cls.__get(tenant_id=tenant_id, integration_id=integration_id) + if integration is None: + return {"errors": ["slack integration not found"]} + r = requests.post( + url=integration["endpoint"], + json={ + "attachments": [ + { + "fallback": fallback, + "pretext": pretext, + "title": title, + "title_link": title_link, + "text": text, + "ts": datetime.now().timestamp() + } + ] + }) + return r.text + + @classmethod + def share_session(cls, tenant_id, project_id, session_id, user, comment, integration_id=None): + args = {"fallback": f"{user} has shared the below session!", + "pretext": f"{user} has shared the below session!", + "title": f"{environ['SITE_URL']}/{project_id}/session/{session_id}", + "title_link": f"{environ['SITE_URL']}/{project_id}/session/{session_id}", + "text": comment} + return {"data": cls.__share_to_slack(tenant_id, integration_id, **args)} + + @classmethod + def share_error(cls, tenant_id, project_id, error_id, user, comment, integration_id=None): + args = {"fallback": f"{user} has shared the below error!", + "pretext": f"{user} has shared the below error!", + "title": f"{environ['SITE_URL']}/{project_id}/errors/{error_id}", + "title_link": f"{environ['SITE_URL']}/{project_id}/errors/{error_id}", + "text": comment} + return {"data": cls.__share_to_slack(tenant_id, integration_id, **args)} + + @classmethod + def has_slack(cls, tenant_id): + integration = cls.__get(tenant_id=tenant_id) + return not (integration is None or len(integration) == 0) + + @classmethod + def __get(cls, tenant_id, integration_id=None): + if integration_id is not None: + return webhook.get(tenant_id=tenant_id, webhook_id=integration_id) + + integrations = webhook.get_by_type(tenant_id=tenant_id, webhook_type="slack") + if integrations is None or len(integrations) == 0: + return None + return integrations[0] diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py new file mode 100644 index 000000000..e739ceb04 --- /dev/null +++ b/api/chalicelib/core/dashboard.py @@ -0,0 +1,2034 @@ +from chalicelib.core import metadata +from chalicelib.utils import args_transformer +from chalicelib.utils import helper, dev +from chalicelib.utils import pg_client +from chalicelib.utils.TimeUTC import TimeUTC +from chalicelib.utils.metrics_helper import __get_step_size +import math + + +# Written by David Aznaurov, inspired by numpy.quantile +def __quantiles(a, q, interpolation='higher'): + arr = a.copy() + arr = sorted(arr) + if isinstance(q, list): + ind = [qi * (len(arr) - 1) for qi in q] + else: + ind = q * (len(arr) - 1) + if interpolation == 'higher': + if isinstance(q, list): + ind = [math.ceil(i) for i in ind] + else: + ind = math.ceil(ind) + if isinstance(q, list): + return [arr[i] for i in ind] + else: + return arr[ind] + + +def __get_constraints(project_id, time_constraint=True, chart=False, duration=True, project=True, + project_identifier="project_id", + main_table="sessions", time_column="start_ts", data={}): + pg_sub_query = [] + if project: + pg_sub_query.append(f"{main_table}.{project_identifier} =%({project_identifier})s") + if duration: + pg_sub_query.append(f"{main_table}.duration>0") + if time_constraint: + pg_sub_query.append(f"{main_table}.{time_column} >= %(startTimestamp)s") + pg_sub_query.append(f"{main_table}.{time_column} < %(endTimestamp)s") + if chart: + pg_sub_query.append(f"{main_table}.{time_column} >= generated_timestamp") + pg_sub_query.append(f"{main_table}.{time_column} < generated_timestamp + %(step_size)s") + return pg_sub_query + __get_meta_constraint(project_id=project_id, data=data) + + +def __merge_charts(list1, list2, time_key="timestamp"): + if len(list1) != len(list2): + raise Exception("cannot merge unequal lists") + result = [] + for i in range(len(list1)): + timestamp = min(list1[i][time_key], list2[i][time_key]) + result.append({**list1[i], **list2[i], time_key: timestamp}) + return result + + +def __get_constraint_values(data): + params = {} + for i, f in enumerate(data.get("filters", [])): + params[f"{f['key']}_{i}"] = f["value"] + return params + + +METADATA_FIELDS = {"userId": "user_id", + "userAnonymousId": "user_anonymous_id", + "metadata1": "metadata_1", + "metadata2": "metadata_2", + "metadata3": "metadata_3", + "metadata4": "metadata_4", + "metadata5": "metadata_5", + "metadata6": "metadata_6", + "metadata7": "metadata_7", + "metadata8": "metadata_8", + "metadata9": "metadata_9", + "metadata10": "metadata_10"} + +from chalicelib.core import sessions_metas + + +def __get_meta_constraint(project_id, data): + if len(data.get("filters", [])) == 0: + return [] + constraints = [] + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + print(meta_keys) + + for i, f in enumerate(data.get("filters", [])): + if f["key"] in meta_keys.keys(): + key = f"sessions.metadata_{meta_keys[f['key']]})" + if f["value"] in ["*", ""]: + constraints.append(f"{key} IS NOT NULL") + else: + constraints.append(f"{key} = %({f['key']}_{i})s") + else: + filter_type = f["key"].upper() + if filter_type == sessions_metas.meta_type.USERBROWSER: + constraints.append(f"sessions.user_browser = %({f['key']}_{i})s") + elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: + constraints.append(f"sessions.user_os = %({f['key']}_{i})s") + elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: + constraints.append(f"sessions.user_device = %({f['key']}_{i})s") + elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: + constraints.append(f"sessions.user_country = %({f['key']}_{i})s") + elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + constraints.append(f"sessions.user_id = %({f['key']}_{i})s") + elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, + sessions_metas.meta_type.USERANONYMOUSID_IOS]: + constraints.append(f"sessions.user_anonymous_id = %({f['key']}_{i})s") + elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: + constraints.append(f"sessions.rev_id = %({f['key']}_{i})s") + return constraints + + +SESSIONS_META_FIELDS = {"revId": "rev_id", + "country": "user_country", + "os": "user_os", + "platform": "user_device_type", + "device": "user_device", + "browser": "user_browser"} + + +@dev.timed +def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + with pg_client.PostgresClient() as cur: + pg_query = f"""\ + SELECT generated_timestamp AS timestamp, + COALESCE(COUNT(sessions), 0) AS count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT 1 + FROM public.sessions + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + results = { + "count": sum([r["count"] for r in rows]), + "chart": rows + } + + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + + pg_query = f"""\ + SELECT COUNT(sessions.session_id) AS count + FROM public.sessions + WHERE {" AND ".join(pg_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + cur.execute(cur.mogrify(pg_query, params)) + + count = cur.fetchone()["count"] + + results["countProgress"] = helper.__progress(old_val=count, new_val=results["count"]) + + return results + + +@dev.timed +def get_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), + density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("m_errors.source = 'js_exception'") + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + pg_sub_query_chart.append("m_errors.source = 'js_exception'") + + with pg_client.PostgresClient() as cur: + pg_query = f"""\ + SELECT generated_timestamp AS timestamp, + COALESCE(COUNT(sessions), 0) AS count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT DISTINCT session_id + FROM events.errors + INNER JOIN public.errors AS m_errors USING(error_id) + INNER JOIN public.sessions USING(session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + results = { + "count": 0 if len(rows) == 0 else __count_distinct_errors(cur, project_id, startTimestamp, endTimestamp, + pg_sub_query), + "impactedSessions": sum([r["count"] for r in rows]), + "chart": rows + } + + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + count = __count_distinct_errors(cur, project_id, startTimestamp, endTimestamp, pg_sub_query, **args) + results["progress"] = helper.__progress(old_val=count, new_val=results["count"]) + return results + + +def __count_distinct_errors(cur, project_id, startTimestamp, endTimestamp, pg_sub_query, **args): + pg_query = f"""\ + SELECT COALESCE(COUNT(DISTINCT errors.error_id),0) AS count + FROM events.errors + INNER JOIN public.errors AS m_errors USING(error_id) + INNER JOIN public.sessions USING(session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + return cur.fetchone()["count"] + + +@dev.timed +def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("m_errors.project_id = %(project_id)s") + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, duration=False, + project=False, main_table="errors", time_column="timestamp", data=args) + pg_sub_query_chart.append("error_id = errors_details.error_id") + + with pg_client.PostgresClient() as cur: + pg_query = f""" + SELECT * + FROM (SELECT errors.error_id AS error_id, + m_errors.message AS error, + COUNT(errors.session_id) AS count, + COUNT(DISTINCT errors.session_id) AS sessions_count + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY errors.error_id, m_errors.message + ORDER BY sessions_count DESC, count DESC + LIMIT 10) AS errors_details + INNER JOIN LATERAL (SELECT MAX(timestamp) AS last_occurrence_at, + MIN(timestamp) AS first_occurrence_at + FROM events.errors + WHERE error_id = errors_details.error_id + GROUP BY error_id) AS errors_time ON (TRUE) + INNER JOIN LATERAL (SELECT jsonb_agg(chart) AS chart + FROM (SELECT generated_timestamp AS timestamp, COALESCE(COUNT(sessions), 0) AS count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT DISTINCT session_id + FROM events.errors + WHERE {" AND ".join(pg_sub_query_chart)}) AS sessions + ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp) AS chart) AS chart ON (TRUE);""" + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + + for i in range(len(rows)): + rows[i] = helper.dict_to_camel_case(rows[i]) + rows[i]["sessions"] = rows[i].pop("sessionsCount") + rows[i]["error_id"] = rows[i]["errorId"] + rows[i]["startTimestamp"] = startTimestamp + rows[i]["endTimestamp"] = endTimestamp + + return rows + + +@dev.timed +def get_page_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + rows = __get_page_metrics(cur, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + results = helper.dict_to_camel_case(rows[0]) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + rows = __get_page_metrics(cur, project_id, startTimestamp, endTimestamp, **args) + if len(rows) > 0: + previous = helper.dict_to_camel_case(rows[0]) + for key in previous.keys(): + results[key + "Progress"] = helper.__progress(old_val=previous[key], new_val=results[key]) + return results + + +def __get_page_metrics(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + pg_query = f"""\ + SELECT + COALESCE(AVG(NULLIF(pages.dom_content_loaded_time ,0)),0) AS avg_dom_content_load_start, + COALESCE(AVG(NULLIF(pages.first_contentful_paint_time,0)),0) AS avg_first_contentful_pixel + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + return rows + + +@dev.timed +def get_application_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_application_activity(cur, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_application_activity(cur, project_id, startTimestamp, endTimestamp, **args) + previous = helper.dict_to_camel_case(row) + for key in previous.keys(): + results[key + "Progress"] = helper.__progress(old_val=previous[key], new_val=results[key]) + return results + + +def __get_application_activity(cur, project_id, startTimestamp, endTimestamp, **args): + result = {} + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("pages.load_time > 0") + pg_query = f"""\ + SELECT COALESCE(AVG(pages.load_time) ,0) AS avg_page_load_time + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + result = {**result, **row} + + pg_sub_query[-1] = "resources.duration > 0" + pg_sub_query.append("resources.type= %(type)s") + pg_query = f"""\ + SELECT COALESCE(AVG(resources.duration),0) AS avg + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'img', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + row = cur.fetchone() + result = {**result, "avg_image_load_time": row["avg"]} + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "type": 'fetch', "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + + row = cur.fetchone() + result = {**result, "avg_request_load_time": row["avg"]} + + return result + + +@dev.timed +def get_user_activity(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + with pg_client.PostgresClient() as cur: + row = __get_user_activity(cur, project_id, startTimestamp, endTimestamp, **args) + results = helper.dict_to_camel_case(row) + diff = endTimestamp - startTimestamp + endTimestamp = startTimestamp + startTimestamp = endTimestamp - diff + row = __get_user_activity(cur, project_id, startTimestamp, endTimestamp, **args) + + previous = helper.dict_to_camel_case(row) + for key in previous: + results[key + "Progress"] = helper.__progress(old_val=previous[key], new_val=results[key]) + return results + + +def __get_user_activity(cur, project_id, startTimestamp, endTimestamp, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + pg_query = f"""\ + SELECT COALESCE(CEIL(AVG(NULLIF(sessions.pages_count,0))),0) AS avg_visited_pages, + COALESCE(AVG(NULLIF(sessions.duration,0)),0) AS avg_session_duration + FROM public.sessions + WHERE {" AND ".join(pg_sub_query)};""" + params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, + **__get_constraint_values(args)} + + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return row + + +@dev.timed +def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=7, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("resources.type = 'img'") + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + pg_sub_query_chart.append("resources.type = 'img'") + pg_sub_query_chart.append("resources.url = %(url)s") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT resources.url, + COALESCE(AVG(NULLIF(resources.duration,0)),0) AS avg_duration, + COUNT(resources.session_id) AS sessions_count + FROM events.resources INNER JOIN sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY resources.url + ORDER BY avg_duration DESC LIMIT 10;""" + + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + urls = [row["url"] for row in rows] + + charts = {} + for url in urls: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(AVG(duration), 0) AS avg_duration + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT resources.duration + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, "url": url, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + r = cur.fetchall() + charts[url] = helper.list_to_camel_case(r) + for i in range(len(rows)): + rows[i]["sessions"] = rows[i].pop("sessions_count") + rows[i] = helper.dict_to_camel_case(rows[i]) + rows[i]["chart"] = charts[rows[i]["url"]] + + return sorted(rows, key=lambda k: k["sessions"], reverse=True) + + +@dev.timed +def __get_performance_constraint(l): + if len(l) == 0: + return "" + l = [s.decode('UTF-8').replace("%", "%%") for s in l] + return f"AND ({' OR '.join(l)})" + + +@dev.timed +def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), + density=19, resources=None, **args): + step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1) + location_constraints = [] + img_constraints = [] + request_constraints = [] + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + + img_constraints_vals = {} + location_constraints_vals = {} + request_constraints_vals = {} + + if resources and len(resources) > 0: + for r in resources: + if r["type"] == "IMG": + img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s") + img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value'] + elif r["type"] == "LOCATION": + location_constraints.append(f"pages.path = %(val_{len(location_constraints)})s") + location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value'] + else: + request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s") + request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] + params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT + generated_timestamp AS timestamp, + COALESCE(AVG(NULLIF(resources.duration,0)),0) AS avg_image_load_time + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT resources.duration + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + AND resources.type = 'img' + {(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""} + ) AS resources ON (TRUE) + GROUP BY timestamp + ORDER BY timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **img_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() + images = helper.list_to_camel_case(rows) + pg_query = f"""SELECT + generated_timestamp AS timestamp, + COALESCE(AVG(NULLIF(resources.duration,0)),0) AS avg_request_load_time + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT resources.duration + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + AND resources.type = 'fetch' + {(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""} + ) AS resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {**params, **request_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() + requests = helper.list_to_camel_case(rows) + + pg_query = f"""SELECT + generated_timestamp AS timestamp, + COALESCE(AVG(NULLIF(pages.load_time ,0)),0) AS avg_page_load_time + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT pages.load_time + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + {(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""} + ) AS pages ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + + cur.execute(cur.mogrify(pg_query, {**params, **location_constraints_vals, **__get_constraint_values(args)})) + rows = cur.fetchall() + pages = helper.list_to_camel_case(rows) + + rows = helper.merge_lists_by_key(helper.merge_lists_by_key(pages, requests, "timestamp"), images, "timestamp") + + return {"chart": rows} + + +RESOURCS_TYPE_TO_DB_TYPE = { + "img": "IMG", + "fetch": "REQUEST", + "stylesheet": "STYLESHEET", + "script": "SCRIPT", + "other": "OTHER", + "media": "MEDIA" +} + + +def __get_resource_type_from_db_type(db_type): + db_type = db_type.lower() + return RESOURCS_TYPE_TO_DB_TYPE.get(db_type, db_type) + + +def __get_resource_db_type_from_type(resource_type): + resource_type = resource_type.upper() + return {v: k for k, v in RESOURCS_TYPE_TO_DB_TYPE.items()}.get(resource_type, resource_type) + + +@dev.timed +def search(text, resource_type, project_id, performance=False, pages_only=False, events_only=False, + metadata=False, key=None, platform=None): + if not resource_type: + data = [] + if metadata: + resource_type = "METADATA" + elif pages_only or performance: + resource_type = "LOCATION" + else: + resource_type = "ALL" + data.extend(search(text=text, resource_type=resource_type, project_id=project_id, + performance=performance, pages_only=pages_only, events_only=events_only, key=key, + platform=platform)) + return data + + pg_sub_query = __get_constraints(project_id=project_id, time_constraint=False, duration=True, + data={} if platform is None else {"platform": platform}) + + if resource_type == "ALL" and not pages_only and not events_only: + pg_sub_query.append("url_hostpath ILIKE %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT key, value + FROM ( SELECT DISTINCT ON (url) ROW_NUMBER() OVER (PARTITION BY type ORDER BY url) AS r, + url AS value, + type AS key + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + ORDER BY url, type ASC) AS ranked_values + WHERE ranked_values.r<=5;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text)})) + rows = cur.fetchall() + rows = [{"value": i["value"], "type": __get_resource_type_from_db_type(i["key"])} for i in rows] + elif resource_type == "ALL" and events_only: + with pg_client.PostgresClient() as cur: + pg_query = f"""(SELECT DISTINCT label AS value, 'INPUT' AS key + FROM events.inputs INNER JOIN public.sessions USING(session_id) + WHERE {" AND ".join(pg_sub_query)} AND positionUTF8(lowerUTF8(label), %(value)s) != 0 + LIMIT 10) + UNION ALL + (SELECT DISTINCT label AS value, 'CLICK' AS key + FROM events.clicks INNER JOIN public.sessions USING(session_id) + WHERE {" AND ".join(pg_sub_query)} AND positionUTF8(lowerUTF8(label), %(value)s) != 0 + LIMIT 10) + UNION ALL + (SELECT DISTINCT path AS value, 'LOCATION' AS key + FROM events.pages INNER JOIN public.sessions USING(session_id) + WHERE {" AND ".join(pg_sub_query)} AND positionUTF8(url_path, %(value)s) != 0 + LIMIT 10);""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "value": helper.string_to_sql_like(text.lower()), + "platform_0": platform})) + rows = cur.fetchall() + rows = [{"value": i["value"], "type": i["key"]} for i in rows] + elif resource_type in ['IMG', 'REQUEST', 'STYLESHEET', 'OTHER', 'SCRIPT'] and not pages_only: + pg_sub_query.append("url_hostpath ILIKE %(value)s") + pg_sub_query.append(f"resources.type = '{__get_resource_db_type_from_type(resource_type)}'") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT + DISTINCT url_hostpath AS value, + %(resource_type)s AS type + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + LIMIT 10;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "value": helper.string_to_sql_like(text), + "resource_type": resource_type, + "platform_0": platform})) + rows = cur.fetchall() + elif resource_type == 'LOCATION': + with pg_client.PostgresClient() as cur: + pg_sub_query.append("path ILIKE %(value)s") + pg_query = f"""SELECT + DISTINCT path AS value, + 'LOCATION' AS type + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + LIMIT 10;""" + print(cur.mogrify(pg_query, {"project_id": project_id, + "value": helper.string_to_sql_like(text), + "platform_0": platform})) + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "value": helper.string_to_sql_like(text), + "platform_0": platform})) + rows = cur.fetchall() + elif resource_type == "INPUT": + with pg_client.PostgresClient() as cur: + pg_sub_query.append("label ILIKE %(value)s") + pg_query = f"""SELECT DISTINCT label AS value, 'INPUT' AS type + FROM events.inputs INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + LIMIT 10;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "value": helper.string_to_sql_like(text), + "platform_0": platform})) + rows = cur.fetchall() + elif resource_type == "CLICK": + with pg_client.PostgresClient() as cur: + pg_sub_query.append("label ILIKE %(value)s") + pg_query = f"""SELECT DISTINCT label AS value, 'CLICK' AS key + FROM events.clicks INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + LIMIT 10;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "value": helper.string_to_sql_like(text), + "platform_0": platform})) + rows = cur.fetchall() + elif resource_type == "METADATA": + if key and len(key) > 0 and key in {**METADATA_FIELDS, **SESSIONS_META_FIELDS}.keys(): + if key in METADATA_FIELDS.keys(): + column = METADATA_FIELDS[key] + pg_sub_query.append(f"{METADATA_FIELDS[key]} ILIKE %(value)s") + else: + column = SESSIONS_META_FIELDS[key] + pg_sub_query.append(f"{SESSIONS_META_FIELDS[key]} ILIKE %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT DISTINCT {column} AS value, + %(key)s AS key + FROM sessions + WHERE {" AND ".join(pg_sub_query)} + LIMIT 10;""" + cur.execute(cur.mogrify(pg_query, + {"project_id": project_id, "value": helper.string_to_sql_like(text), "key": key, + "platform_0": platform})) + rows = cur.fetchall() + else: + with pg_client.PostgresClient() as cur: + pg_query = [] + for k in METADATA_FIELDS.keys(): + pg_query.append(f"""(SELECT DISTINCT sessions.{METADATA_FIELDS[k]} AS value, + '{k}' AS key + FROM public.sessions + WHERE {" AND ".join(pg_sub_query)} + AND {METADATA_FIELDS[k]} ILIKE %(value)s + LIMIT 10)""") + for k in SESSIONS_META_FIELDS.keys(): + if k in ["platform", "country"]: + continue + pg_query.append(f"""(SELECT DISTINCT sessions.{SESSIONS_META_FIELDS[k]} AS value, + '{k}' AS key + FROM sessions + WHERE {" AND ".join(pg_sub_query)} + AND sessions.{SESSIONS_META_FIELDS[k]} ILIKE %(value)s + LIMIT 10)""") + pg_query = " UNION ALL ".join(pg_query) + cur.execute(cur.mogrify(pg_query, + {"project_id": project_id, "value": helper.string_to_sql_like(text), + "key": key, + "platform_0": platform})) + rows = cur.fetchall() + else: + return [] + return [helper.dict_to_camel_case(row) for row in rows] + + +@dev.timed +def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, data=args) + pg_sub_query.append("resources.success = FALSE") + pg_sub_query_chart.append("resources.success = FALSE") + pg_sub_query.append("resources.type != 'fetch'") + pg_sub_query_chart.append("resources.type != 'fetch'") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT + resources.url_hostpath AS url, + COUNT(resources.session_id) AS sessions + FROM events.resources INNER JOIN sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY url_hostpath + ORDER BY sessions DESC + LIMIT 10;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + + rows = cur.fetchall() + # rows = [{"url": i["key"], "sessions": i["doc_count"]} for i in rows] + if len(rows) == 0: + return [] + pg_sub_query.append("resources.url_hostpath = %(value)s") + pg_query = f"""SELECT generated_timestamp AS timestamp, + COUNT(resources.session_id) AS count, + MAX(resources.timestamp) AS max_datatime + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT resources.session_id, + resources.timestamp + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + for e in rows: + e["startedAt"] = startTimestamp + e["startTimestamp"] = startTimestamp + e["endTimestamp"] = endTimestamp + + cur.execute(cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": e["url"], + **__get_constraint_values(args)})) + r = cur.fetchall() + e["endedAt"] = r[-1]["max_datatime"] + e["chart"] = [{"timestamp": i["timestamp"], "count": i["count"]} for i in r] + return rows + + +@dev.timed +def get_network(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + resources.url_hostpath, + COUNT(resources.session_id) AS doc_count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT resources.session_id, + resources.url_hostpath + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS resources ON (TRUE) + GROUP BY generated_timestamp, resources.url_hostpath + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {"step_size": step_size, "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + r = cur.fetchall() + results = [] + + i = 0 + while i < len(r): + results.append({"timestamp": r[i]["timestamp"], "domains": []}) + i += 1 + while i < len(r) and r[i]["timestamp"] == results[-1]["timestamp"]: + results[-1]["domains"].append({r[i]["url_hostpath"]: r[i]["doc_count"]}) + i += 1 + + return {"startTimestamp": startTimestamp, "endTimestamp": endTimestamp, "chart": results} + + +KEYS = { + 'startTimestamp': args_transformer.int_arg, + 'endTimestamp': args_transformer.int_arg, + 'density': args_transformer.int_arg, + 'performanceDensity': args_transformer.int_arg, + 'platform': args_transformer.string +} + + +def dashboard_args(params): + args = {} + if params is not None: + for key in params.keys(): + if key in KEYS.keys(): + args[key] = KEYS[key](params.get(key)) + return args + + +@dev.timed +def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), + density=19, type=None, url=None, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + if type is not None: + pg_sub_query.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'") + pg_sub_query_chart.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'") + if url is not None: + pg_sub_query.append(f"resources.url = %(value)s") + pg_sub_query_chart.append(f"resources.url = %(value)s") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(AVG(NULLIF(resources.duration,0)),0) AS avg + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT resources.duration + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + params = {"step_size": step_size, "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": url, "type": type, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + if len(rows) > 0: + pg_query = f"""SELECT COALESCE(AVG(NULLIF(resources.duration,0)),0) AS avg + FROM events.resources INNER JOIN sessions USING(session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, params)) + avg = cur.fetchone()["avg"] + else: + avg = 0 + return {"avg": avg, "chart": rows} + + +@dev.timed +def get_pages_dom_build_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=19, url=None, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + if url is not None: + pg_sub_query.append(f"pages.path = %(value)s") + pg_sub_query_chart.append(f"pages.path = %(value)s") + pg_sub_query.append("pages.dom_building_time>0") + pg_sub_query_chart.append("pages.dom_building_time>0") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(avg, 0) AS avg, chart + FROM (SELECT AVG(dom_building_time) + FROM public.sessions + INNER JOIN events.pages USING (session_id) + WHERE {" AND ".join(pg_sub_query)}) AS avg + LEFT JOIN + (SELECT jsonb_agg(chart) AS chart + FROM ( + SELECT generated_timestamp AS timestamp, + COALESCE(AVG(dom_building_time), 0) AS avg + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT pages.dom_building_time + FROM public.sessions + INNER JOIN events.pages USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS sessionsBD ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp) AS chart) AS chart ON (TRUE);""" + params = {"step_size": step_size, "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": url, **__get_constraint_values(args)} + + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return row + + +@dev.timed +def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), type="all", density=19, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + + if type is not None and type.upper() != "ALL": + sq = f"resources.type = '{__get_resource_db_type_from_type(type.upper())}'" + else: + sq = "resources.type != 'fetch'" + pg_sub_query.append(sq) + pg_sub_query_chart.append(sq) + pg_sub_query_chart.append("resources.duration IS NOT NULL") + pg_sub_query_chart.append("resources.duration>0") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT * + FROM (SELECT regexp_replace(resources.url_hostpath, '^.*/', '') AS name, + AVG(NULLIF(resources.duration, 0)) AS avg + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY name + ORDER BY avg DESC + LIMIT 10) AS main_list + INNER JOIN LATERAL ( + SELECT url, type + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND resources.url_hostpath ILIKE '%%' || main_list.name + ORDER BY resources.duration DESC + LIMIT 1 + ) AS resource_details ON (TRUE) + INNER JOIN LATERAL ( + SELECT JSONB_AGG(chart_details) AS chart + FROM ( + SELECT generated_timestamp AS timestamp, + COALESCE(AVG(resources.duration), 0) AS avg + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT resources.duration + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + AND resources.url_hostpath ILIKE '%%' || main_list.name + ) AS resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp + ) AS chart_details + ) AS chart_details ON (TRUE);""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "step_size": step_size, + **__get_constraint_values(args)})) + rows = cur.fetchall() + for r in rows: + r["type"] = __get_resource_type_from_db_type(r["type"]) + return rows + + +@dev.timed +def get_sessions_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT user_country, COUNT(session_id) AS count + FROM public.sessions + WHERE {" AND ".join(pg_sub_query)} + GROUP BY user_country + ORDER BY user_country;""" + cur.execute(cur.mogrify(pg_query, + {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + return {"count": sum(i["count"] for i in rows), "chart": helper.list_to_camel_case(rows)} + + +@dev.timed +def get_speed_index_location(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("pages.speed_index IS NOT NULL") + pg_sub_query.append("pages.speed_index>0") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT sessions.user_country, AVG(pages.speed_index) AS avg + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY sessions.user_country + ORDER BY avg,sessions.user_country;""" + params = {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + if len(rows) > 0: + pg_query = f"""SELECT AVG(pages.speed_index) AS avg + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, params)) + avg = cur.fetchone()["avg"] + else: + avg = 0 + return {"avg": avg, "chart": helper.list_to_camel_case(rows)} + + +@dev.timed +def get_pages_response_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=7, url=None, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("pages.response_time IS NOT NULL") + pg_sub_query.append("pages.response_time>0") + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, + data=args) + pg_sub_query_chart.append("pages.response_time IS NOT NULL") + pg_sub_query_chart.append("pages.response_time>0") + + if url is not None: + pg_sub_query_chart.append(f"url = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(AVG(pages.response_time),0) AS avg + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT response_time + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS pages ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + params = {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": url, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + pg_query = f"""SELECT COALESCE(AVG(pages.response_time),0) AS avg + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, params)) + avg = cur.fetchone()["avg"] + return {"avg": avg, "chart": rows} + + +@dev.timed +def get_pages_response_time_distribution(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=20, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("pages.response_time IS NOT NULL") + pg_sub_query.append("pages.response_time>0") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT pages.response_time AS response_time, + COUNT(pages.session_id) AS count + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY response_time + ORDER BY pages.response_time;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + pg_query = f"""SELECT COALESCE(AVG(pages.response_time),0) AS avg + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + avg = cur.fetchone()["avg"] + quantiles_keys = [50, 90, 95, 99] + pg_query = f"""SELECT pages.response_time AS value + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + response_times = cur.fetchall() + response_times = [i["value"] for i in response_times] + if len(response_times) > 0: + quantiles = __quantiles(a=response_times, + q=[i / 100 for i in quantiles_keys], + interpolation='higher') + else: + quantiles = [0 for i in range(len(quantiles_keys))] + result = { + "avg": avg, + "total": sum(r["count"] for r in rows), + "chart": [], + "percentiles": [{ + "percentile": float(v), + "responseTime": int(quantiles[i]) + } for i, v in enumerate(quantiles_keys) + ], + "extremeValues": [{"count": 0}] + } + rows = helper.list_to_camel_case(rows) + _99 = result["percentiles"][-1]["responseTime"] + extreme_values_first_index = -1 + for i, r in enumerate(rows): + if r["responseTime"] > _99: + extreme_values_first_index = i + break + + if extreme_values_first_index >= 0: + extreme_values_first_index += 1 + result["extremeValues"][0]["count"] = sum(r["count"] for r in rows[extreme_values_first_index:]) + rows = rows[:extreme_values_first_index] + + # ------- Merge points to reduce chart length till density + if density < len(quantiles_keys): + density = len(quantiles_keys) + + while len(rows) > density: + true_length = len(rows) + rows_partitions = [] + offset = 0 + for p in result["percentiles"]: + rows_partitions.append([]) + for r in rows[offset:]: + if r["responseTime"] < p["responseTime"]: + rows_partitions[-1].append(r) + offset += 1 + else: + break + rows_partitions.append(rows[offset:]) + + largest_partition = 0 + for i in range(len(rows_partitions)): + if len(rows_partitions[i]) > len(rows_partitions[largest_partition]): + largest_partition = i + + if len(rows_partitions[largest_partition]) <= 2: + break + # computing lowest merge diff + diff = rows[-1]["responseTime"] + for i in range(1, len(rows_partitions[largest_partition]) - 1, 1): + v1 = rows_partitions[largest_partition][i] + v2 = rows_partitions[largest_partition][i + 1] + if (v2["responseTime"] - v1["responseTime"]) < diff: + diff = v2["responseTime"] - v1["responseTime"] + + i = 1 + while i < len(rows_partitions[largest_partition]) - 1 and true_length > density - 1: + v1 = rows_partitions[largest_partition][i] + v2 = rows_partitions[largest_partition][i + 1] + if (v2["responseTime"] - v1["responseTime"]) == diff: + rows_partitions[largest_partition][i]["count"] += v2["count"] + rows_partitions[largest_partition][i]["responseTime"] = v2["responseTime"] + del rows_partitions[largest_partition][i + 1] + true_length -= 1 + else: + i += 1 + + rows = [r for rp in rows_partitions for r in rp] + + if extreme_values_first_index == len(rows): + rows.append({"count": 0, "responseTime": rows[-1]["responseTime"] + 10}) + + result["chart"] = rows + return result + + +@dev.timed +def get_busiest_time_of_day(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT + (EXTRACT(HOUR FROM TO_TIMESTAMP(sessions.start_ts))::INTEGER / 2) * 2 AS hour, + COUNT(sessions.session_id) AS count + FROM public.sessions + WHERE {" AND ".join(pg_sub_query)} + GROUP BY hour + ORDER BY hour ASC;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + return rows + + +@dev.timed +def get_top_metrics(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT (SELECT COALESCE(AVG(pages.response_time),0) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} AND pages.response_time IS NOT NULL AND pages.response_time>0) AS avg_response_time, + (SELECT COUNT(pages.session_id) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)}) AS count_requests, + (SELECT COALESCE(AVG(pages.first_paint_time),0) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} AND pages.first_paint_time IS NOT NULL AND pages.first_paint_time>0) AS avg_first_paint, + (SELECT COALESCE(AVG(pages.dom_content_loaded_time),0) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} AND pages.dom_content_loaded_time IS NOT NULL AND pages.dom_content_loaded_time>0) AS avg_dom_content_loaded, + (SELECT COALESCE(AVG(pages.ttfb),0) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} AND pages.ttfb IS NOT NULL AND pages.ttfb>0) AS avg_till_first_bit, + (SELECT COALESCE(AVG(pages.time_to_interactive),0) FROM events.pages INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} AND pages.time_to_interactive IS NOT NULL AND pages.time_to_interactive >0) AS avg_time_to_interactive;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +@dev.timed +def get_time_to_render(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=7, url=None, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, + data=args) + pg_sub_query.append("pages.visually_complete>0") + pg_sub_query_chart.append("pages.visually_complete>0") + if url is not None: + pg_sub_query_chart.append("pages.path = %(value)s") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT COALESCE(avg,0) AS avg, chart + FROM (SELECT AVG(pages.visually_complete) + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)}) AS avg + LEFT JOIN + (SELECT jsonb_agg(chart) AS chart + FROM (SELECT generated_timestamp AS timestamp, + COALESCE(AVG(visually_complete), 0) AS avg + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT pages.visually_complete + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS pages ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp) AS chart) AS chart ON(TRUE);""" + params = {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, "value": url, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + row = cur.fetchone() + return row + + +@dev.timed +def get_impacted_sessions_by_slow_pages(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), value=None, density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, + data=args) + pg_sub_query.append("pages.response_time IS NOT NULL") + pg_sub_query_chart.append("pages.response_time IS NOT NULL") + pg_sub_query.append("pages.response_time>0") + pg_sub_query_chart.append("pages.response_time>0") + if value is not None: + pg_sub_query.append("pages.path = %(value)s") + pg_sub_query_chart.append("pages.path = %(value)s") + pg_sub_query_chart.append("avg_response_time>0") + pg_sub_query_chart.append("pages.response_time>avg_response_time*2") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COUNT(pages.session_id) AS count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN ( SELECT AVG(pages.response_time) AS avg_response_time + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + ) AS avg_response_time ON (avg_response_time>0) + LEFT JOIN LATERAL ( SELECT DISTINCT pages.session_id + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS pages ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "value": value, **__get_constraint_values(args)})) + rows = cur.fetchall() + return rows + + +@dev.timed +def get_memory_consumption(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(AVG(performance.avg_used_js_heap_size),0) AS avg_used_js_heap_size + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT avg_used_js_heap_size + FROM events.performance INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS performance ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp ASC;""" + params = {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + pg_query = f"""SELECT COALESCE(AVG(performance.avg_used_js_heap_size),0) AS avg + FROM events.performance INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, params)) + avg = cur.fetchone()["avg"] + return {"avgUsedJsHeapSize": avg, "chart": helper.list_to_camel_case(rows)} + + +@dev.timed +def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(AVG(performance.avg_cpu),0) AS avg_cpu + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT avg_cpu + FROM events.performance INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS performance ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp ASC;""" + params = {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + pg_query = f"""SELECT COALESCE(AVG(performance.avg_cpu),0) AS avg + FROM events.performance INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, params)) + avg = cur.fetchone()["avg"] + return {"avgCpu": avg, "chart": helper.list_to_camel_case(rows)} + + +@dev.timed +def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(AVG(NULLIF(performance.avg_fps,0)),0) AS avg_fps + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT avg_fps + FROM events.performance INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS performance ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp ASC;""" + params = {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + pg_query = f"""SELECT COALESCE(AVG(NULLIF(performance.avg_fps,0)),0) AS avg + FROM events.performance INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, params)) + avg = cur.fetchone()["avg"] + return {"avgFps": avg, "chart": helper.list_to_camel_case(rows)} + + +@dev.timed +def get_crashes(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("m_issues.type = 'crash'") + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + pg_sub_query_chart.append("m_issues.type = 'crash'") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COUNT(sessions) AS count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( + SELECT sessions.session_id + FROM public.sessions + INNER JOIN events_common.issues USING (session_id) + INNER JOIN public.issues AS m_issues USING (issue_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + **__get_constraint_values(args)})) + rows = cur.fetchall() + pg_query = f"""SELECT b.user_browser AS browser, + sum(bv.count) AS total, + JSONB_AGG(bv) AS versions + FROM ( + SELECT sessions.user_browser + FROM public.sessions + INNER JOIN events_common.issues USING (session_id) + INNER JOIN public.issues AS m_issues USING (issue_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY sessions.user_browser + ORDER BY COUNT(sessions.session_id) DESC + LIMIT 3 + ) AS b + INNER JOIN LATERAL + ( + SELECT sessions.user_browser_version AS version, + COUNT(sessions.session_id) AS count + FROM sessions + INNER JOIN events_common.issues USING (session_id) + INNER JOIN public.issues AS m_issues USING (issue_id) + WHERE {" AND ".join(pg_sub_query)} + AND sessions.user_browser = b.user_browser + GROUP BY sessions.user_browser_version + ORDER BY count DESC + ) AS bv ON (TRUE) + GROUP BY b.user_browser + ORDER BY b.user_browser;""" + cur.execute(cur.mogrify(pg_query, {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + **__get_constraint_values(args)})) + browsers = cur.fetchall() + total = sum(r["total"] for r in browsers) + for r in browsers: + r["percentage"] = r["total"] / (total / 100) + versions = [] + for v in r["versions"][:3]: + versions.append({v["version"]: v["count"] / (r["total"] / 100)}) + r["versions"] = versions + + return {"chart": rows, "browsers": browsers} + + +def __get_neutral(rows, add_All_if_empty=True): + neutral = {l: 0 for l in [i for k in [list(v.keys()) for v in rows] for i in k]} + if add_All_if_empty and len(neutral.keys()) == 0: + neutral = {"All": 0} + return neutral + + +def __merge_rows_with_neutral(rows, neutral): + for i in range(len(rows)): + rows[i] = {**neutral, **rows[i]} + return rows + + +@dev.timed +def get_domains_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=6, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, + data=args) + pg_sub_query_chart.append("resources.status/100 = %(status_code)s") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(JSONB_AGG(resources) FILTER ( WHERE resources IS NOT NULL ), '[]'::JSONB) AS keys + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.session_id) AS count + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + GROUP BY url_host + ORDER BY count DESC + LIMIT 5 + ) AS resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + params = {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "step_size": step_size, + "status_code": 4, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + rows = __nested_array_to_dict_array(rows) + neutral = __get_neutral(rows) + rows = __merge_rows_with_neutral(rows, neutral) + + result = {"4xx": rows} + params["status_code"] = 5 + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + rows = __nested_array_to_dict_array(rows) + neutral = __get_neutral(rows) + rows = __merge_rows_with_neutral(rows, neutral) + result["5xx"] = rows + return result + + +@dev.timed +def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=6, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, + data=args) + pg_sub_query_chart.append("resources.status/100 = %(status_code)s") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(JSONB_AGG(resources) FILTER ( WHERE resources IS NOT NULL ), '[]'::JSONB) AS keys + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.session_id) AS count + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + GROUP BY url_host + ORDER BY count DESC + LIMIT 5 + ) AS resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + params = {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "step_size": step_size, + "status_code": 4, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + rows = __nested_array_to_dict_array(rows) + neutral = __get_neutral(rows) + rows = __merge_rows_with_neutral(rows, neutral) + + return rows + + +@dev.timed +def get_domains_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=6, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, + data=args) + pg_sub_query_chart.append("resources.status/100 = %(status_code)s") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(JSONB_AGG(resources) FILTER ( WHERE resources IS NOT NULL ), '[]'::JSONB) AS keys + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT resources.url_host, COUNT(resources.session_id) AS count + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + GROUP BY url_host + ORDER BY count DESC + LIMIT 5 + ) AS resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + params = {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "step_size": step_size, + "status_code": 5, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + rows = __nested_array_to_dict_array(rows) + neutral = __get_neutral(rows) + rows = __merge_rows_with_neutral(rows, neutral) + + return rows + + +def __nested_array_to_dict_array(rows, key="url_host", value="count"): + for r in rows: + for i in range(len(r["keys"])): + r[r["keys"][i][key]] = r["keys"][i][value] + r.pop("keys") + return rows + + +@dev.timed +def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("resources.duration IS NOT NULL") + pg_sub_query.append("resources.duration>0") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT + resources.url_host AS domain, + AVG(resources.duration) AS avg + FROM events.resources INNER JOIN sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY resources.url_host + ORDER BY avg DESC + LIMIT 5;""" + params = {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + if len(rows) > 0: + pg_query = f"""SELECT AVG(resources.duration) AS avg + FROM events.resources INNER JOIN sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)};""" + cur.execute(cur.mogrify(pg_query, params)) + avg = cur.fetchone()["avg"] + else: + avg = 0 + return {"avg": avg, "partition": rows} + + +@dev.timed +def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("resources.success = FALSE") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT + resources.url_host AS domain, + COUNT(resources.session_id) AS errors_count + FROM events.resources INNER JOIN sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY resources.url_host + ORDER BY errors_count DESC + LIMIT 10;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + return helper.list_to_camel_case(rows) + + +@dev.timed +def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), + platform=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query2 = pg_sub_query[:] + pg_sub_query2.append("sessions.user_browser = b.user_browser") + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT b.user_browser AS browser, + b.count, + jsonb_agg(bv) AS versions + FROM ( + SELECT sessions.user_browser, + COUNT(sessions.session_id) AS count + FROM sessions + WHERE {" AND ".join(pg_sub_query)} + GROUP BY sessions.user_browser + ORDER BY count DESC + LIMIT 3 + ) AS b + INNER JOIN LATERAL + ( + SELECT sessions.user_browser_version, + COUNT(sessions.session_id) AS count + FROM public.sessions + WHERE {" AND ".join(pg_sub_query2)} + GROUP BY sessions.user_browser, + sessions.user_browser_version + ORDER BY count DESC + LIMIT 3 + ) AS bv ON (TRUE) + GROUP BY b.user_browser, b.count + ORDER BY b.count DESC;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + for r in rows: + for j in r["versions"]: + r[j["user_browser_version"]] = j["count"] + r.pop("versions") + return {"count": sum(i["count"] for i in rows), "chart": rows} + + +@dev.timed +def get_calls_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), + platform=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("resources.type = 'fetch'") + pg_sub_query.append("resources.status/100 != 2") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT resources.method, + resources.url_hostpath, + COUNT(resources.session_id) AS all_requests, + SUM(CASE WHEN resources.status/100 = 4 THEN 1 ELSE 0 END) AS _4xx, + SUM(CASE WHEN resources.status/100 = 5 THEN 1 ELSE 0 END) AS _5xx + FROM events.resources INNER JOIN sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY resources.method, resources.url_hostpath + ORDER BY (4 + 5), 3 DESC + LIMIT 50;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + return helper.list_to_camel_case(rows) + + +@dev.timed +def get_calls_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), + platform=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("resources.type = 'fetch'") + pg_sub_query.append("resources.status/100 = 4") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT resources.method, + resources.url_hostpath, + COUNT(resources.session_id) AS all_requests + FROM events.resources INNER JOIN sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY resources.method, resources.url_hostpath + ORDER BY all_requests DESC + LIMIT 10;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + return helper.list_to_camel_case(rows) + + +@dev.timed +def get_calls_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), + platform=None, **args): + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query.append("resources.type = 'fetch'") + pg_sub_query.append("resources.status/100 = 5") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT resources.method, + resources.url_hostpath, + COUNT(resources.session_id) AS all_requests + FROM events.resources INNER JOIN sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY resources.method, resources.url_hostpath + ORDER BY all_requests DESC + LIMIT 10;""" + cur.execute(cur.mogrify(pg_query, {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + return helper.list_to_camel_case(rows) + + +@dev.timed +def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), + platform=None, density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(SUM(CASE WHEN type = 'fetch' AND status / 100 = 4 THEN 1 ELSE 0 END), 0) AS _4xx, + COALESCE(SUM(CASE WHEN type = 'fetch' AND status / 100 = 5 THEN 1 ELSE 0 END), 0) AS _5xx, + COALESCE(SUM(CASE WHEN type = 'js_exception' THEN 1 ELSE 0 END), 0) AS js, + COALESCE(SUM(CASE WHEN type != 'fetch' AND type != 'js_exception' THEN 1 ELSE 0 END), 0) AS integrations + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ((SELECT status, 'fetch' AS type + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + AND resources.timestamp >= %(startTimestamp)s - %(step_size)s + AND resources.timestamp < %(endTimestamp)s + %(step_size)s + AND resources.type = 'fetch' + AND resources.status > 200) + UNION ALL + (SELECT 0 AS status, m_errors.source::text AS type + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + AND errors.timestamp >= %(startTimestamp)s - %(step_size)s + AND errors.timestamp < %(endTimestamp)s + %(step_size)s) + ) AS errors_partition ON (TRUE) + GROUP BY timestamp + ORDER BY timestamp;""" + params = {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + cur.execute(cur.mogrify(pg_query, params)) + rows = cur.fetchall() + rows = helper.list_to_camel_case(rows) + return rows + + +@dev.timed +def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + + params = {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)} + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COUNT(resources.session_id) AS total, + SUM(CASE WHEN resources.type='fetch' THEN 1 ELSE 0 END) AS xhr + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL (SELECT resources.session_id, resources.type + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)}) AS resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, params)) + actions = cur.fetchall() + pg_sub_query_chart.append("pages.response_end IS NOT NULL") + pg_sub_query_chart.append("pages.response_end>0") + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(AVG(pages.response_end),0) AS avg_response_end + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL (SELECT pages.response_end + FROM events.pages INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)}) AS pages ON(TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, params)) + response_end = cur.fetchall() + return helper.list_to_camel_case(__merge_charts(response_end, actions)) + + +@dev.timed +def get_impacted_sessions_by_js_errors(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query = __get_constraints(project_id=project_id, data=args) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + pg_sub_query.append("m_errors.source = 'js_exception'") + pg_sub_query.append("m_errors.project_id = %(project_id)s") + pg_sub_query.append("errors.timestamp >= %(startTimestamp)s") + pg_sub_query.append("errors.timestamp < %(endTimestamp)s") + pg_sub_query_chart.append("m_errors.source = 'js_exception'") + pg_sub_query_chart.append("m_errors.project_id = %(project_id)s") + pg_sub_query_chart.append("errors.timestamp >= generated_timestamp") + pg_sub_query_chart.append("errors.timestamp < generated_timestamp+ %(step_size)s") + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT * + FROM (SELECT COUNT(DISTINCT errors.session_id) AS sessions_count + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + INNER JOIN public.sessions USING(session_id) + WHERE {" AND ".join(pg_sub_query)}) AS counts + LEFT JOIN + (SELECT jsonb_agg(chart) AS chart + FROM(SELECT generated_timestamp AS timestamp, + COALESCE(COUNT(session_id), 0) AS sessions_count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT DISTINCT errors.session_id + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp) AS chart) AS chart ON(TRUE);""" + cur.execute(cur.mogrify(pg_query, {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + **__get_constraint_values(args)})) + row_sessions = cur.fetchone() + pg_query = f"""SELECT * + FROM (SELECT COUNT(DISTINCT errors.error_id) AS errors_count + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + INNER JOIN public.sessions USING(session_id) + WHERE {" AND ".join(pg_sub_query)}) AS counts + LEFT JOIN + (SELECT jsonb_agg(chart) AS chart + FROM(SELECT generated_timestamp AS timestamp, + COALESCE(COUNT(error_id), 0) AS errors_count + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT DISTINCT errors.error_id + FROM events.errors + INNER JOIN public.errors AS m_errors USING (error_id) + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS errors ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp) AS chart) AS chart ON(TRUE);""" + cur.execute(cur.mogrify(pg_query, {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + **__get_constraint_values(args)})) + row_errors = cur.fetchone() + chart = __merge_charts(row_sessions.pop("chart"), row_errors.pop("chart")) + row_sessions = helper.dict_to_camel_case(row_sessions) + row_errors = helper.dict_to_camel_case(row_errors) + return {**row_sessions, **row_errors, "chart": chart} + + +@dev.timed +def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(jsonb_agg(resources_avg_count_by_type) + FILTER ( WHERE resources_avg_count_by_type IS NOT NULL ), '[]'::jsonb) AS types, + COALESCE(AVG(total_count), 0) AS avg_count_resources, + COALESCE(AVG(avg_time_to_render), 0) AS avg_time_to_render + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL (SELECT resources_count_by_session_by_type.type, + avg(resources_count_by_session_by_type.count) AS avg_count, + sum(resources_count_by_session_by_type.count) AS total_count + FROM (SELECT resources.type, COUNT(resources.url) AS count + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + GROUP BY resources.session_id, resources.type) AS resources_count_by_session_by_type + GROUP BY resources_count_by_session_by_type.type) AS resources_avg_count_by_type ON (TRUE) + LEFT JOIN LATERAL (SELECT AVG(visually_complete) AS avg_time_to_render + FROM events.pages + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + AND pages.visually_complete > 0) AS time_to_render ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + for r in rows: + r["types"] = {t["type"]: t["avg_count"] for t in r["types"]} + + return helper.list_to_camel_case(rows) + + +@dev.timed +def get_resources_count_by_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, + chart=True, data=args) + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + COALESCE(JSONB_AGG(t) FILTER (WHERE t IS NOT NULL), '[]'::JSONB) AS types + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL (SELECT resources.type, COUNT(resources.session_id) AS count + FROM events.resources INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + GROUP BY resources.type + ) AS t ON(TRUE) + GROUP BY timestamp + ORDER BY timestamp;""" + cur.execute(cur.mogrify(pg_query, {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + for r in rows: + for t in r["types"]: + r[t["type"]] = t["count"] + r.pop("types") + rows = __merge_rows_with_neutral(rows, {k: 0 for k in RESOURCS_TYPE_TO_DB_TYPE.keys()}) + return rows + + +@dev.timed +def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1), + endTimestamp=TimeUTC.now(), density=7, **args): + step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1) + pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, + data=args) + pg_sub_query_chart.append("resources.success = FALSE") + pg_sub_query = ["sessions.project_id =%(project_id)s", "rs.type IN ('fetch','script')"] + + with pg_client.PostgresClient() as cur: + pg_query = f"""SELECT generated_timestamp AS timestamp, + SUM(CASE WHEN first.url_host = sub_resources.url_host THEN 1 ELSE 0 END) AS first_party, + SUM(CASE WHEN first.url_host != sub_resources.url_host THEN 1 ELSE 0 END) AS third_party + FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp + LEFT JOIN ( + SELECT resources.url_host, + COUNT(resources.session_id) AS count + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE sessions.project_id = '1' + AND resources.type IN ('fetch', 'script') + AND sessions.start_ts > (EXTRACT(EPOCH FROM now() - INTERVAL '31 days') * 1000)::BIGINT + AND resources.timestamp > (EXTRACT(EPOCH FROM now() - INTERVAL '31 days') * 1000)::BIGINT + GROUP BY resources.url_host + ORDER BY count DESC + LIMIT 1 + ) AS first ON (TRUE) + LEFT JOIN LATERAL ( + SELECT resources.url_host + FROM events.resources + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS sub_resources ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""" + cur.execute(cur.mogrify(pg_query, {"step_size": step_size, + "project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, **__get_constraint_values(args)})) + rows = cur.fetchall() + return rows diff --git a/api/chalicelib/core/errors.py b/api/chalicelib/core/errors.py new file mode 100644 index 000000000..10ad52f59 --- /dev/null +++ b/api/chalicelib/core/errors.py @@ -0,0 +1,777 @@ +import json + +from chalicelib.utils import pg_client, helper, dev +from chalicelib.core import sourcemaps, sessions +from chalicelib.utils.TimeUTC import TimeUTC +from chalicelib.utils.metrics_helper import __get_step_size + + +def get(error_id, family=False): + if family: + return get_batch([error_id]) + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + "SELECT * FROM events.errors AS e INNER JOIN public.errors AS re USING(error_id) WHERE error_id = %(error_id)s;", + {"error_id": error_id}) + cur.execute(query=query) + result = cur.fetchone() + if result is not None: + result["stacktrace_parsed_at"] = TimeUTC.datetime_to_timestamp(result["stacktrace_parsed_at"]) + return helper.dict_to_camel_case(result) + + +def get_batch(error_ids): + if len(error_ids) == 0: + return [] + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """ + WITH RECURSIVE error_family AS ( + SELECT * + FROM public.errors + WHERE error_id IN %(error_ids)s + UNION + SELECT child_errors.* + FROM public.errors AS child_errors + INNER JOIN error_family ON error_family.error_id = child_errors.parent_error_id OR error_family.parent_error_id = child_errors.error_id + ) + SELECT * + FROM error_family;""", + {"error_ids": tuple(error_ids)}) + cur.execute(query=query) + return helper.list_to_camel_case(cur.fetchall()) + + +def __flatten_sort_key_count_version(data, merge_nested=False): + if data is None: + return [] + return sorted( + [ + { + "name": f'{o["name"]}@{v["version"]}', + "count": v["count"] + } for o in data for v in o["partition"] + ], + key=lambda o: o["count"], reverse=True) if merge_nested else \ + [ + { + "name": o["name"], + "count": o["count"], + } for o in data + ] + + +def __process_tags(row): + return [ + {"name": "browser", "partitions": __flatten_sort_key_count_version(data=row.get("browsers_partition"))}, + {"name": "browser.ver", + "partitions": __flatten_sort_key_count_version(data=row.pop("browsers_partition"), merge_nested=True)}, + {"name": "OS", "partitions": __flatten_sort_key_count_version(data=row.get("os_partition"))}, + {"name": "OS.ver", + "partitions": __flatten_sort_key_count_version(data=row.pop("os_partition"), merge_nested=True)}, + {"name": "device.family", "partitions": __flatten_sort_key_count_version(data=row.get("device_partition"))}, + {"name": "device", + "partitions": __flatten_sort_key_count_version(data=row.pop("device_partition"), merge_nested=True)}, + {"name": "country", "partitions": row.pop("country_partition")} + ] + + +def get_details(project_id, error_id, user_id, **data): + pg_sub_query24 = __get_basic_constraints(time_constraint=False, chart=True, step_size_name="step_size24") + pg_sub_query24.append("error_id = %(error_id)s") + pg_sub_query30 = __get_basic_constraints(time_constraint=False, chart=True, step_size_name="step_size30") + pg_sub_query30.append("error_id = %(error_id)s") + pg_basic_query = __get_basic_constraints(time_constraint=False) + pg_basic_query.append("error_id = %(error_id)s") + with pg_client.PostgresClient() as cur: + data["startDate24"] = TimeUTC.now(-1) + data["endDate24"] = TimeUTC.now() + data["startDate30"] = TimeUTC.now(-30) + data["endDate30"] = TimeUTC.now() + density24 = int(data.get("density24", 24)) + step_size24 = __get_step_size(data["startDate24"], data["endDate24"], density24, factor=1) + density30 = int(data.get("density30", 30)) + step_size30 = __get_step_size(data["startDate30"], data["endDate30"], density30, factor=1) + params = { + "startDate24": data['startDate24'], + "endDate24": data['endDate24'], + "startDate30": data['startDate30'], + "endDate30": data['endDate30'], + "project_id": project_id, + "userId": user_id, + "step_size24": step_size24, + "step_size30": step_size30, + "error_id": error_id} + + main_pg_query = f"""\ + SELECT error_id, + name, + message, + users, + sessions, + last_occurrence, + first_occurrence, + last_session_id, + browsers_partition, + os_partition, + device_partition, + country_partition, + chart24, + chart30 + FROM (SELECT error_id, + name, + message, + COUNT(DISTINCT user_uuid) AS users, + COUNT(DISTINCT session_id) AS sessions + FROM public.errors + INNER JOIN events.errors AS s_errors USING (error_id) + INNER JOIN public.sessions USING (session_id) + WHERE error_id = %(error_id)s + GROUP BY error_id, name, message) AS details + INNER JOIN (SELECT error_id, + MAX(timestamp) AS last_occurrence, + MIN(timestamp) AS first_occurrence + FROM events.errors + WHERE error_id = %(error_id)s + GROUP BY error_id) AS time_details USING (error_id) + INNER JOIN (SELECT error_id, + session_id AS last_session_id, + user_os, + user_os_version, + user_browser, + user_browser_version, + user_device, + user_device_type, + user_uuid + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE error_id = %(error_id)s + ORDER BY errors.timestamp DESC + LIMIT 1) AS last_session_details USING (error_id) + INNER JOIN (SELECT jsonb_agg(browser_details) AS browsers_partition + FROM (SELECT * + FROM (SELECT user_browser AS name, + COUNT(session_id) AS count + FROM events.errors + INNER JOIN sessions USING (session_id) + WHERE {" AND ".join(pg_basic_query)} + GROUP BY user_browser + ORDER BY count DESC) AS count_per_browser_query + INNER JOIN LATERAL (SELECT JSONB_AGG(version_details) AS partition + FROM (SELECT user_browser_version AS version, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_basic_query)} + AND sessions.user_browser = count_per_browser_query.name + GROUP BY user_browser_version + ORDER BY count DESC) AS version_details + ) AS browser_version_details ON (TRUE)) AS browser_details) AS browser_details ON (TRUE) + INNER JOIN (SELECT jsonb_agg(os_details) AS os_partition + FROM (SELECT * + FROM (SELECT user_os AS name, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_basic_query)} + GROUP BY user_os + ORDER BY count DESC) AS count_per_os_details + INNER JOIN LATERAL (SELECT jsonb_agg(count_per_version_details) AS partition + FROM (SELECT COALESCE(user_os_version,'unknown') AS version, COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_basic_query)} + AND sessions.user_os = count_per_os_details.name + GROUP BY user_os_version + ORDER BY count DESC) AS count_per_version_details + GROUP BY count_per_os_details.name ) AS os_version_details + ON (TRUE)) AS os_details) AS os_details ON (TRUE) + INNER JOIN (SELECT jsonb_agg(device_details) AS device_partition + FROM (SELECT * + FROM (SELECT user_device_type AS name, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_basic_query)} + GROUP BY user_device_type + ORDER BY count DESC) AS count_per_device_details + INNER JOIN LATERAL (SELECT jsonb_agg(count_per_device_v_details) AS partition + FROM (SELECT CASE + WHEN user_device = '' OR user_device ISNULL + THEN 'unknown' + ELSE user_device END AS version, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_basic_query)} + AND sessions.user_device_type = count_per_device_details.name + GROUP BY user_device + ORDER BY count DESC) AS count_per_device_v_details + GROUP BY count_per_device_details.name ) AS device_version_details + ON (TRUE)) AS device_details) AS device_details ON (TRUE) + INNER JOIN (SELECT jsonb_agg(count_per_country_details) AS country_partition + FROM (SELECT user_country AS name, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_basic_query)} + GROUP BY user_country + ORDER BY count DESC) AS count_per_country_details) AS country_details ON (TRUE) + INNER JOIN (SELECT jsonb_agg(chart_details) AS chart24 + FROM (SELECT generated_timestamp AS timestamp, + COUNT(session_id) AS count + FROM generate_series(%(startDate24)s, %(endDate24)s, %(step_size24)s) AS generated_timestamp + LEFT JOIN LATERAL (SELECT DISTINCT session_id + FROM events.errors + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query24)} + ) AS chart_details ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp) AS chart_details) AS chart_details24 ON (TRUE) + INNER JOIN (SELECT jsonb_agg(chart_details) AS chart30 + FROM (SELECT generated_timestamp AS timestamp, + COUNT(session_id) AS count + FROM generate_series(%(startDate30)s, %(endDate30)s, %(step_size30)s) AS generated_timestamp + LEFT JOIN LATERAL (SELECT DISTINCT session_id + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query30)}) AS chart_details + ON (TRUE) + GROUP BY timestamp + ORDER BY timestamp) AS chart_details) AS chart_details30 ON (TRUE); + """ + + # print("--------------------") + # print(cur.mogrify(main_pg_query, params)) + # print("--------------------") + cur.execute(cur.mogrify(main_pg_query, params)) + row = cur.fetchone() + if row is None: + return {"errors": ["error doesn't exist"]} + row["tags"] = __process_tags(row) + + query = cur.mogrify( + f"""SELECT error_id, status, session_id, start_ts, + parent_error_id,session_id, user_anonymous_id, + user_id, user_uuid, user_browser, user_browser_version, + user_os, user_os_version, user_device, payload, + COALESCE((SELECT TRUE + FROM public.user_favorite_errors AS fe + WHERE pe.error_id = fe.error_id + AND fe.user_id = %(user_id)s), FALSE) AS favorite, + True AS viewed + FROM public.errors AS pe + INNER JOIN events.errors AS ee USING (error_id) + INNER JOIN public.sessions USING (session_id) + WHERE pe.project_id = %(project_id)s + AND error_id = %(error_id)s + ORDER BY start_ts DESC + LIMIT 1;""", + {"project_id": project_id, "error_id": error_id, "user_id": user_id}) + cur.execute(query=query) + status = cur.fetchone() + + if status is not None: + row["stack"] = format_first_stack_frame(status).pop("stack") + row["status"] = status.pop("status") + row["parent_error_id"] = status.pop("parent_error_id") + row["favorite"] = status.pop("favorite") + row["viewed"] = status.pop("viewed") + row["last_hydrated_session"] = status + else: + row["stack"] = [] + row["last_hydrated_session"] = None + row["status"] = "untracked" + row["parent_error_id"] = None + row["favorite"] = False + row["viewed"] = False + return {"data": helper.dict_to_camel_case(row)} + + +def get_details_chart(project_id, error_id, user_id, **data): + pg_sub_query = __get_basic_constraints() + pg_sub_query.append("error_id = %(error_id)s") + pg_sub_query_chart = __get_basic_constraints(time_constraint=False, chart=True) + pg_sub_query_chart.append("error_id = %(error_id)s") + with pg_client.PostgresClient() as cur: + if data.get("startDate") is None: + data["startDate"] = TimeUTC.now(-7) + else: + data["startDate"] = int(data["startDate"]) + if data.get("endDate") is None: + data["endDate"] = TimeUTC.now() + else: + data["endDate"] = int(data["endDate"]) + density = int(data.get("density", 7)) + step_size = __get_step_size(data["startDate"], data["endDate"], density, factor=1) + params = { + "startDate": data['startDate'], + "endDate": data['endDate'], + "project_id": project_id, + "userId": user_id, + "step_size": step_size, + "error_id": error_id} + + main_pg_query = f"""\ + SELECT %(error_id)s AS error_id, + browsers_partition, + os_partition, + device_partition, + country_partition, + chart + FROM (SELECT jsonb_agg(browser_details) AS browsers_partition + FROM (SELECT * + FROM (SELECT user_browser AS name, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY user_browser + ORDER BY count DESC) AS count_per_browser_query + INNER JOIN LATERAL (SELECT jsonb_agg(count_per_version_details) AS partition + FROM (SELECT user_browser_version AS version, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND user_browser = count_per_browser_query.name + GROUP BY user_browser_version + ORDER BY count DESC) AS count_per_version_details) AS browesr_version_details + ON (TRUE)) AS browser_details) AS browser_details + INNER JOIN (SELECT jsonb_agg(os_details) AS os_partition + FROM (SELECT * + FROM (SELECT user_os AS name, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY user_os + ORDER BY count DESC) AS count_per_os_details + INNER JOIN LATERAL (SELECT jsonb_agg(count_per_version_query) AS partition + FROM (SELECT COALESCE(user_os_version, 'unknown') AS version, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND user_os = count_per_os_details.name + GROUP BY user_os_version + ORDER BY count DESC) AS count_per_version_query + ) AS os_version_query ON (TRUE)) AS os_details) AS os_details ON (TRUE) + INNER JOIN (SELECT jsonb_agg(device_details) AS device_partition + FROM (SELECT * + FROM (SELECT user_device_type AS name, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY user_device_type + ORDER BY count DESC) AS count_per_device_details + INNER JOIN LATERAL (SELECT jsonb_agg(count_per_device_details) AS partition + FROM (SELECT CASE + WHEN user_device = '' OR user_device ISNULL + THEN 'unknown' + ELSE user_device END AS version, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + AND user_device_type = count_per_device_details.name + GROUP BY user_device_type, user_device + ORDER BY count DESC) AS count_per_device_details + ) AS device_version_details ON (TRUE)) AS device_details) AS device_details ON (TRUE) + INNER JOIN (SELECT jsonb_agg(count_per_country_details) AS country_partition + FROM (SELECT user_country AS name, + COUNT(session_id) AS count + FROM events.errors INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY user_country + ORDER BY count DESC) AS count_per_country_details) AS country_details ON (TRUE) + INNER JOIN (SELECT jsonb_agg(chart_details) AS chart + FROM (SELECT generated_timestamp AS timestamp, + COUNT(session_id) AS count + FROM generate_series(%(startDate)s, %(endDate)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL (SELECT DISTINCT session_id + FROM events.errors + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS chart_details ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp) AS chart_details) AS chart_details ON (TRUE);""" + + cur.execute(cur.mogrify(main_pg_query, params)) + row = cur.fetchone() + if row is None: + return {"errors": ["error doesn't exist"]} + row["tags"] = __process_tags(row) + return {"data": helper.dict_to_camel_case(row)} + + +def __get_basic_constraints(platform=None, time_constraint=True, startTime_arg_name="startDate", + endTime_arg_name="endDate", chart=False, step_size_name="step_size", + project_key="project_id"): + ch_sub_query = [f"{project_key} =%(project_id)s"] + if time_constraint: + ch_sub_query += [f"timestamp >= %({startTime_arg_name})s", + f"timestamp < %({endTime_arg_name})s"] + if chart: + ch_sub_query += [f"timestamp >= generated_timestamp", + f"timestamp < generated_timestamp + %({step_size_name})s"] + if platform == 'mobile': + ch_sub_query.append("user_device_type = 'mobile'") + elif platform == 'desktop': + ch_sub_query.append("user_device_type = 'desktop'") + return ch_sub_query + + +def __get_sort_key(key): + return { + "datetime": "max_datetime", + "lastOccurrence": "max_datetime", + "firstOccurrence": "min_datetime" + }.get(key, 'max_datetime') + + +@dev.timed +def search(data, project_id, user_id, flows=False, status="ALL", favorite_only=False): + status = status.upper() + if status.lower() not in ['all', 'unresolved', 'resolved', 'ignored']: + return {"errors": ["invalid error status"]} + pg_sub_query = __get_basic_constraints(data.get('platform'), project_key="sessions.project_id") + pg_sub_query += ["sessions.start_ts>=%(startDate)s", "sessions.start_ts<%(endDate)s", "source ='js_exception'", + "pe.project_id=%(project_id)s"] + pg_sub_query_chart = __get_basic_constraints(data.get('platform'), time_constraint=False, chart=True) + pg_sub_query_chart.append("source ='js_exception'") + pg_sub_query_chart.append("errors.error_id =details.error_id") + statuses = [] + error_ids = None + if data.get("startDate") is None: + data["startDate"] = TimeUTC.now(-30) + if data.get("endDate") is None: + data["endDate"] = TimeUTC.now(1) + if len(data.get("events", [])) > 0 or len(data.get("filters", [])) > 0 or status != "ALL" or favorite_only: + statuses = sessions.search2_pg(data=data, project_id=project_id, user_id=user_id, errors_only=True, + error_status=status, favorite_only=favorite_only) + if len(statuses) == 0: + return {"data": { + 'total': 0, + 'errors': [] + }} + error_ids = [e["error_id"] for e in statuses] + with pg_client.PostgresClient() as cur: + if data.get("startDate") is None: + data["startDate"] = TimeUTC.now(-7) + if data.get("endDate") is None: + data["endDate"] = TimeUTC.now() + density = data.get("density", 7) + step_size = __get_step_size(data["startDate"], data["endDate"], density, factor=1) + sort = __get_sort_key('datetime') + if data.get("sort") is not None: + sort = __get_sort_key(data["sort"]) + order = "DESC" + if data.get("order") is not None: + order = data["order"] + + params = { + "startDate": data['startDate'], + "endDate": data['endDate'], + "project_id": project_id, + "userId": user_id, + "step_size": step_size} + if error_ids is not None: + params["error_ids"] = tuple(error_ids) + pg_sub_query.append("error_id IN %(error_ids)s") + main_pg_query = f"""\ + SELECT error_id, + name, + message, + users, + sessions, + last_occurrence, + first_occurrence, + chart + FROM (SELECT error_id, + name, + message, + COUNT(DISTINCT user_uuid) AS users, + COUNT(DISTINCT session_id) AS sessions, + MAX(timestamp) AS max_datetime, + MIN(timestamp) AS min_datetime + FROM events.errors + INNER JOIN public.errors AS pe USING (error_id) + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY error_id, name, message + ORDER BY {sort} {order}) AS details + INNER JOIN LATERAL (SELECT MAX(timestamp) AS last_occurrence, + MIN(timestamp) AS first_occurrence + FROM events.errors + WHERE errors.error_id = details.error_id) AS time_details ON (TRUE) + INNER JOIN LATERAL (SELECT jsonb_agg(chart_details) AS chart + FROM (SELECT generated_timestamp AS timestamp, + COUNT(session_id) AS count + FROM generate_series(%(startDate)s, %(endDate)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL (SELECT DISTINCT session_id + FROM events.errors INNER JOIN public.errors AS m_errors USING (error_id) + WHERE {" AND ".join(pg_sub_query_chart)} + ) AS sessions ON (TRUE) + GROUP BY timestamp + ORDER BY timestamp) AS chart_details) AS chart_details ON (TRUE);""" + + print("--------------------") + print(cur.mogrify(main_pg_query, params)) + cur.execute(cur.mogrify(main_pg_query, params)) + total = cur.rowcount + if flows: + return {"data": {"count": total}} + row = cur.fetchone() + rows = [] + limit = 200 + while row is not None and len(rows) < limit: + rows.append(row) + row = cur.fetchone() + if total == 0: + rows = [] + else: + if len(statuses) == 0: + query = cur.mogrify( + """SELECT error_id, status, parent_error_id, payload, + COALESCE((SELECT TRUE + FROM public.user_favorite_errors AS fe + WHERE errors.error_id = fe.error_id + AND fe.user_id = %(user_id)s LIMIT 1), FALSE) AS favorite, + COALESCE((SELECT TRUE + FROM public.user_viewed_errors AS ve + WHERE errors.error_id = ve.error_id + AND ve.user_id = %(user_id)s LIMIT 1), FALSE) AS viewed + FROM public.errors + WHERE project_id = %(project_id)s AND error_id IN %(error_ids)s;""", + {"project_id": project_id, "error_ids": tuple([r["error_id"] for r in rows]), + "user_id": user_id}) + cur.execute(query=query) + statuses = cur.fetchall() + statuses = { + s["error_id"]: s for s in statuses + } + + for r in rows: + if r["error_id"] in statuses: + r["status"] = statuses[r["error_id"]]["status"] + r["parent_error_id"] = statuses[r["error_id"]]["parent_error_id"] + r["favorite"] = statuses[r["error_id"]]["favorite"] + r["viewed"] = statuses[r["error_id"]]["viewed"] + r["stack"] = format_first_stack_frame(statuses[r["error_id"]])["stack"] + else: + r["status"] = "untracked" + r["parent_error_id"] = None + r["favorite"] = False + r["viewed"] = False + r["stack"] = None + + offset = len(rows) + rows = [r for r in rows if r["stack"] is None + or (len(r["stack"]) == 0 or len(r["stack"]) > 1 + or len(r["stack"]) > 0 + and (r["message"].lower() != "script error." or len(r["stack"][0]["absPath"]) > 0))] + offset -= len(rows) + return { + "data": { + 'total': total - offset, + 'errors': helper.list_to_camel_case(rows) + } + } + + +def __save_stacktrace(error_id, data): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """UPDATE public.errors + SET stacktrace=%(data)s::jsonb, stacktrace_parsed_at=timezone('utc'::text, now()) + WHERE error_id = %(error_id)s;""", + {"error_id": error_id, "data": json.dumps(data)}) + cur.execute(query=query) + + +def get_trace(project_id, error_id): + error = get(error_id=error_id) + if error is None: + return {"errors": ["error not found"]} + if error.get("source", "") != "js_exception": + return {"errors": ["this source of errors doesn't have a sourcemap"]} + if error.get("payload") is None: + return {"errors": ["null payload"]} + if error.get("stacktrace") is not None: + return {"sourcemapUploaded": True, + "trace": error.get("stacktrace"), + "preparsed": True} + trace, all_exists = sourcemaps.get_traces_group(project_id=project_id, payload=error["payload"]) + if all_exists: + __save_stacktrace(error_id=error_id, data=trace) + return {"sourcemapUploaded": all_exists, + "trace": trace, + "preparsed": False} + + +def get_sessions(start_date, end_date, project_id, user_id, error_id): + extra_constraints = ["s.project_id = %(project_id)s", + "s.start_ts >= %(startDate)s", + "s.start_ts <= %(endDate)s", + "e.error_id = %(error_id)s"] + if start_date is None: + start_date = TimeUTC.now(-7) + if end_date is None: + end_date = TimeUTC.now() + + params = { + "startDate": start_date, + "endDate": end_date, + "project_id": project_id, + "userId": user_id, + "error_id": error_id} + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + f"""SELECT s.project_id, + s.session_id::text AS session_id, + s.user_uuid, + s.user_id, + s.user_agent, + s.user_os, + s.user_browser, + s.user_device, + s.user_country, + s.start_ts, + s.duration, + s.events_count, + s.pages_count, + s.errors_count, + s.issue_types, + COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS favorite, + COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + FROM public.sessions AS s INNER JOIN events.errors AS e USING (session_id) + WHERE {" AND ".join(extra_constraints)} + ORDER BY s.start_ts DESC;""", + params) + cur.execute(query=query) + sessions_list = [] + total = cur.rowcount + row = cur.fetchone() + while row is not None and len(sessions_list) < 100: + sessions_list.append(row) + row = cur.fetchone() + + return { + 'total': total, + 'sessions': helper.list_to_camel_case(sessions_list) + } + + +ACTION_STATE = { + "unsolve": 'unresolved', + "solve": 'resolved', + "ignore": 'ignored' +} + + +def change_state(project_id, user_id, error_id, action): + errors = get(error_id, family=True) + print(len(errors)) + status = ACTION_STATE.get(action) + if errors is None or len(errors) == 0: + return {"errors": ["error not found"]} + if errors[0]["status"] == status: + return {"errors": [f"error is already {status}"]} + + if errors[0]["status"] == ACTION_STATE["solve"] and status == ACTION_STATE["ignore"]: + return {"errors": [f"state transition not permitted {errors[0]['status']} -> {status}"]} + + params = { + "userId": user_id, + "error_ids": tuple([e["errorId"] for e in errors]), + "status": status} + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """UPDATE public.errors + SET status = %(status)s + WHERE error_id IN %(error_ids)s + RETURNING status""", + params) + cur.execute(query=query) + row = cur.fetchone() + if row is not None: + for e in errors: + e["status"] = row["status"] + return {"data": errors} + + +MAX_RANK = 2 + + +def __status_rank(status): + return { + 'unresolved': MAX_RANK - 2, + 'ignored': MAX_RANK - 1, + 'resolved': MAX_RANK + }.get(status) + + +def merge(error_ids): + error_ids = list(set(error_ids)) + errors = get_batch(error_ids) + if len(error_ids) <= 1 or len(error_ids) > len(errors): + return {"errors": ["invalid list of ids"]} + error_ids = [e["errorId"] for e in errors] + parent_error_id = error_ids[0] + status = "unresolved" + for e in errors: + if __status_rank(status) < __status_rank(e["status"]): + status = e["status"] + if __status_rank(status) == MAX_RANK: + break + params = { + "error_ids": tuple(error_ids), + "parent_error_id": parent_error_id, + "status": status + } + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """UPDATE public.errors + SET parent_error_id = %(parent_error_id)s, status = %(status)s + WHERE error_id IN %(error_ids)s OR parent_error_id IN %(error_ids)s;""", + params) + cur.execute(query=query) + # row = cur.fetchone() + + return {"data": "success"} + + +def format_first_stack_frame(error): + error["stack"] = sourcemaps.format_payload(error.pop("payload"), truncate_to_first=True) + for s in error["stack"]: + for c in s.get("context", []): + for sci, sc in enumerate(c): + if isinstance(sc, str) and len(sc) > 1000: + c[sci] = sc[:1000] + # convert bytes to string: + if isinstance(s["filename"], bytes): + s["filename"] = s["filename"].decode("utf-8") + return error + + +def stats(project_id, user_id, startTimestamp=TimeUTC.now(delta_days=-7), endTimestamp=TimeUTC.now()): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """ + SELECT COUNT(errors.*) AS unresolved_and_unviewed + FROM public.errors + INNER JOIN (SELECT root_error.error_id + FROM events.errors + INNER JOIN public.errors AS root_error USING (error_id) + WHERE project_id = %(project_id)s + AND timestamp >= %(startTimestamp)s + AND timestamp <= %(endTimestamp)s + AND source = 'js_exception') AS timed_errors USING (error_id) + LEFT JOIN (SELECT error_id FROM public.user_viewed_errors WHERE user_id = %(user_id)s) AS user_viewed + USING (error_id) + WHERE user_viewed.error_id ISNULL + AND errors.project_id = %(project_id)s + AND errors.status = 'unresolved' + AND errors.source = 'js_exception';""", + {"project_id": project_id, "user_id": user_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp}) + cur.execute(query=query) + row = cur.fetchone() + + return { + "data": helper.dict_to_camel_case(row) + } diff --git a/api/chalicelib/core/errors_favorite_viewed.py b/api/chalicelib/core/errors_favorite_viewed.py new file mode 100644 index 000000000..0bbc10b68 --- /dev/null +++ b/api/chalicelib/core/errors_favorite_viewed.py @@ -0,0 +1,91 @@ +from chalicelib.utils import pg_client + + +def add_favorite_error(project_id, user_id, error_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""\ + INSERT INTO public.user_favorite_errors + (user_id, error_id) + VALUES + (%(userId)s,%(error_id)s);""", + {"userId": user_id, "error_id": error_id}) + ) + return {"errorId": error_id, "favorite": True} + + +def remove_favorite_error(project_id, user_id, error_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""\ + DELETE FROM public.user_favorite_errors + WHERE + user_id = %(userId)s + AND error_id = %(error_id)s;""", + {"userId": user_id, "error_id": error_id}) + ) + return {"errorId": error_id, "favorite": False} + + +def favorite_error(project_id, user_id, error_id): + exists, favorite = error_exists_and_favorite(user_id=user_id, error_id=error_id) + if not exists: + return {"errors": ["cannot bookmark non-rehydrated errors"]} + if favorite: + return remove_favorite_error(project_id=project_id, user_id=user_id, error_id=error_id) + return add_favorite_error(project_id=project_id, user_id=user_id, error_id=error_id) + + +def error_exists_and_favorite(user_id, error_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT errors.error_id AS exists, ufe.error_id AS favorite + FROM public.errors + LEFT JOIN (SELECT error_id FROM public.user_favorite_errors WHERE user_id = %(userId)s) AS ufe USING (error_id) + WHERE error_id = %(error_id)s;""", + {"userId": user_id, "error_id": error_id}) + ) + r = cur.fetchone() + if r is None: + return False, False + return True, r.get("favorite") is not None + + +def add_viewed_error(project_id, user_id, error_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + INSERT INTO public.user_viewed_errors + (user_id, error_id) + VALUES + (%(userId)s,%(error_id)s);""", + {"userId": user_id, "error_id": error_id}) + ) + + +def viewed_error_exists(user_id, error_id): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """SELECT + errors.error_id AS hydrated, + COALESCE((SELECT TRUE + FROM public.user_viewed_errors AS ve + WHERE ve.error_id = %(error_id)s + AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + FROM public.errors + WHERE error_id = %(error_id)s""", + {"userId": user_id, "error_id": error_id}) + cur.execute( + query=query + ) + r = cur.fetchone() + if r: + return r.get("viewed") + return True + + +def viewed_error(project_id, user_id, error_id): + if viewed_error_exists(user_id=user_id, error_id=error_id): + return None + return add_viewed_error(project_id=project_id, user_id=user_id, error_id=error_id) diff --git a/api/chalicelib/core/events.py b/api/chalicelib/core/events.py new file mode 100644 index 000000000..65ade49ed --- /dev/null +++ b/api/chalicelib/core/events.py @@ -0,0 +1,422 @@ +from chalicelib.utils import pg_client, helper +from chalicelib.core import sessions_metas, metadata +from chalicelib.core import issues +from chalicelib.utils.TimeUTC import TimeUTC +from chalicelib.utils.event_filter_definition import SupportedFilter, Event + + +def get_customs_by_sessionId2_pg(session_id, project_id): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify("""\ + SELECT + c.*, + 'CUSTOM' AS type + FROM events_common.customs AS c + WHERE + c.session_id = %(session_id)s + ORDER BY c.timestamp;""", + {"project_id": project_id, "session_id": session_id}) + ) + rows = cur.fetchall() + return helper.dict_to_camel_case(rows) + + +def __merge_cells(rows, start, count, replacement): + rows[start] = replacement + rows = rows[:start + 1] + rows[start + count:] + return rows + + +def __get_grouped_clickrage(rows, session_id): + click_rage_issues = issues.get_by_session_id(session_id=session_id, issue_type="click_rage") + if len(click_rage_issues) == 0: + return rows + + for c in click_rage_issues: + merge_count = c.get("payload") + if merge_count is not None: + merge_count = merge_count.get("count", 3) + else: + merge_count = 3 + for i in range(len(rows)): + if rows[i]["timestamp"] == c["timestamp"]: + rows = __merge_cells(rows=rows, + start=i, + count=merge_count, + replacement={**rows[i], "type": "CLICKRAGE", "count": merge_count}) + break + return rows + + +def get_by_sessionId2_pg(session_id, project_id, group_clickrage=False): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify("""\ + SELECT + c.*, + 'CLICK' AS type + FROM events.clicks AS c + WHERE + c.session_id = %(session_id)s + ORDER BY c.timestamp;""", + {"project_id": project_id, "session_id": session_id}) + ) + rows = cur.fetchall() + if group_clickrage: + rows = __get_grouped_clickrage(rows=rows, session_id=session_id) + + cur.execute(cur.mogrify(""" + SELECT + i.*, + 'INPUT' AS type + FROM events.inputs AS i + WHERE + i.session_id = %(session_id)s + ORDER BY i.timestamp;""", + {"project_id": project_id, "session_id": session_id}) + ) + rows += cur.fetchall() + cur.execute(cur.mogrify("""\ + SELECT + l.*, + l.path AS value, + l.path AS url, + 'LOCATION' AS type + FROM events.pages AS l + WHERE + l.session_id = %(session_id)s + ORDER BY l.timestamp;""", {"project_id": project_id, "session_id": session_id})) + rows += cur.fetchall() + rows = helper.list_to_camel_case(rows) + rows = sorted(rows, key=lambda k: k["messageId"]) + return rows + + +def __get_data_for_extend(data): + if "errors" not in data: + return data["data"] + + +def __pg_errors_query(source=None): + return f"""((SELECT DISTINCT ON(lg.message) + lg.message AS value, + source, + '{event_type.ERROR.ui_type}' AS type + FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.message ILIKE %(svalue)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5) + UNION ALL + (SELECT DISTINCT ON(lg.name) + lg.name AS value, + source, + '{event_type.ERROR.ui_type}' AS type + FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.name ILIKE %(svalue)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5) + UNION + (SELECT DISTINCT ON(lg.message) + lg.message AS value, + source, + '{event_type.ERROR.ui_type}' AS type + FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.message ILIKE %(value)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5) + UNION ALL + (SELECT DISTINCT ON(lg.name) + lg.name AS value, + source, + '{event_type.ERROR.ui_type}' AS type + FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.name ILIKE %(value)s + {"AND source = %(source)s" if source is not None else ""} + LIMIT 5));""" + + +def __search_pg_errors(project_id, value, key=None, source=None): + now = TimeUTC.now() + + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(__pg_errors_query(source), {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value), + "source": source})) + results = helper.list_to_camel_case(cur.fetchall()) + print(f"{TimeUTC.now() - now} : errors") + return results + + +def __search_pg_errors_ios(project_id, value, key=None, source=None): + now = TimeUTC.now() + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""(SELECT DISTINCT ON(lg.reason) + lg.reason AS value, + '{event_type.ERROR_IOS.ui_type}' AS type + FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.reason ILIKE %(value)s + LIMIT 5) + UNION ALL + (SELECT DISTINCT ON(lg.name) + lg.name AS value, + '{event_type.ERROR_IOS.ui_type}' AS type + FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id) + WHERE + s.project_id = %(project_id)s + AND lg.name ILIKE %(value)s + LIMIT 5);""", + {"project_id": project_id, "value": helper.string_to_sql_like(value)})) + results = helper.list_to_camel_case(cur.fetchall()) + print(f"{TimeUTC.now() - now} : errors") + return results + + +def __search_pg_metadata(project_id, value, key=None, source=None): + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + if len(meta_keys) == 0 or key is not None and key not in meta_keys.keys(): + return [] + sub_from = [] + if key is not None: + meta_keys = {key: meta_keys[key]} + + for k in meta_keys.keys(): + colname = metadata.index_to_colname(meta_keys[k]) + sub_from.append( + f"(SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key FROM public.sessions WHERE project_id = %(project_id)s AND {colname} ILIKE %(value)s LIMIT 5)") + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify(f"""\ + SELECT key, value, 'METADATA' AS TYPE + FROM({" UNION ALL ".join(sub_from)}) AS all_metas + LIMIT 5;""", {"project_id": project_id, "value": helper.string_to_sql_like(value)})) + results = helper.list_to_camel_case(cur.fetchall()) + return results + + +def __generic_query(typename): + return f"""\ + (SELECT value, type + FROM public.autocomplete + WHERE + project_id = %(project_id)s + AND type='{typename}' + AND value ILIKE %(svalue)s + LIMIT 5) + UNION + (SELECT value, type + FROM public.autocomplete + WHERE + project_id = %(project_id)s + AND type='{typename}' + AND value ILIKE %(value)s + LIMIT 5)""" + + +def __generic_autocomplete(event: Event): + def f(project_id, value, key=None, source=None): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify(__generic_query(event.ui_type), + {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value)})) + return helper.list_to_camel_case(cur.fetchall()) + + return f + + +class event_type: + CLICK = Event(ui_type="CLICK", table="events.clicks", column="label") + INPUT = Event(ui_type="INPUT", table="events.inputs", column="label") + LOCATION = Event(ui_type="LOCATION", table="events.pages", column="base_path") + CUSTOM = Event(ui_type="CUSTOM", table="events_common.customs", column="name") + REQUEST = Event(ui_type="REQUEST", table="events_common.requests", column="url") + GRAPHQL = Event(ui_type="GRAPHQL", table="events.graphql", column="name") + STATEACTION = Event(ui_type="STATEACTION", table="events.state_actions", column="name") + ERROR = Event(ui_type="ERROR", table="events.errors", + column=None) # column=None because errors are searched by name or message + METADATA = Event(ui_type="METADATA", table="public.sessions", column=None) + # IOS + CLICK_IOS = Event(ui_type="CLICK_IOS", table="events_ios.clicks", column="label") + INPUT_IOS = Event(ui_type="INPUT_IOS", table="events_ios.inputs", column="label") + VIEW_IOS = Event(ui_type="VIEW_IOS", table="events_ios.views", column="name") + CUSTOM_IOS = Event(ui_type="CUSTOM_IOS", table="events_common.customs", column="name") + REQUEST_IOS = Event(ui_type="REQUEST_IOS", table="events_common.requests", column="url") + ERROR_IOS = Event(ui_type="ERROR_IOS", table="events_ios.crashes", + column=None) # column=None because errors are searched by name or message + + +SUPPORTED_TYPES = { + event_type.CLICK.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CLICK), + query=__generic_query(typename=event_type.CLICK.ui_type), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + event_type.INPUT.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.INPUT), + query=__generic_query(typename=event_type.INPUT.ui_type), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + event_type.LOCATION.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.LOCATION), + query=__generic_query(typename=event_type.LOCATION.ui_type), + value_limit=3, + starts_with="/", + starts_limit=3, + ignore_if_starts_with=[]), + event_type.CUSTOM.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CUSTOM), + query=__generic_query(typename=event_type.CUSTOM.ui_type), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=[""]), + event_type.REQUEST.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.REQUEST), + query=__generic_query(typename=event_type.REQUEST.ui_type), + value_limit=3, + starts_with="/", + starts_limit=3, + ignore_if_starts_with=[""]), + event_type.GRAPHQL.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.GRAPHQL), + query=__generic_query(typename=event_type.GRAPHQL.ui_type), + value_limit=3, + starts_with="/", + starts_limit=4, + ignore_if_starts_with=[]), + event_type.STATEACTION.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.STATEACTION), + query=__generic_query(typename=event_type.STATEACTION.ui_type), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=[]), + event_type.ERROR.ui_type: SupportedFilter(get=__search_pg_errors, + query=None, + value_limit=4, + starts_with="", + starts_limit=4, + ignore_if_starts_with=["/"]), + event_type.METADATA.ui_type: SupportedFilter(get=__search_pg_metadata, + query=None, + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + # IOS + event_type.CLICK_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CLICK_IOS), + query=__generic_query(typename=event_type.CLICK_IOS.ui_type), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + event_type.INPUT_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.INPUT_IOS), + query=__generic_query(typename=event_type.INPUT_IOS.ui_type), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + event_type.VIEW_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.VIEW_IOS), + query=__generic_query(typename=event_type.VIEW_IOS.ui_type), + value_limit=3, + starts_with="/", + starts_limit=3, + ignore_if_starts_with=[]), + event_type.CUSTOM_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CUSTOM_IOS), + query=__generic_query(typename=event_type.CUSTOM_IOS.ui_type), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=[""]), + event_type.REQUEST_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.REQUEST_IOS), + query=__generic_query(typename=event_type.REQUEST_IOS.ui_type), + value_limit=3, + starts_with="/", + starts_limit=3, + ignore_if_starts_with=[""]), + event_type.ERROR_IOS.ui_type: SupportedFilter(get=__search_pg_errors, + query=None, + value_limit=4, + starts_with="", + starts_limit=4, + ignore_if_starts_with=["/"]), +} + + +def __get_merged_queries(queries, value, project_id): + if len(queries) == 0: + return [] + now = TimeUTC.now() + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify("(" + ")UNION ALL(".join(queries) + ")", + {"project_id": project_id, "value": helper.string_to_sql_like(value)})) + results = helper.list_to_camel_case(cur.fetchall()) + print(f"{TimeUTC.now() - now} : merged-queries for len: {len(queries)}") + return results + + +def __get_autocomplete_table(value, project_id): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify("""SELECT DISTINCT ON(value,type) project_id, value, type + FROM (SELECT * + FROM (SELECT *, + ROW_NUMBER() OVER (PARTITION BY type ORDER BY value) AS Row_ID + FROM public.autocomplete + WHERE project_id = %(project_id)s + AND value ILIKE %(svalue)s + UNION + SELECT *, + ROW_NUMBER() OVER (PARTITION BY type ORDER BY value) AS Row_ID + FROM public.autocomplete + WHERE project_id = %(project_id)s + AND value ILIKE %(value)s) AS u + WHERE Row_ID <= 5) AS sfa + ORDER BY sfa.type;""", + {"project_id": project_id, "value": helper.string_to_sql_like(value), + "svalue": helper.string_to_sql_like("^" + value)})) + results = helper.list_to_camel_case(cur.fetchall()) + return results + + +def search_pg2(text, event_type, project_id, source, key): + if not event_type: + return {"data": __get_autocomplete_table(text, project_id)} + + if event_type.upper() in SUPPORTED_TYPES.keys(): + rows = SUPPORTED_TYPES[event_type.upper()].get(project_id=project_id, value=text, key=key, source=source) + if event_type.upper() + "_IOS" in SUPPORTED_TYPES.keys(): + rows += SUPPORTED_TYPES[event_type.upper() + "_IOS"].get(project_id=project_id, value=text, key=key, + source=source) + elif event_type.upper() + "_IOS" in SUPPORTED_TYPES.keys(): + rows = SUPPORTED_TYPES[event_type.upper() + "_IOS"].get(project_id=project_id, value=text, key=key, + source=source) + elif event_type.upper() in sessions_metas.SUPPORTED_TYPES.keys(): + return sessions_metas.search(text, event_type, project_id) + elif event_type.upper().endswith("_IOS") \ + and event_type.upper()[:-len("_IOS")] in sessions_metas.SUPPORTED_TYPES.keys(): + return sessions_metas.search(text, event_type, project_id) + else: + return {"errors": ["unsupported event"]} + + return {"data": rows} + + +def get_errors_by_session_id(session_id): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify(f"""\ + SELECT er.*,ur.*, er.timestamp - s.start_ts AS time + FROM {event_type.ERROR.table} AS er INNER JOIN public.errors AS ur USING (error_id) INNER JOIN public.sessions AS s USING (session_id) + WHERE + er.session_id = %(session_id)s + ORDER BY timestamp;""", {"session_id": session_id})) + errors = cur.fetchall() + for e in errors: + e["stacktrace_parsed_at"] = TimeUTC.datetime_to_timestamp(e["stacktrace_parsed_at"]) + return helper.list_to_camel_case(errors) diff --git a/api/chalicelib/core/events_ios.py b/api/chalicelib/core/events_ios.py new file mode 100644 index 000000000..bae48599f --- /dev/null +++ b/api/chalicelib/core/events_ios.py @@ -0,0 +1,69 @@ +from chalicelib.utils import pg_client, helper +from chalicelib.core import events + + +def get_customs_by_sessionId(session_id, project_id): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify(f"""\ + SELECT + c.*, + '{events.event_type.CUSTOM_IOS.ui_type}' AS type + FROM {events.event_type.CUSTOM_IOS.table} AS c + WHERE + c.session_id = %(session_id)s + ORDER BY c.timestamp;""", + {"project_id": project_id, "session_id": session_id}) + ) + rows = cur.fetchall() + return helper.dict_to_camel_case(rows) + + +def get_by_sessionId(session_id, project_id): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify(f""" + SELECT + c.*, + '{events.event_type.CLICK_IOS.ui_type}' AS type + FROM {events.event_type.CLICK_IOS.table} AS c + WHERE + c.session_id = %(session_id)s + ORDER BY c.timestamp;""", + {"project_id": project_id, "session_id": session_id}) + ) + rows = cur.fetchall() + + cur.execute(cur.mogrify(f""" + SELECT + i.*, + '{events.event_type.INPUT_IOS.ui_type}' AS type + FROM {events.event_type.INPUT_IOS.table} AS i + WHERE + i.session_id = %(session_id)s + ORDER BY i.timestamp;""", + {"project_id": project_id, "session_id": session_id}) + ) + rows += cur.fetchall() + cur.execute(cur.mogrify(f""" + SELECT + v.*, + '{events.event_type.VIEW_IOS.ui_type}' AS type + FROM {events.event_type.VIEW_IOS.table} AS v + WHERE + v.session_id = %(session_id)s + ORDER BY v.timestamp;""", {"project_id": project_id, "session_id": session_id})) + rows += cur.fetchall() + rows = helper.list_to_camel_case(rows) + rows = sorted(rows, key=lambda k: k["timestamp"]) + return rows + + +def get_crashes_by_session_id(session_id): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify(f""" + SELECT cr.*,uc.*, cr.timestamp - s.start_ts AS time + FROM {events.event_type.ERROR_IOS.table} AS cr INNER JOIN public.crashes_ios AS uc USING (crash_id) INNER JOIN public.sessions AS s USING (session_id) + WHERE + cr.session_id = %(session_id)s + ORDER BY timestamp;""", {"session_id": session_id})) + errors = cur.fetchall() + return helper.list_to_camel_case(errors) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py new file mode 100644 index 000000000..0c3717038 --- /dev/null +++ b/api/chalicelib/core/funnels.py @@ -0,0 +1,269 @@ +import chalicelib.utils.helper +from chalicelib.core import events, significance, sessions +from chalicelib.utils.TimeUTC import TimeUTC + +from chalicelib.utils import helper, pg_client +from chalicelib.utils import dev +import json + +REMOVE_KEYS = ["key", "_key", "startDate", "endDate"] + +ALLOW_UPDATE_FOR = ["name", "filter"] + + +def filter_stages(stages): + ALLOW_TYPES = [events.event_type.CLICK.ui_type, events.event_type.INPUT.ui_type, + events.event_type.LOCATION.ui_type, events.event_type.CUSTOM.ui_type, + events.event_type.CLICK_IOS.ui_type, events.event_type.INPUT_IOS.ui_type, + events.event_type.VIEW_IOS.ui_type, events.event_type.CUSTOM_IOS.ui_type, ] + return [s for s in stages if s["type"] in ALLOW_TYPES and s.get("value") is not None] + + +def create(project_id, user_id, name, filter, is_public): + helper.delete_keys_from_dict(filter, REMOVE_KEYS) + filter["events"] = filter_stages(stages=filter.get("events", [])) + with pg_client.PostgresClient() as cur: + query = cur.mogrify("""\ + INSERT INTO public.funnels (project_id, user_id, name, filter,is_public) + VALUES (%(project_id)s, %(user_id)s, %(name)s, %(filter)s::jsonb,%(is_public)s) + RETURNING *;""", + {"user_id": user_id, "project_id": project_id, "name": name, "filter": json.dumps(filter), + "is_public": is_public}) + + cur.execute( + query + ) + r = cur.fetchone() + r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r = helper.dict_to_camel_case(r) + r["filter"]["startDate"], r["filter"]["endDate"] = TimeUTC.get_start_end_from_range(r["filter"]["rangeValue"]) + return {"data": r} + + +def update(funnel_id, user_id, name=None, filter=None, is_public=None): + s_query = [] + if filter is not None: + helper.delete_keys_from_dict(filter, REMOVE_KEYS) + s_query.append("filter = %(filter)s::jsonb") + if name is not None and len(name) > 0: + s_query.append("name = %(name)s") + if is_public is not None: + s_query.append("is_public = %(is_public)s") + if len(s_query) == 0: + return {"errors": ["Nothing to update"]} + with pg_client.PostgresClient() as cur: + query = cur.mogrify(f"""\ + UPDATE public.funnels + SET {" , ".join(s_query)} + WHERE funnel_id=%(funnel_id)s + RETURNING *;""", + {"user_id": user_id, "funnel_id": funnel_id, "name": name, + "filter": json.dumps(filter) if filter is not None else None, "is_public": is_public}) + # print("--------------------") + # print(query) + # print("--------------------") + cur.execute( + query + ) + r = cur.fetchone() + r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r = helper.dict_to_camel_case(r) + r["filter"]["startDate"], r["filter"]["endDate"] = TimeUTC.get_start_end_from_range(r["filter"]["rangeValue"]) + return {"data": r} + + +def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date=None, details=False): + with pg_client.PostgresClient() as cur: + team_query = "" + cur.execute( + cur.mogrify( + f"""\ + SELECT DISTINCT ON (funnels.funnel_id) funnel_id,project_id, user_id, name, created_at, deleted_at, is_public + {",filter" if details else ""} + FROM public.funnels {team_query} + WHERE project_id = %(project_id)s + AND funnels.deleted_at IS NULL + AND (funnels.user_id = %(user_id)s OR funnels.is_public);""", + {"project_id": project_id, "user_id": user_id} + ) + ) + + rows = cur.fetchall() + rows = helper.list_to_camel_case(rows) + for row in rows: + row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"]) + if details: + row["filter"]["events"] = filter_stages(row["filter"]["events"]) + get_start_end_time(filter_d=row["filter"], range_value=range_value, start_date=start_date, + end_date=end_date) + counts = sessions.search2_pg(data=row["filter"], project_id=project_id, user_id=None, count_only=True) + row["sessionsCount"] = counts["countSessions"] + row["usersCount"] = counts["countUsers"] + overview = significance.get_overview(filter_d=row["filter"], project_id=project_id) + row["stages"] = overview["stages"] + row.pop("filter") + row["stagesCount"] = len(row["stages"]) + # TODO: ask david to count it alone + row["criticalIssuesCount"] = overview["criticalIssuesCount"] + row["missedConversions"] = 0 if len(row["stages"]) < 2 \ + else row["stages"][0]["sessionsCount"] - row["stages"][-1]["sessionsCount"] + return rows + + +def get_possible_issue_types(project_id): + return [{"type": t, "title": chalicelib.utils.helper.get_issue_title(t)} for t in + ['click_rage', 'dead_click', 'excessive_scrolling', + 'bad_request', 'missing_resource', 'memory', 'cpu', + 'slow_resource', 'slow_page_load', 'crash', 'custom_event_error', + 'js_error']] + + +def get_start_end_time(filter_d, range_value, start_date, end_date): + if start_date is not None and end_date is not None: + filter_d["startDate"], filter_d["endDate"] = start_date, end_date + elif range_value is not None and len(range_value) > 0: + filter_d["rangeValue"] = range_value + filter_d["startDate"], filter_d["endDate"] = TimeUTC.get_start_end_from_range(range_value) + else: + filter_d["startDate"], filter_d["endDate"] = TimeUTC.get_start_end_from_range(filter_d["rangeValue"]) + + +def delete(project_id, funnel_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + UPDATE public.funnels + SET deleted_at = timezone('utc'::text, now()) + WHERE project_id = %(project_id)s + AND funnel_id = %(funnel_id)s;""", + {"funnel_id": funnel_id, "project_id": project_id, "user_id": user_id}) + ) + + return {"data": {"state": "success"}} + + +def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=None, end_date=None): + f = get(funnel_id=funnel_id, project_id=project_id) + if f is None: + return {"errors": ["filter not found"]} + get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) + return sessions.search2_pg(data=f["filter"], project_id=project_id, user_id=user_id) + + +def get_sessions_on_the_fly(funnel_id, project_id, user_id, data): + data["events"] = filter_stages(data.get("events", [])) + if len(data["events"]) == 0: + f = get(funnel_id=funnel_id, project_id=project_id) + if f is None: + return {"errors": ["funnel not found"]} + get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), + start_date=data.get('startDate', None), + end_date=data.get('endDate', None)) + data = f["filter"] + return sessions.search2_pg(data=data, project_id=project_id, user_id=user_id) + + +def get_top_insights(project_id, funnel_id, range_value=None, start_date=None, end_date=None): + f = get(funnel_id=funnel_id, project_id=project_id) + if f is None: + return {"errors": ["filter not found"]} + get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) + insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=f["filter"], project_id=project_id) + insights[-1]["dropDueToIssues"] = total_drop_due_to_issues + return {"stages": helper.list_to_camel_case(insights), + "totalDropDueToIssues": total_drop_due_to_issues} + + +def get_top_insights_on_the_fly(funnel_id, project_id, data): + data["events"] = filter_stages(data.get("events", [])) + if len(data["events"]) == 0: + f = get(funnel_id=funnel_id, project_id=project_id) + if f is None: + return {"errors": ["funnel not found"]} + get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), + start_date=data.get('startDate', None), + end_date=data.get('endDate', None)) + data = f["filter"] + insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data, project_id=project_id) + if len(insights) > 0: + insights[-1]["dropDueToIssues"] = total_drop_due_to_issues + return {"stages": helper.list_to_camel_case(insights), + "totalDropDueToIssues": total_drop_due_to_issues} + + +def get_issues(project_id, funnel_id, range_value=None, start_date=None, end_date=None): + f = get(funnel_id=funnel_id, project_id=project_id) + if f is None: + return {"errors": ["funnel not found"]} + get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) + return {"data": { + "issues": helper.dict_to_camel_case(significance.get_issues_list(filter_d=f["filter"], project_id=project_id)) + }} + + +@dev.timed +def get_issues_on_the_fly(funnel_id, project_id, data): + first_stage = data.get("firstStage") + last_stage = data.get("lastStage") + data["events"] = filter_stages(data.get("events", [])) + if len(data["events"]) == 0: + f = get(funnel_id=funnel_id, project_id=project_id) + if f is None: + return {"errors": ["funnel not found"]} + get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), + start_date=data.get('startDate', None), + end_date=data.get('endDate', None)) + data = f["filter"] + return { + "issues": helper.dict_to_camel_case( + significance.get_issues_list(filter_d=data, project_id=project_id, first_stage=first_stage, + last_stage=last_stage))} + + +def get(funnel_id, project_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """\ + SELECT + * + FROM public.funnels + WHERE project_id = %(project_id)s + AND deleted_at IS NULL + AND funnel_id = %(funnel_id)s;""", + {"funnel_id": funnel_id, "project_id": project_id} + ) + ) + + f = helper.dict_to_camel_case(cur.fetchone()) + if f is None: + return None + + f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) + f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) + return f + + +@dev.timed +def search_by_issue(user_id, project_id, funnel_id, issue_id, data, range_value=None, start_date=None, end_date=None): + if len(data.get("events", [])) == 0: + f = get(funnel_id=funnel_id, project_id=project_id) + if f is None: + return {"errors": ["funnel not found"]} + get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=data.get('startDate', start_date), + end_date=data.get('endDate', end_date)) + data = f["filter"] + + # insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data, project_id=project_id) + issues = get_issues_on_the_fly(funnel_id=funnel_id, project_id=project_id, data=data).get("issues", {}) + issues = issues.get("significant", []) + issues.get("insignificant", []) + issue = None + for i in issues: + if i.get("issueId", "") == issue_id: + issue = i + break + return {"sessions": sessions.search2_pg(user_id=user_id, project_id=project_id, issue=issue, + data=data) if issue is not None else {"total": 0, "sessions": []}, + # "stages": helper.list_to_camel_case(insights), + # "totalDropDueToIssues": total_drop_due_to_issues, + "issue": issue} diff --git a/api/chalicelib/core/integration_base.py b/api/chalicelib/core/integration_base.py new file mode 100644 index 000000000..45e1891a1 --- /dev/null +++ b/api/chalicelib/core/integration_base.py @@ -0,0 +1,53 @@ +from abc import ABC, abstractmethod +from chalicelib.utils import pg_client, helper + + +class BaseIntegration(ABC): + + def __init__(self, user_id, ISSUE_CLASS): + self._user_id = user_id + self.issue_handler = ISSUE_CLASS(self.integration_token) + + @property + @abstractmethod + def provider(self): + pass + + @property + def integration_token(self): + integration = self.get() + if integration is None: + print("no token configured yet") + return None + return integration["token"] + + def get(self): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT * + FROM public.oauth_authentication + WHERE user_id=%(user_id)s AND provider=%(provider)s;""", + {"user_id": self._user_id, "provider": self.provider.lower()}) + ) + return helper.dict_to_camel_case(cur.fetchone()) + + @abstractmethod + def get_obfuscated(self): + pass + + @abstractmethod + def update(self, changes): + pass + + @abstractmethod + def _add(self, data): + pass + + @abstractmethod + def delete(self): + pass + + @abstractmethod + def add_edit(self, data): + pass diff --git a/api/chalicelib/core/integration_base_issue.py b/api/chalicelib/core/integration_base_issue.py new file mode 100644 index 000000000..deb5792e6 --- /dev/null +++ b/api/chalicelib/core/integration_base_issue.py @@ -0,0 +1,41 @@ +from abc import ABC, abstractmethod + + +class RequestException(Exception): + pass + + +def proxy_issues_handler(e): + print("=======__proxy_issues_handler=======") + print(str(e)) + return {"errors": [str(e)]} + + +class BaseIntegrationIssue(ABC): + def __init__(self, provider, integration_token): + self.provider = provider + self.integration_token = integration_token + + @abstractmethod + def create_new_assignment(self, integration_project_id, title, description, assignee, issue_type): + pass + + @abstractmethod + def get_by_ids(self, saved_issues): + pass + + @abstractmethod + def get(self, integration_project_id, assignment_id): + pass + + @abstractmethod + def comment(self, integration_project_id, assignment_id, comment): + pass + + @abstractmethod + def get_metas(self, integration_project_id): + pass + + @abstractmethod + def get_projects(self): + pass diff --git a/api/chalicelib/core/integration_github.py b/api/chalicelib/core/integration_github.py new file mode 100644 index 000000000..a13946e46 --- /dev/null +++ b/api/chalicelib/core/integration_github.py @@ -0,0 +1,78 @@ +from chalicelib.utils import pg_client, helper +from chalicelib.core.integration_github_issue import GithubIntegrationIssue +from chalicelib.core import integration_base + +PROVIDER = "GITHUB" + + +class GitHubIntegration(integration_base.BaseIntegration): + + def __init__(self, tenant_id, user_id): + self.__tenant_id = tenant_id + super(GitHubIntegration, self).__init__(user_id=user_id, ISSUE_CLASS=GithubIntegrationIssue) + + @property + def provider(self): + return PROVIDER + + + + def get_obfuscated(self): + integration = self.get() + if integration is None: + return None + token = "*" * (len(integration["token"]) - 4) + integration["token"][-4:] + return {"token": token, "provider": self.provider.lower()} + + def update(self, changes): + with pg_client.PostgresClient() as cur: + sub_query = [f"{helper.key_to_snake_case(k)} = %({k})s" for k in changes.keys()] + cur.execute( + cur.mogrify(f"""\ + UPDATE public.oauth_authentication + SET {','.join(sub_query)} + WHERE user_id=%(user_id)s + RETURNING token;""", + {"user_id": self._user_id, + **changes}) + ) + w = helper.dict_to_camel_case(cur.fetchone()) + return w + + def _add(self, data): + pass + + def add(self, token): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + INSERT INTO public.oauth_authentication(user_id, provider, provider_user_id, token) + VALUES(%(user_id)s, 'github', '', %(token)s) + RETURNING token;""", + {"user_id": self._user_id, + "token": token}) + ) + w = helper.dict_to_camel_case(cur.fetchone()) + return w + + # TODO: make a revoke token call + def delete(self): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + DELETE FROM public.oauth_authentication + WHERE user_id=%(user_id)s AND provider=%(provider)s;""", + {"user_id": self._user_id, "provider": self.provider.lower()}) + ) + return {"state": "success"} + + def add_edit(self, data): + s = self.get() + if s is not None: + return self.update( + changes={ + "token": data["token"] + } + ) + else: + return self.add(token=data["token"]) diff --git a/api/chalicelib/core/integration_github_issue.py b/api/chalicelib/core/integration_github_issue.py new file mode 100644 index 000000000..8bac3c619 --- /dev/null +++ b/api/chalicelib/core/integration_github_issue.py @@ -0,0 +1,100 @@ +from chalicelib.core.integration_base_issue import BaseIntegrationIssue +from chalicelib.utils import github_client_v3 +from chalicelib.utils.github_client_v3 import github_formatters as formatter + + +class GithubIntegrationIssue(BaseIntegrationIssue): + def __init__(self, integration_token): + self.__client = github_client_v3.githubV3Request(integration_token) + super(GithubIntegrationIssue, self).__init__("GITHUB", integration_token) + + def get_current_user(self): + return formatter.user(self.__client.get("/user")) + + def get_meta(self, repoId): + current_user = self.get_current_user() + try: + users = self.__client.get(f"/repositories/{repoId}/collaborators") + except Exception as e: + users = [] + users = [formatter.user(u) for u in users] + if current_user not in users: + users.insert(0, current_user) + meta = { + 'users': users, + 'issueTypes': [formatter.label(l) for l in + self.__client.get(f"/repositories/{repoId}/labels")] + } + + return meta + + def create_new_assignment(self, integration_project_id, title, description, assignee, + issue_type): + repoId = integration_project_id + assignees = [assignee] + labels = [str(issue_type)] + + metas = self.get_meta(repoId) + real_assignees = [] + for a in assignees: + for u in metas["users"]: + if a == str(u["id"]): + real_assignees.append(u["login"]) + break + real_labels = ["Asayer"] + for l in labels: + found = False + for ll in metas["issueTypes"]: + if l == str(ll["id"]): + real_labels.append(ll["name"]) + found = True + break + if not found: + real_labels.append(l) + issue = self.__client.post(f"/repositories/{repoId}/issues", body={"title": title, "body": description, + "assignees": real_assignees, + "labels": real_labels}) + return formatter.issue(issue) + + def get_by_ids(self, saved_issues): + results = [] + for i in saved_issues: + results.append(self.get(integration_project_id=i["integrationProjectId"], assignment_id=i["id"])) + return {"issues": results} + + def get(self, integration_project_id, assignment_id): + repoId = integration_project_id + issueNumber = assignment_id + issue = self.__client.get(f"/repositories/{repoId}/issues/{issueNumber}") + issue = formatter.issue(issue) + if issue["commentsCount"] > 0: + issue["comments"] = [formatter.comment(c) for c in + self.__client.get(f"/repositories/{repoId}/issues/{issueNumber}/comments")] + return issue + + def comment(self, integration_project_id, assignment_id, comment): + repoId = integration_project_id + issueNumber = assignment_id + commentCreated = self.__client.post(f"/repositories/{repoId}/issues/{issueNumber}/comments", + body={"body": comment}) + return formatter.comment(commentCreated) + + def get_metas(self, integration_project_id): + current_user = self.get_current_user() + try: + users = self.__client.get(f"/repositories/{integration_project_id}/collaborators") + except Exception as e: + users = [] + users = [formatter.user(u) for u in users] + if current_user not in users: + users.insert(0, current_user) + + return {"provider": self.provider.lower(), + 'users': users, + 'issueTypes': [formatter.label(l) for l in + self.__client.get(f"/repositories/{integration_project_id}/labels")] + } + + def get_projects(self): + repos = self.__client.get("/user/repos") + return [formatter.repo(r) for r in repos] diff --git a/api/chalicelib/core/integration_jira_cloud.py b/api/chalicelib/core/integration_jira_cloud.py new file mode 100644 index 000000000..65bad1e4a --- /dev/null +++ b/api/chalicelib/core/integration_jira_cloud.py @@ -0,0 +1,101 @@ +from chalicelib.utils import pg_client, helper +from chalicelib.core.integration_jira_cloud_issue import JIRACloudIntegrationIssue +from chalicelib.core import integration_base + +PROVIDER = "JIRA" + + +class JIRAIntegration(integration_base.BaseIntegration): + def __init__(self, tenant_id, user_id): + self.__tenant_id = tenant_id + # TODO: enable super-constructor when OAuth is done + # super(JIRAIntegration, self).__init__(jwt, user_id, JIRACloudIntegrationProxy) + self._user_id = user_id + i = self.get() + if i is None: + return + self.issue_handler = JIRACloudIntegrationIssue(token=i["token"], username=i["username"], url=i["url"]) + + @property + def provider(self): + return PROVIDER + + # TODO: remove this once jira-oauth is done + def get(self): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT username, token, url + FROM public.jira_cloud + WHERE user_id=%(user_id)s;""", + {"user_id": self._user_id}) + ) + return helper.dict_to_camel_case(cur.fetchone()) + + def get_obfuscated(self): + integration = self.get() + if integration is None: + return None + integration["token"] = "*" * (len(integration["token"]) - 4) + integration["token"][-4:] + integration["provider"] = self.provider.lower() + return integration + + def update(self, changes): + with pg_client.PostgresClient() as cur: + sub_query = [f"{helper.key_to_snake_case(k)} = %({k})s" for k in changes.keys()] + cur.execute( + cur.mogrify(f"""\ + UPDATE public.jira_cloud + SET {','.join(sub_query)} + WHERE user_id=%(user_id)s + RETURNING username, token, url;""", + {"user_id": self._user_id, + **changes}) + ) + w = helper.dict_to_camel_case(cur.fetchone()) + return w + + # TODO: make this generic for all issue tracking integrations + def _add(self, data): + print("a pretty defined abstract method") + return + + def add(self, username, token, url): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + INSERT INTO public.jira_cloud(username, token, user_id,url) + VALUES (%(username)s, %(token)s, %(user_id)s,%(url)s) + RETURNING username, token, url;""", + {"user_id": self._user_id, "username": username, + "token": token, "url": url}) + ) + w = helper.dict_to_camel_case(cur.fetchone()) + return w + + def delete(self): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + DELETE FROM public.jira_cloud + WHERE user_id=%(user_id)s;""", + {"user_id": self._user_id}) + ) + return {"state": "success"} + + def add_edit(self, data): + s = self.get() + if s is not None: + return self.update( + changes={ + "username": data["username"], + "token": data["token"], + "url": data["url"] + } + ) + else: + return self.add( + username=data["username"], + token=data["token"], + url=data["url"] + ) diff --git a/api/chalicelib/core/integration_jira_cloud_issue.py b/api/chalicelib/core/integration_jira_cloud_issue.py new file mode 100644 index 000000000..b765903c2 --- /dev/null +++ b/api/chalicelib/core/integration_jira_cloud_issue.py @@ -0,0 +1,56 @@ +from chalicelib.utils import jira_client +from chalicelib.core.integration_base_issue import BaseIntegrationIssue + + +class JIRACloudIntegrationIssue(BaseIntegrationIssue): + def __init__(self, token, username, url): + self.username = username + self.url = url + self._client = jira_client.JiraManager(self.url, self.username, token, None) + super(JIRACloudIntegrationIssue, self).__init__("JIRA", token) + + def create_new_assignment(self, integration_project_id, title, description, assignee, issue_type): + self._client.set_jira_project_id(integration_project_id) + data = { + 'summary': title, + 'description': description, + 'issuetype': {'id': issue_type}, + 'assignee': {"id": assignee}, + "labels": ["Asayer"] + } + return self._client.create_issue(data) + + def get_by_ids(self, saved_issues): + projects_map = {} + for i in saved_issues: + if i["integrationProjectId"] not in projects_map.keys(): + projects_map[i["integrationProjectId"]] = [] + projects_map[i["integrationProjectId"]].append(i["id"]) + + results = [] + for integration_project_id in projects_map: + self._client.set_jira_project_id(integration_project_id) + jql = 'labels = Asayer' + if len(projects_map[integration_project_id]) > 0: + jql += f" AND ID IN ({','.join(projects_map[integration_project_id])})" + issues = self._client.get_issues(jql, offset=0) + results += [issues] + return {"issues": results} + + def get(self, integration_project_id, assignment_id): + self._client.set_jira_project_id(integration_project_id) + return self._client.get_issue_v3(assignment_id) + + def comment(self, integration_project_id, assignment_id, comment): + self._client.set_jira_project_id(integration_project_id) + return self._client.add_comment_v3(assignment_id, comment) + + def get_metas(self, integration_project_id): + meta = {} + self._client.set_jira_project_id(integration_project_id) + meta['issueTypes'] = self._client.get_issue_types() + meta['users'] = self._client.get_assignable_users() + return {"provider": self.provider.lower(), **meta} + + def get_projects(self): + return self._client.get_projects() diff --git a/api/chalicelib/core/integrations_manager.py b/api/chalicelib/core/integrations_manager.py new file mode 100644 index 000000000..fca271870 --- /dev/null +++ b/api/chalicelib/core/integrations_manager.py @@ -0,0 +1,42 @@ +from chalicelib.core import integration_github, integration_jira_cloud +from chalicelib.utils import pg_client + +SUPPORTED_TOOLS = [integration_github.PROVIDER, integration_jira_cloud.PROVIDER] + + +def get_available_integrations(user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""\ + SELECT EXISTS((SELECT 1 + FROM public.oauth_authentication + WHERE user_id = %(user_id)s + AND provider = 'github')) AS github, + EXISTS((SELECT 1 + FROM public.jira_cloud + WHERE user_id = %(user_id)s)) AS jira;""", + {"user_id": user_id}) + ) + current_integrations = cur.fetchone() + return dict(current_integrations) + + +def __get_default_integration(user_id): + current_integrations = get_available_integrations(user_id) + return integration_github.PROVIDER if current_integrations["github"] else integration_jira_cloud.PROVIDER if \ + current_integrations["jira"] else None + + +def get_integration(tenant_id, user_id, tool=None): + if tool is None: + tool = __get_default_integration(user_id=user_id) + if tool is None: + return {"errors": [f"no issue tracking tool found"]}, None + tool = tool.upper() + if tool not in SUPPORTED_TOOLS: + return {"errors": [f"issue tracking tool not supported yet, available: {SUPPORTED_TOOLS}"]}, None + if tool == integration_jira_cloud.PROVIDER: + return None, integration_jira_cloud.JIRAIntegration(tenant_id=tenant_id, user_id=user_id) + elif tool == integration_github.PROVIDER: + return None, integration_github.GitHubIntegration(tenant_id=tenant_id, user_id=user_id) + return {"errors": ["lost integration"]}, None diff --git a/api/chalicelib/core/issues.py b/api/chalicelib/core/issues.py new file mode 100644 index 000000000..61a6946da --- /dev/null +++ b/api/chalicelib/core/issues.py @@ -0,0 +1,72 @@ +from chalicelib.utils import pg_client, helper + +ISSUE_TYPES = ['click_rage', 'dead_click', 'excessive_scrolling', 'bad_request', 'missing_resource', 'memory', 'cpu', + 'slow_resource', 'slow_page_load', 'crash', 'ml_cpu', 'ml_memory', 'ml_dead_click', 'ml_click_rage', + 'ml_mouse_thrashing', 'ml_excessive_scrolling', 'ml_slow_resources', 'custom', 'js_exception', + 'custom_event_error', 'js_error'] +ORDER_QUERY = """\ +(CASE WHEN type = 'js_exception' THEN 0 + WHEN type = 'bad_request' THEN 1 + WHEN type = 'missing_resource' THEN 2 + WHEN type = 'click_rage' THEN 3 + WHEN type = 'dead_click' THEN 4 + WHEN type = 'memory' THEN 5 + WHEN type = 'cpu' THEN 6 + WHEN type = 'crash' THEN 7 + ELSE -1 END)::INTEGER +""" +NAME_QUERY = """\ +(CASE WHEN type = 'js_exception' THEN 'Errors' + WHEN type = 'bad_request' THEN 'Bad Requests' + WHEN type = 'missing_resource' THEN 'Missing Images' + WHEN type = 'click_rage' THEN 'Click Rage' + WHEN type = 'dead_click' THEN 'Dead Clicks' + WHEN type = 'memory' THEN 'High Memory' + WHEN type = 'cpu' THEN 'High CPU' + WHEN type = 'crash' THEN 'Crashes' + ELSE type::text END)::text +""" + + +def get(project_id, issue_id): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + """\ + SELECT + * + FROM public.issues + WHERE project_id = %(project_id)s + AND issue_id = %(issue_id)s;""", + {"project_id": project_id, "issue_id": issue_id} + ) + cur.execute(query=query) + data = cur.fetchone() + return helper.dict_to_camel_case(data) + + +def get_by_session_id(session_id, issue_type=None): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""\ + SELECT * + FROM events_common.issues + INNER JOIN public.issues USING (issue_id) + WHERE session_id = %(session_id)s {"AND type = %(type)s" if issue_type is not None else ""} + ORDER BY timestamp;""", + {"session_id": session_id, "type": issue_type}) + ) + return helper.list_to_camel_case(cur.fetchall()) + + +def get_types(project_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""SELECT type, + {ORDER_QUERY}>=0 AS visible, + {ORDER_QUERY} AS order, + {NAME_QUERY} AS name + FROM (SELECT DISTINCT type + FROM public.issues + WHERE project_id = %(project_id)s) AS types + ORDER BY "order";""", {"project_id": project_id})) + return helper.list_to_camel_case(cur.fetchall()) diff --git a/api/chalicelib/core/log_tool_bugsnag.py b/api/chalicelib/core/log_tool_bugsnag.py new file mode 100644 index 000000000..8354142ab --- /dev/null +++ b/api/chalicelib/core/log_tool_bugsnag.py @@ -0,0 +1,73 @@ +from chalicelib.core import log_tools +import requests + +IN_TY = "bugsnag" + + +def list_projects(auth_token): + r = requests.get(url="https://api.bugsnag.com/user/organizations", + params={"per_page": "100"}, + headers={"Authorization": "token " + auth_token, "X-Version": "2"}) + if r.status_code != 200: + print("=======> bugsnag get organizations: something went wrong") + print(r) + print(r.status_code) + print(r.text) + return [] + + orgs = [] + for i in r.json(): + + pr = requests.get(url="https://api.bugsnag.com/organizations/%s/projects" % i["id"], + params={"per_page": "100"}, + headers={"Authorization": "token " + auth_token, "X-Version": "2"}) + if pr.status_code != 200: + print("=======> bugsnag get projects: something went wrong") + print(pr) + print(r.status_code) + print(r.text) + continue + orgs.append({"name": i["name"], "projects": [{"name": p["name"], "id": p["id"]} for p in pr.json()]}) + return orgs + + +def get_all(tenant_id): + return log_tools.get_all_by_tenant(tenant_id=tenant_id, integration=IN_TY) + + +def get(project_id): + return log_tools.get(project_id=project_id, integration=IN_TY) + + +def update(tenant_id, project_id, changes): + options = {} + if "authorizationToken" in changes: + options["authorizationToken"] = changes.pop("authorizationToken") + if "bugsnagProjectId" in changes: + options["bugsnagProjectId"] = changes.pop("bugsnagProjectId") + return log_tools.edit(project_id=project_id, integration=IN_TY, changes=options) + + +def add(tenant_id, project_id, authorization_token, bugsnag_project_id): + options = { + "bugsnagProjectId": bugsnag_project_id, + "authorizationToken": authorization_token, + } + return log_tools.add(project_id=project_id, integration=IN_TY, options=options) + + +def delete(tenant_id, project_id): + return log_tools.delete(project_id=project_id, integration=IN_TY) + + +def add_edit(tenant_id, project_id, data): + s = get(project_id) + if s is not None: + return update(tenant_id=tenant_id, project_id=project_id, + changes={"authorizationToken": data["authorizationToken"], + "bugsnagProjectId": data["bugsnagProjectId"]}) + else: + return add(tenant_id=tenant_id, + project_id=project_id, + authorization_token=data["authorizationToken"], + bugsnag_project_id=data["bugsnagProjectId"]) diff --git a/api/chalicelib/core/log_tool_cloudwatch.py b/api/chalicelib/core/log_tool_cloudwatch.py new file mode 100644 index 000000000..bdd4a2644 --- /dev/null +++ b/api/chalicelib/core/log_tool_cloudwatch.py @@ -0,0 +1,119 @@ +import boto3 +from chalicelib.core import log_tools + +IN_TY = "cloudwatch" + + +def __find_groups(client, token): + d_args = { + "limit": 50 + } + if token is not None: + d_args["nextToken"] = token + response = client.describe_log_groups(**d_args) + response["logGroups"] = [i['logGroupName'] for i in response["logGroups"]] + if "nextToken" not in response: + return response["logGroups"] + + return response["logGroups"] + __find_groups(client, response["nextToken"]) + + +def __make_stream_filter(start_time, end_time): + def __valid_stream(stream): + return "firstEventTimestamp" in stream and not ( + stream['firstEventTimestamp'] <= start_time and stream["lastEventTimestamp"] <= start_time + or stream['firstEventTimestamp'] >= end_time and stream["lastEventTimestamp"] >= end_time + ) + + return __valid_stream + + +def __find_streams(project_id, log_group, client, token, stream_filter): + d_args = {"logGroupName": log_group, "orderBy": 'LastEventTime', 'limit': 50} + if token is not None and len(token) > 0: + d_args["nextToken"] = token + data = client.describe_log_streams(**d_args) + streams = list(filter(stream_filter, data['logStreams'])) + if 'nextToken' not in data: + save_new_token(project_id=project_id, token=token) + return streams + return streams + __find_streams(project_id, log_group, client, data['nextToken'], stream_filter) + + +def __find_events(client, log_group, streams, last_token, start_time, end_time): + f_args = { + "logGroupName": log_group, + "logStreamNames": streams, + "startTime": start_time, + "endTime": end_time, + "limit": 10000, + "filterPattern": "asayer_session_id" + } + if last_token is not None: + f_args["nextToken"] = last_token + response = client.filter_log_events( + **f_args + ) + if "nextToken" not in response: + return response["events"] + + return response["events"] + __find_events(client, log_group, streams, response["nextToken"], start_time, end_time) + + +def list_log_groups(aws_access_key_id, aws_secret_access_key, region): + logs = boto3.client('logs', aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=region + ) + return __find_groups(logs, None) + + +def get_all(tenant_id): + return log_tools.get_all_by_tenant(tenant_id=tenant_id, integration=IN_TY) + + +def get(project_id): + return log_tools.get(project_id=project_id, integration=IN_TY) + + +def update(tenant_id, project_id, changes): + options = {} + if "authorization_token" in changes: + options["authorization_token"] = changes.pop("authorization_token") + if "project_id" in changes: + options["project_id"] = changes.pop("project_id") + if len(options.keys()) > 0: + changes["options"] = options + return log_tools.edit(project_id=project_id, integration=IN_TY, changes=changes) + + +def add(tenant_id, project_id, aws_access_key_id, aws_secret_access_key, log_group_name, region): + return log_tools.add(project_id=project_id, integration=IN_TY, + options={"awsAccessKeyId": aws_access_key_id, + "awsSecretAccessKey": aws_secret_access_key, + "logGroupName": log_group_name, "region": region}) + + +def save_new_token(project_id, token): + update(tenant_id=None, project_id=project_id, changes={"last_token": token}) + + +def delete(tenant_id, project_id): + return log_tools.delete(project_id=project_id, integration=IN_TY) + + +def add_edit(tenant_id, project_id, data): + s = get(project_id) + if s is not None: + return update(tenant_id=tenant_id, project_id=project_id, + changes={"awsAccessKeyId": data["awsAccessKeyId"], + "awsSecretAccessKey": data["awsSecretAccessKey"], + "logGroupName": data["logGroupName"], + "region": data["region"]}) + else: + return add(tenant_id=tenant_id, + project_id=project_id, + aws_access_key_id=data["awsAccessKeyId"], + aws_secret_access_key=data["awsSecretAccessKey"], + log_group_name=data["logGroupName"], + region=data["region"]) diff --git a/api/chalicelib/core/log_tool_datadog.py b/api/chalicelib/core/log_tool_datadog.py new file mode 100644 index 000000000..68735f833 --- /dev/null +++ b/api/chalicelib/core/log_tool_datadog.py @@ -0,0 +1,43 @@ +from chalicelib.core import log_tools + +IN_TY = "datadog" + + +def get_all(tenant_id): + return log_tools.get_all_by_tenant(tenant_id=tenant_id, integration=IN_TY) + + +def get(project_id): + return log_tools.get(project_id=project_id, integration=IN_TY) + + +def update(tenant_id, project_id, changes): + options = {} + if "apiKey" in changes: + options["apiKey"] = changes["apiKey"] + if "applicationKey" in changes: + options["applicationKey"] = changes["applicationKey"] + + return log_tools.edit(project_id=project_id, integration=IN_TY, changes=options) + + +def add(tenant_id, project_id, api_key, application_key): + options = {"apiKey": api_key, "applicationKey": application_key} + return log_tools.add(project_id=project_id, integration=IN_TY, options=options) + + +def delete(tenant_id, project_id): + return log_tools.delete(project_id=project_id, integration=IN_TY) + + +def add_edit(tenant_id, project_id, data): + s = get(project_id) + if s is not None: + return update(tenant_id=tenant_id, project_id=project_id, + changes={"apiKey": data["apiKey"], + "applicationKey": data["applicationKey"]}) + else: + return add(tenant_id=tenant_id, + project_id=project_id, + api_key=data["apiKey"], + application_key=data["applicationKey"]) diff --git a/api/chalicelib/core/log_tool_elasticsearch.py b/api/chalicelib/core/log_tool_elasticsearch.py new file mode 100644 index 000000000..5fdfccf24 --- /dev/null +++ b/api/chalicelib/core/log_tool_elasticsearch.py @@ -0,0 +1,94 @@ +from elasticsearch import Elasticsearch, RequestsHttpConnection +from chalicelib.core import log_tools +import base64 +import logging + +logging.getLogger('elasticsearch').level = logging.ERROR + +IN_TY = "elasticsearch" + + +def get_all(tenant_id): + return log_tools.get_all_by_tenant(tenant_id=tenant_id, integration=IN_TY) + + +def get(project_id): + return log_tools.get(project_id=project_id, integration=IN_TY) + + +def update(tenant_id, project_id, changes): + options = {} + + if "host" in changes: + options["host"] = changes["host"] + if "apiKeyId" in changes: + options["apiKeyId"] = changes["apiKeyId"] + if "apiKey" in changes: + options["apiKey"] = changes["apiKey"] + if "indexes" in changes: + options["indexes"] = changes["indexes"] + if "port" in changes: + options["port"] = changes["port"] + + return log_tools.edit(project_id=project_id, integration=IN_TY, changes=options) + + +def add(tenant_id, project_id, host, api_key_id, api_key, indexes, port): + options = { + "host": host, "apiKeyId": api_key_id, "apiKey": api_key, "indexes": indexes, "port": port + } + return log_tools.add(project_id=project_id, integration=IN_TY, options=options) + + +def delete(tenant_id, project_id): + return log_tools.delete(project_id=project_id, integration=IN_TY) + + +def add_edit(tenant_id, project_id, data): + s = get(project_id) + if s is not None: + return update(tenant_id=tenant_id, project_id=project_id, + changes={"host": data["host"], "apiKeyId": data["apiKeyId"], "apiKey": data["apiKey"], + "indexes": data["indexes"], "port": data["port"]}) + else: + return add(tenant_id=tenant_id, + project_id=project_id, + host=data["host"], api_key=data["apiKeyId"], api_key_id=data["apiKey"], indexes=data["indexes"], + port=data["port"]) + + +def __get_es_client(host, port, api_key_id, api_key, use_ssl=False, timeout=29): + host = host.replace("http://", "").replace("https://", "") + try: + args = { + "hosts": [{"host": host, "port": port}], + "use_ssl": use_ssl, + "verify_certs": False, + "ca_certs": False, + "connection_class": RequestsHttpConnection, + "timeout": timeout + } + if api_key_id is not None and len(api_key_id) > 0: + # args["http_auth"] = (username, password) + token = "ApiKey " + base64.b64encode(f"{api_key_id}:{api_key}".encode("utf-8")).decode("utf-8") + args["headers"] = {"Authorization": token} + es = Elasticsearch( + **args + ) + r = es.ping() + if not r and not use_ssl: + return __get_es_client(host, port, api_key_id, api_key, use_ssl=True, timeout=timeout) + if not r: + return None + except Exception as err: + print("================exception connecting to ES host:") + print(err) + return None + return es + + +def ping(tenant_id, host, port, apiKeyId, apiKey): + es = __get_es_client(host, port, apiKeyId, apiKey, timeout=3) + if es is None: + return {"state": False} + return {"state": es.ping()} diff --git a/api/chalicelib/core/log_tool_newrelic.py b/api/chalicelib/core/log_tool_newrelic.py new file mode 100644 index 000000000..14b0d4e57 --- /dev/null +++ b/api/chalicelib/core/log_tool_newrelic.py @@ -0,0 +1,50 @@ +from chalicelib.core import log_tools + +IN_TY = "newrelic" + + +def get_all(tenant_id): + return log_tools.get_all_by_tenant(tenant_id=tenant_id, integration=IN_TY) + + +def get(project_id): + return log_tools.get(project_id=project_id, integration=IN_TY) + + +def update(tenant_id, project_id, changes): + options = {} + if "region" in changes and len(changes["region"]) == 0: + options["region"] = "US" + if "applicationId" in changes: + options["applicationId"] = changes["applicationId"] + if "xQueryKey" in changes: + options["xQueryKey"] = changes["xQueryKey"] + + return log_tools.edit(project_id=project_id, integration=IN_TY, changes=options) + + +def add(tenant_id, project_id, application_id, x_query_key, region): + if region is None or len(region) == 0: + region = "US" + options = {"applicationId": application_id, "xQueryKey": x_query_key, "region": region} + return log_tools.add(project_id=project_id, integration=IN_TY, options=options) + + +def delete(tenant_id, project_id): + return log_tools.delete(project_id=project_id, integration=IN_TY) + + +def add_edit(tenant_id, project_id, data): + s = get(project_id) + if s is not None: + return update(tenant_id=tenant_id, project_id=project_id, + changes={"applicationId": data["applicationId"], + "xQueryKey": data["xQueryKey"], + "region": data["region"]}) + else: + return add(tenant_id=tenant_id, + project_id=project_id, + application_id=data["applicationId"], + x_query_key=data["xQueryKey"], + region=data["region"] + ) diff --git a/api/chalicelib/core/log_tool_rollbar.py b/api/chalicelib/core/log_tool_rollbar.py new file mode 100644 index 000000000..34f6c6114 --- /dev/null +++ b/api/chalicelib/core/log_tool_rollbar.py @@ -0,0 +1,38 @@ +from chalicelib.core import log_tools + +IN_TY = "rollbar" + + +def get_all(tenant_id): + return log_tools.get_all_by_tenant(tenant_id=tenant_id, integration=IN_TY) + + +def get(project_id): + return log_tools.get(project_id=project_id, integration=IN_TY) + + +def update(tenant_id, project_id, changes): + options = {} + if "accessToken" in changes: + options["accessToken"] = changes["accessToken"] + return log_tools.edit(project_id=project_id, integration=IN_TY, changes=options) + + +def add(tenant_id, project_id, access_token): + options = {"accessToken": access_token} + return log_tools.add(project_id=project_id, integration=IN_TY, options=options) + + +def delete(tenant_id, project_id): + return log_tools.delete(project_id=project_id, integration=IN_TY) + + +def add_edit(tenant_id, project_id, data): + s = get(project_id) + if s is not None: + return update(tenant_id=tenant_id, project_id=project_id, + changes={"accessToken": data["accessToken"]}) + else: + return add(tenant_id=tenant_id, + project_id=project_id, + access_token=data["accessToken"]) diff --git a/api/chalicelib/core/log_tool_sentry.py b/api/chalicelib/core/log_tool_sentry.py new file mode 100644 index 000000000..1f51f1556 --- /dev/null +++ b/api/chalicelib/core/log_tool_sentry.py @@ -0,0 +1,65 @@ +import requests +from chalicelib.core import log_tools + +IN_TY = "sentry" + + +def get_all(tenant_id): + return log_tools.get_all_by_tenant(tenant_id=tenant_id, integration=IN_TY) + + +def get(project_id): + return log_tools.get(project_id=project_id, integration=IN_TY) + + +def update(tenant_id, project_id, changes): + options = {} + if "organizationSlug" in changes: + options["organizationSlug"] = changes["organizationSlug"] + if "projectSlug" in changes: + options["projectSlug"] = changes["projectSlug"] + if "token" in changes: + options["token"] = changes["token"] + + return log_tools.edit(project_id=project_id, integration=IN_TY, changes=changes) + + +def add(tenant_id, project_id, project_slug, organization_slug, token): + options = { + "organizationSlug": organization_slug, "projectSlug": project_slug, "token": token + } + return log_tools.add(project_id=project_id, integration=IN_TY, options=options) + + +def delete(tenant_id, project_id): + return log_tools.delete(project_id=project_id, integration=IN_TY) + + +def add_edit(tenant_id, project_id, data): + s = get(project_id) + if s is not None: + return update(tenant_id=tenant_id, project_id=project_id, + changes={"projectSlug": data["projectSlug"], + "organizationSlug": data["organizationSlug"], + "token": data["token"]}) + else: + return add(tenant_id=tenant_id, + project_id=project_id, + project_slug=data["projectSlug"], + organization_slug=data["organizationSlug"], token=data["token"]) + + +def proxy_get(tenant_id, project_id, event_id): + i = get(project_id) + if i is None: + return {} + r = requests.get( + url="https://sentry.io/api/0/projects/%(organization_slug)s/%(project_slug)s/events/%(event_id)s/" % { + "organization_slug": i["organizationSlug"], "project_slug": i["projectSlug"], "event_id": event_id}, + headers={"Authorization": "Bearer " + i["token"]}) + if r.status_code != 200: + print("=======> sentry get: something went wrong") + print(r) + print(r.status_code) + print(r.text) + return r.json() diff --git a/api/chalicelib/core/log_tool_stackdriver.py b/api/chalicelib/core/log_tool_stackdriver.py new file mode 100644 index 000000000..611a7c637 --- /dev/null +++ b/api/chalicelib/core/log_tool_stackdriver.py @@ -0,0 +1,42 @@ +from chalicelib.core import log_tools + +IN_TY = "stackdriver" + + +def get_all(tenant_id): + return log_tools.get_all_by_tenant(tenant_id=tenant_id, integration=IN_TY) + + +def get(project_id): + return log_tools.get(project_id=project_id, integration=IN_TY) + + +def update(tenant_id, project_id, changes): + options = {} + if "serviceAccountCredentials" in changes: + options["serviceAccountCredentials"] = changes["serviceAccountCredentials"] + if "logName" in changes: + options["logName"] = changes["logName"] + return log_tools.edit(project_id=project_id, integration=IN_TY, changes=options) + + +def add(tenant_id, project_id, service_account_credentials, log_name): + options = {"serviceAccountCredentials": service_account_credentials, "logName": log_name} + return log_tools.add(project_id=project_id, integration=IN_TY, options=options) + + +def delete(tenant_id, project_id): + return log_tools.delete(project_id=project_id, integration=IN_TY) + + +def add_edit(tenant_id, project_id, data): + s = get(project_id) + if s is not None: + return update(tenant_id=tenant_id, project_id=project_id, + changes={"serviceAccountCredentials": data["serviceAccountCredentials"], + "logName": data["logName"]}) + else: + return add(tenant_id=tenant_id, + project_id=project_id, + service_account_credentials=data["serviceAccountCredentials"], + log_name=data["logName"]) diff --git a/api/chalicelib/core/log_tool_sumologic.py b/api/chalicelib/core/log_tool_sumologic.py new file mode 100644 index 000000000..8e34ea70a --- /dev/null +++ b/api/chalicelib/core/log_tool_sumologic.py @@ -0,0 +1,53 @@ +from chalicelib.core import log_tools + +IN_TY = "sumologic" + + +def get_all(tenant_id): + return log_tools.get_all_by_tenant(tenant_id=tenant_id, integration=IN_TY) + + +def get(project_id): + return log_tools.get(project_id=project_id, integration=IN_TY) + + +def update(tenant_id, project_id, changes): + options = {} + + if "region" in changes: + options["region"] = changes["region"] + + if "accessId" in changes: + options["accessId"] = changes["accessId"] + + if "accessKey" in changes: + options["accessKey"] = changes["accessKey"] + return log_tools.edit(project_id=project_id, integration=IN_TY, changes=options) + + +def add(tenant_id, project_id, access_id, access_key, region): + options = { + "accessId": access_id, + "accessKey": access_key, + "region": region + } + return log_tools.add(project_id=project_id, integration=IN_TY, options=options) + + +def delete(tenant_id, project_id): + return log_tools.delete(project_id=project_id, integration=IN_TY) + + +def add_edit(tenant_id, project_id, data): + s = get(project_id) + if s is not None: + return update(tenant_id=tenant_id, project_id=project_id, + changes={"accessId": data["accessId"], + "accessKey": data["accessKey"], + "region": data["region"]}) + else: + return add(tenant_id=tenant_id, + project_id=project_id, + access_id=data["accessId"], + access_key=data["accessKey"], + region=data["region"]) diff --git a/api/chalicelib/core/log_tools.py b/api/chalicelib/core/log_tools.py new file mode 100644 index 000000000..31d7e8abc --- /dev/null +++ b/api/chalicelib/core/log_tools.py @@ -0,0 +1,118 @@ +from chalicelib.utils import pg_client, helper +import json + +EXCEPT = ["jira_server", "jira_cloud"] + + +def search(project_id): + result = [] + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """\ + SELECT supported_integrations.name, + (SELECT COUNT(*) + FROM public.integrations + INNER JOIN public.projects USING (project_id) + WHERE provider = supported_integrations.name + AND project_id = %(project_id)s + AND projects.deleted_at ISNULL + LIMIT 1) AS count + FROM unnest(enum_range(NULL::integration_provider)) AS supported_integrations(name);""", + {"project_id": project_id}) + ) + r = cur.fetchall() + for k in r: + if k["count"] > 0 and k["name"] not in EXCEPT: + result.append({"value": helper.key_to_camel_case(k["name"]), "type": "logTool"}) + return {"data": result} + + +def add(project_id, integration, options): + options = json.dumps(options) + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """\ + INSERT INTO public.integrations(project_id, provider, options) + VALUES (%(project_id)s, %(provider)s, %(options)s::jsonb) + RETURNING *;""", + {"project_id": project_id, "provider": integration, "options": options}) + ) + r = cur.fetchone() + return helper.dict_to_camel_case(helper.flatten_nested_dicts(r)) + + +def get(project_id, integration): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """\ + SELECT integrations.* + FROM public.integrations INNER JOIN public.projects USING(project_id) + WHERE provider = %(provider)s + AND project_id = %(project_id)s + AND projects.deleted_at ISNULL + LIMIT 1;""", + {"project_id": project_id, "provider": integration}) + ) + r = cur.fetchone() + return helper.dict_to_camel_case(helper.flatten_nested_dicts(r)) + + +def get_all_by_type(integration): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """\ + SELECT integrations.* + FROM public.integrations INNER JOIN public.projects USING(project_id) + WHERE provider = %(provider)s AND projects.deleted_at ISNULL;""", + {"provider": integration}) + ) + r = cur.fetchall() + return helper.list_to_camel_case(r, flatten=True) + + +def edit(project_id, integration, changes): + if "projectId" in changes: + changes.pop("project_id") + if "integration" in changes: + changes.pop("integration") + if len(changes.keys()) == 0: + return None + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + UPDATE public.integrations + SET options=options||%(changes)s + WHERE project_id =%(project_id)s AND provider = %(provider)s + RETURNING *;""", + {"project_id": project_id, "provider": integration, "changes": json.dumps(changes)}) + ) + return helper.dict_to_camel_case(helper.flatten_nested_dicts(cur.fetchone())) + + +def delete(project_id, integration): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + DELETE FROM public.integrations + WHERE project_id=%(project_id)s AND provider=%(provider)s;""", + {"project_id": project_id, "provider": integration}) + ) + return {"state": "success"} + + +def get_all_by_tenant(tenant_id, integration): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT integrations.* + FROM public.integrations INNER JOIN public.projects USING(project_id) + WHERE provider = %(provider)s + AND projects.deleted_at ISNULL;""", + {"provider": integration}) + ) + r = cur.fetchall() + return helper.list_to_camel_case(r, flatten=True) diff --git a/api/chalicelib/core/metadata.py b/api/chalicelib/core/metadata.py new file mode 100644 index 000000000..ed983f235 --- /dev/null +++ b/api/chalicelib/core/metadata.py @@ -0,0 +1,262 @@ +from chalicelib.utils import pg_client, helper, dev + + +from chalicelib.core import projects +import re + +MAX_INDEXES = 10 + + +def _get_column_names(): + return [f"metadata_{i}" for i in range(1, MAX_INDEXES + 1)] + + +def get(project_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""\ + SELECT + {",".join(_get_column_names())} + FROM public.projects + WHERE project_id = %(project_id)s AND deleted_at ISNULL + LIMIT 1;""", {"project_id": project_id}) + ) + metas = cur.fetchone() + results = [] + for i, k in enumerate(metas.keys()): + if metas[k] is not None: + results.append({"key": metas[k], "index": i + 1}) + return results + + +regex = re.compile(r'^[a-z0-9_-]+$', re.IGNORECASE) + + +def index_to_colname(index): + if index <= 0 or index > MAX_INDEXES: + raise Exception("metadata index out or bound") + return f"metadata_{index}" + + +def __get_available_index(project_id): + used_indexs = get(project_id) + used_indexs = [i["index"] for i in used_indexs] + if len(used_indexs) >= MAX_INDEXES: + return -1 + i = 1 + while i in used_indexs: + i += 1 + return i + + +def __edit(project_id, col_index, colname, new_name): + if new_name is None or len(new_name) == 0: + return {"errors": ["key value invalid"]} + old_metas = get(project_id) + old_metas = {k["index"]: k for k in old_metas} + if col_index not in list(old_metas.keys()): + return {"errors": ["custom field doesn't exist"]} + + with pg_client.PostgresClient() as cur: + if old_metas[col_index]["key"].lower() != new_name: + cur.execute(cur.mogrify(f"""UPDATE public.projects + SET {colname} = %(value)s + WHERE project_id = %(project_id)s AND deleted_at ISNULL + RETURNING {colname};""", + {"project_id": project_id, "value": new_name})) + new_name = cur.fetchone()[colname] + old_metas[col_index]["key"] = new_name + return {"data": old_metas[col_index]} + + +def edit(tenant_id, project_id, index: int, new_name: str): + return __edit(project_id=project_id, col_index=index, colname=index_to_colname(index), new_name=new_name) + + +def delete(tenant_id, project_id, index: int): + index = int(index) + old_segments = get(project_id) + old_segments = [k["index"] for k in old_segments] + if index not in old_segments: + return {"errors": ["custom field doesn't exist"]} + + with pg_client.PostgresClient() as cur: + colname = index_to_colname(index) + query = cur.mogrify(f"""UPDATE public.projects + SET {colname}= NULL + WHERE project_id = %(project_id)s AND deleted_at ISNULL;""", + {"project_id": project_id}) + cur.execute(query=query) + query = cur.mogrify(f"""UPDATE public.sessions + SET {colname}= NULL + WHERE project_id = %(project_id)s""", + {"project_id": project_id}) + cur.execute(query=query) + + return {"data": get(project_id)} + + +def add(tenant_id, project_id, new_name): + index = __get_available_index(project_id=project_id) + if index < 1: + return {"errors": ["maximum allowed metadata reached"]} + with pg_client.PostgresClient() as cur: + colname = index_to_colname(index) + cur.execute( + cur.mogrify( + f"""UPDATE public.projects SET {colname}= %(key)s WHERE project_id =%(project_id)s RETURNING {colname};""", + {"key": new_name, "project_id": project_id})) + col_val = cur.fetchone()[colname] + return {"data": {"key": col_val, "index": index}} + + +def search(tenant_id, project_id, key, value): + value = value + "%" + s_query = [] + for f in _get_column_names(): + s_query.append(f"CASE WHEN {f}=%(key)s THEN TRUE ELSE FALSE END AS {f}") + + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""\ + SELECT + {",".join(s_query)} + FROM public.projects + WHERE + project_id = %(project_id)s AND deleted_at ISNULL + LIMIT 1;""", + {"key": key, "project_id": project_id}) + ) + all_metas = cur.fetchone() + key = None + for c in all_metas: + if all_metas[c]: + key = c + break + if key is None: + return {"errors": ["key does not exist"]} + cur.execute( + cur.mogrify( + f"""\ + SELECT + DISTINCT "{key}" AS "{key}" + FROM public.sessions + {f'WHERE "{key}"::text ILIKE %(value)s' if value is not None and len(value) > 0 else ""} + ORDER BY "{key}" + LIMIT 20;""", + {"value": value, "project_id": project_id}) + ) + value = cur.fetchall() + return {"data": [k[key] for k in value]} + + +def get_available_keys(project_id): + all_metas = get(project_id=project_id) + return [k["key"] for k in all_metas] + + +def get_by_session_id(project_id, session_id): + all_metas = get(project_id=project_id) + if len(all_metas) == 0: + return [] + keys = {index_to_colname(k["index"]): k["key"] for k in all_metas} + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""\ + select {",".join(keys.keys())} + FROM public.sessions + WHERE project_id= %(project_id)s AND session_id=%(session_id)s;""", + {"session_id": session_id, "project_id": project_id}) + ) + session_metas = cur.fetchall() + results = [] + for m in session_metas: + r = {} + for k in m.keys(): + r[keys[k]] = m[k] + results.append(r) + return results + + +def get_keys_by_projects(project_ids): + if project_ids is None or len(project_ids) == 0: + return {} + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + f"""\ + SELECT + project_id, + {",".join(_get_column_names())} + FROM public.projects + WHERE project_id IN %(project_ids)s AND deleted_at ISNULL;""", + {"project_ids": tuple(project_ids)}) + + cur.execute(query) + rows = cur.fetchall() + results = {} + for r in rows: + project_id = r.pop("project_id") + results[project_id] = {} + for m in r: + if r[m] is not None: + results[project_id][m] = r[m] + return results + + +def add_edit_delete(tenant_id, project_id, new_metas): + old_metas = get(project_id) + old_indexes = [k["index"] for k in old_metas] + new_indexes = [k["index"] for k in new_metas if "index" in k] + new_keys = [k["key"] for k in new_metas] + + add_metas = [k["key"] for k in new_metas + if "index" not in k] + new_metas = {k["index"]: {"key": k["key"]} for + k in new_metas if + "index" in k} + old_metas = {k["index"]: {"key": k["key"]} for k in old_metas} + + if len(new_keys) > 20: + return {"errors": ["you cannot add more than 20 key"]} + for k in new_metas.keys(): + if re.match(regex, new_metas[k]["key"]) is None: + return {"errors": [f"invalid key {k}"]} + for k in add_metas: + if re.match(regex, k) is None: + return {"errors": [f"invalid key {k}"]} + if len(new_indexes) > len(set(new_indexes)): + return {"errors": ["duplicate indexes"]} + if len(new_keys) > len(set(new_keys)): + return {"errors": ["duplicate keys"]} + to_delete = list(set(old_indexes) - set(new_indexes)) + + with pg_client.PostgresClient() as cur: + for d in to_delete: + delete(tenant_id=tenant_id, project_id=project_id, index=d) + + for k in add_metas: + add(tenant_id=tenant_id, project_id=project_id, new_name=k) + + for k in new_metas.keys(): + if new_metas[k]["key"].lower() != old_metas[k]["key"]: + edit(tenant_id=tenant_id, project_id=project_id, index=k, new_name=new_metas[k]["key"]) + + return {"data": get(project_id)} + + +@dev.timed +def get_remaining_metadata_with_count(tenant_id): + all_projects = projects.get_projects(tenant_id=tenant_id) + results = [] + for p in all_projects: + used_metas = get(p["projectId"]) + if MAX_INDEXES < 0: + remaining = -1 + else: + remaining = MAX_INDEXES - len(used_metas) + results.append({**p, "limit": MAX_INDEXES, "remaining": remaining, "count": len(used_metas)}) + + return results diff --git a/api/chalicelib/core/notifications.py b/api/chalicelib/core/notifications.py new file mode 100644 index 000000000..0d9b5be20 --- /dev/null +++ b/api/chalicelib/core/notifications.py @@ -0,0 +1,80 @@ +import json + +from chalicelib.utils import pg_client, helper +from chalicelib.utils.TimeUTC import TimeUTC + + +def get_all(tenant_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + SELECT notifications.*, + user_viewed_notifications.notification_id NOTNULL AS viewed + FROM public.notifications + LEFT JOIN (SELECT notification_id + FROM public.user_viewed_notifications + WHERE user_viewed_notifications.user_id = %(user_id)s) AS user_viewed_notifications USING (notification_id) + WHERE notifications.user_id IS NULL OR notifications.user_id =%(user_id)s + ORDER BY created_at DESC + LIMIT 100;""", + {"user_id": user_id}) + ) + rows = helper.list_to_camel_case(cur.fetchall()) + for r in rows: + r["createdAt"] = TimeUTC.datetime_to_timestamp(r["createdAt"]) + return rows + + +def view_notification(user_id, notification_ids=[], tenant_id=None, startTimestamp=None, endTimestamp=None): + if (notification_ids is None or len(notification_ids) == 0) and endTimestamp is None: + return False + if startTimestamp is None: + startTimestamp = 0 + notification_ids = [(user_id, id) for id in notification_ids] + with pg_client.PostgresClient() as cur: + if len(notification_ids) > 0: + cur.executemany( + "INSERT INTO public.user_viewed_notifications(user_id, notification_id) VALUES (%s,%s) ON CONFLICT DO NOTHING;", + notification_ids) + else: + query = """INSERT INTO public.user_viewed_notifications(user_id, notification_id) + SELECT %(user_id)s AS user_id, notification_id + FROM public.notifications + WHERE (user_id IS NULL OR user_id =%(user_id)s) + AND EXTRACT(EPOCH FROM created_at)*1000>=(%(startTimestamp)s) + AND EXTRACT(EPOCH FROM created_at)*1000<=(%(endTimestamp)s+1000) + ON CONFLICT DO NOTHING;""" + params = {"user_id": user_id, "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp} + # print('-------------------') + # print(cur.mogrify(query, params)) + cur.execute(cur.mogrify(query, params)) + return True + + +def create(notifications): + if len(notifications) == 0: + return [] + with pg_client.PostgresClient() as cur: + values = [] + for n in notifications: + clone = dict(n) + if "userId" not in clone: + clone["userId"] = None + if "options" not in clone: + clone["options"] = '{}' + else: + clone["options"] = json.dumps(clone["options"]) + values.append( + cur.mogrify( + "(%(userId)s, %(title)s, %(description)s, %(buttonText)s, %(buttonUrl)s, %(imageUrl)s,%(options)s)", + clone).decode('UTF-8') + ) + cur.execute( + f"""INSERT INTO public.notifications(user_id, title, description, button_text, button_url, image_url, options) + VALUES {",".join(values)} RETURNING *;""") + rows = helper.list_to_camel_case(cur.fetchall()) + for r in rows: + r["createdAt"] = TimeUTC.datetime_to_timestamp(r["createdAt"]) + r["viewed"] = False + return rows diff --git a/api/chalicelib/core/projects.py b/api/chalicelib/core/projects.py new file mode 100644 index 000000000..62f74961b --- /dev/null +++ b/api/chalicelib/core/projects.py @@ -0,0 +1,229 @@ +import json + +from chalicelib.core import users +from chalicelib.utils import pg_client, helper, dev +from chalicelib.utils.TimeUTC import TimeUTC + + +def __update(tenant_id, project_id, changes): + if len(changes.keys()) == 0: + return None + + sub_query = [] + for key in changes.keys(): + sub_query.append(f"{helper.key_to_snake_case(key)} = %({key})s") + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""\ + UPDATE public.projects + SET + {" ,".join(sub_query)} + WHERE + project_id = %(project_id)s + AND deleted_at ISNULL + RETURNING project_id,name,gdpr;""", + {"project_id": project_id, **changes}) + ) + return helper.dict_to_camel_case(cur.fetchone()) + + +def __create(tenant_id, name): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""\ + INSERT INTO public.projects (name, active) + VALUES (%(name)s,TRUE) + RETURNING project_id;""", + {"name": name}) + ) + project_id = cur.fetchone()["project_id"] + return get_project(tenant_id=tenant_id, project_id=project_id, include_gdpr=True) + + +@dev.timed +def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, stack_integrations=False): + with pg_client.PostgresClient() as cur: + cur.execute(f"""\ + SELECT + s.project_id, s.name, s.project_key + {',s.gdpr' if gdpr else ''} + {',COALESCE((SELECT TRUE FROM public.sessions WHERE sessions.project_id = s.project_id LIMIT 1), FALSE) AS recorded' if recorded else ''} + {',stack_integrations.count>0 AS stack_integrations' if stack_integrations else ''} + FROM public.projects AS s + {'LEFT JOIN LATERAL (SELECT COUNT(*) AS count FROM public.integrations WHERE s.project_id = integrations.project_id LIMIT 1) AS stack_integrations ON TRUE' if stack_integrations else ''} + where s.deleted_at IS NULL + ORDER BY s.project_id;""" + ) + rows = cur.fetchall() + if recording_state: + for r in rows: + query = cur.mogrify( + "select COALESCE(MAX(start_ts),0) AS last from public.sessions where project_id=%(project_id)s;", + {"project_id": r["project_id"]}) + cur.execute( + query=query + ) + status = cur.fetchone() + if status["last"] < TimeUTC.now(-2): + r["status"] = "red" + elif status["last"] < TimeUTC.now(-1): + r["status"] = "yellow" + else: + r["status"] = "green" + + return helper.list_to_camel_case(rows) + + +def get_project(tenant_id, project_id, include_last_session=False, include_gdpr=None): + with pg_client.PostgresClient() as cur: + query = cur.mogrify(f"""\ + SELECT + s.project_id, + s.project_key, + s.name + {",(SELECT max(ss.start_ts) FROM public.sessions AS ss WHERE ss.project_id = %(project_id)s) AS last_recorded_session_at" if include_last_session else ""} + {',s.gdpr' if include_gdpr else ''} + FROM public.projects AS s + where s.project_id =%(project_id)s + AND s.deleted_at IS NULL + LIMIT 1;""", + {"project_id": project_id}) + + cur.execute( + query=query + ) + row = cur.fetchone() + return helper.dict_to_camel_case(row) + + +def create(tenant_id, user_id, data): + admin = users.get(user_id=user_id, tenant_id=tenant_id) + if not admin["admin"] and not admin["superAdmin"]: + return {"errors": ["unauthorized"]} + return {"data": __create(tenant_id=tenant_id, name=data.get("name", "my first project"))} + + +def edit(tenant_id, user_id, project_id, data): + admin = users.get(user_id=user_id, tenant_id=tenant_id) + if not admin["admin"] and not admin["superAdmin"]: + return {"errors": ["unauthorized"]} + return {"data": __update(tenant_id=tenant_id, project_id=project_id, + changes={"name": data.get("name", "my first project")})} + + +def delete(tenant_id, user_id, project_id): + admin = users.get(user_id=user_id, tenant_id=tenant_id) + + if not admin["admin"] and not admin["superAdmin"]: + return {"errors": ["unauthorized"]} + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + UPDATE public.projects + SET + deleted_at = timezone('utc'::text, now()), + active = FALSE + WHERE + project_id = %(project_id)s;""", + {"project_id": project_id}) + ) + return {"data": {"state": "success"}} + + +def count_by_tenant(tenant_id): + with pg_client.PostgresClient() as cur: + cur.execute("""\ + SELECT + count(s.project_id) + FROM public.projects AS s + where s.deleted_at IS NULL;""") + return cur.fetchone()["count"] + + +def get_gdpr(project_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + SELECT + gdpr + FROM public.projects AS s + where s.project_id =%(project_id)s + AND s.deleted_at IS NULL;""", + {"project_id": project_id}) + ) + return cur.fetchone()["gdpr"] + + +def edit_gdpr(project_id, gdpr): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + UPDATE public.projects + SET + gdpr = gdpr|| %(gdpr)s + WHERE + project_id = %(project_id)s + AND deleted_at ISNULL + RETURNING gdpr;""", + {"project_id": project_id, "gdpr": json.dumps(gdpr)}) + ) + return cur.fetchone()["gdpr"] + + +def get_internal_project_id(project_key): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + SELECT project_id + FROM public.projects + where project_key =%(project_key)s AND deleted_at ISNULL;""", + {"project_key": project_key}) + ) + return cur.fetchone()["project_id"] + + +def get_project_key(project_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + SELECT project_key + FROM public.projects + where project_id =%(project_id)s AND deleted_at ISNULL;""", + {"project_id": project_id}) + ) + return cur.fetchone()["project_key"] + + +def get_capture_status(project_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + SELECT + sample_rate AS rate, sample_rate=100 AS capture_all + FROM public.projects + where project_id =%(project_id)s AND deleted_at ISNULL;""", + {"project_id": project_id}) + ) + return helper.dict_to_camel_case(cur.fetchone()) + + +def update_capture_status(project_id, changes): + if "rate" not in changes and "captureAll" not in changes: + return {"errors": ["please provide 'rate' and/or 'captureAll' attributes to update."]} + if int(changes["rate"]) < 0 or int(changes["rate"]) > 100: + return {"errors": ["'rate' must be between 0..100."]} + sample_rate = 0 + if "rate" in changes: + sample_rate = int(changes["rate"]) + if changes.get("captureAll"): + sample_rate = 100 + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + UPDATE public.projects + SET sample_rate= %(sample_rate)s + WHERE project_id =%(project_id)s AND deleted_at ISNULL;""", + {"project_id": project_id, "sample_rate": sample_rate}) + ) + + return changes diff --git a/api/chalicelib/core/reset_password.py b/api/chalicelib/core/reset_password.py new file mode 100644 index 000000000..a792bfbc1 --- /dev/null +++ b/api/chalicelib/core/reset_password.py @@ -0,0 +1,66 @@ +import chalicelib.utils.TimeUTC +from chalicelib.utils import email_helper, captcha, helper +import secrets +from chalicelib.utils import pg_client + +from chalicelib.core import users + + +def step1(data): + print("====================== reset password 1 ===============") + print(data) + if helper.allow_captcha() and not captcha.is_valid(data["g-recaptcha-response"]): + print("error: Invalid captcha.") + return {"errors": ["Invalid captcha."]} + if "email" not in data: + return {"errors": ["email not found in body"]} + + a_users = users.get_by_email_only(data["email"]) + if len(a_users) > 1: + print(f"multiple users found for [{data['email']}] please contact our support") + return {"errors": ["please contact our support"]} + elif len(a_users) == 1: + a_users = a_users[0] + reset_token = secrets.token_urlsafe(6) + users.update(tenant_id=a_users["tenantId"], user_id=a_users["id"], + changes={"token": reset_token}) + email_helper.send_reset_code(recipient=data["email"], reset_code=reset_token) + else: + print(f"invalid email address [{data['email']}]") + return {"errors": ["invalid email address"]} + return {"data": {"state": "success"}} + + +def step2(data): + print("====================== change password 2 ===============") + user = users.get_by_email_reset(data["email"], data["code"]) + if not user: + print("error: wrong email or reset code") + return {"errors": ["wrong email or reset code"]} + users.update(tenant_id=user["tenantId"], user_id=user["id"], + changes={"token": None, "password": data["password"], "generatedPassword": False, + "verifiedEmail": True}) + return {"data": {"state": "success"}} + + +def cron(): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + SELECT user_id + FROM public.basic_authentication + WHERE token notnull + AND (token_requested_at isnull or (EXTRACT(EPOCH FROM token_requested_at)*1000)::BIGINT < %(time)s);""", + {"time": chalicelib.utils.TimeUTC.TimeUTC.now(delta_days=-1)}) + ) + results = cur.fetchall() + if len(results) == 0: + return + results = tuple([r["user_id"] for r in results]) + cur.execute( + cur.mogrify("""\ + UPDATE public.basic_authentication + SET token = NULL, token_requested_at = NULL + WHERE user_id in %(ids)s;""", + {"ids": results}) + ) diff --git a/api/chalicelib/core/resources.py b/api/chalicelib/core/resources.py new file mode 100644 index 000000000..6b9ba9170 --- /dev/null +++ b/api/chalicelib/core/resources.py @@ -0,0 +1,22 @@ +from chalicelib.utils import helper, pg_client + + +def get_by_session_id(session_id): + with pg_client.PostgresClient() as cur: + ch_query = """\ + SELECT + timestamp AS datetime, + url, + type, + duration, + ttfb, + header_size, + encoded_body_size, + decoded_body_size, + success + FROM events.resources + WHERE session_id = %(session_id)s;""" + params = {"session_id": session_id} + cur.execute(cur.mogrify(ch_query, params)) + rows = cur.fetchall() + return helper.list_to_camel_case(rows) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py new file mode 100644 index 000000000..480877468 --- /dev/null +++ b/api/chalicelib/core/sessions.py @@ -0,0 +1,620 @@ +from chalicelib.utils import pg_client, helper +from chalicelib.utils import dev +from chalicelib.core import events, sessions_metas, socket_ios, metadata, events_ios, sessions_mobs + +if helper.is_free_open_source_edition(): + from chalicelib.core import projects, errors +else: + from chalicelib.ee import projects, errors + +from chalicelib.core import resources + +SESSION_PROJECTION_COLS = """s.project_id, + s.session_id::text AS session_id, + s.user_uuid, + s.user_id, + s.user_agent, + s.user_os, + s.user_browser, + s.user_device, + s.user_device_type, + s.user_country, + s.start_ts, + s.duration, + s.events_count, + s.pages_count, + s.errors_count, + s.user_anonymous_id, + s.platform, + s.issue_score, + s.issue_types::text[] AS issue_types, + favorite_sessions.session_id NOTNULL AS favorite, + COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + """ + + +def __group_metadata(session, project_metadata): + meta = [] + for m in project_metadata.keys(): + if project_metadata[m] is not None and session.get(m) is not None: + meta.append({project_metadata[m]: session[m]}) + session.pop(m) + return meta + + +def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_viewed=False, group_metadata=False): + with pg_client.PostgresClient() as cur: + extra_query = [] + if include_fav_viewed: + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite""") + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS viewed""") + query = cur.mogrify( + f"""\ + SELECT + s.*, + s.session_id::text AS session_id + {"," if len(extra_query) > 0 else ""}{",".join(extra_query)} + {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata._get_column_names()]) + ") AS project_metadata") if group_metadata else ''} + FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""} + WHERE s.project_id = %(project_id)s + AND s.session_id = %(session_id)s;""", + {"project_id": project_id, "session_id": session_id, "userId": user_id} + ) + # print("===============") + # print(query) + cur.execute(query=query) + + data = cur.fetchone() + if data is not None: + data = helper.dict_to_camel_case(data) + if full_data: + if data["platform"] == 'ios': + data['events'] = events_ios.get_by_sessionId(project_id=project_id, session_id=session_id) + for e in data['events']: + if e["type"].endswith("_IOS"): + e["type"] = e["type"][:-len("_IOS")] + data['crashes'] = events_ios.get_crashes_by_session_id(session_id=session_id) + data['userEvents'] = events_ios.get_customs_by_sessionId(project_id=project_id, + session_id=session_id) + data['mobsUrl'] = sessions_mobs.get_ios(sessionId=session_id) + data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) + data["socket"] = socket_ios.start_replay(project_id=project_id, session_id=session_id, + device=data["userDevice"], + os_version=data["userOsVersion"], + mob_url=data["mobsUrl"]) + else: + data['events'] = events.get_by_sessionId2_pg(project_id=project_id, session_id=session_id, + group_clickrage=True) + all_errors = events.get_errors_by_session_id(session_id=session_id) + data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] + # to keep only the first stack + data['errors'] = [errors.format_first_stack_frame(e) for e in all_errors if + e['source'] == "js_exception"][ + :500] # limit the number of errors to reduce the response-body size + data['userEvents'] = events.get_customs_by_sessionId2_pg(project_id=project_id, + session_id=session_id) + data['mobsUrl'] = sessions_mobs.get_web(sessionId=session_id) + data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) + data['resources'] = resources.get_by_session_id(session_id=session_id) + + return data + return None + + +def sessions_args(args, params): + if params is not None: + for key in ['userOs', 'userBrowser', 'userCountry', 'path', 'path_in_order', 'after', 'minDuration', + 'maxDuration', 'sortSessions', 'eventsCount', 'consoleLogCount', 'startDate', 'endDate', + 'consoleLog', 'location']: + args[key] = params.get(key) + + +new_line = "\n" + + +def __get_sql_operator(op): + op = op.lower() + return "=" if op == "is" or op == "on" else "!=" if op == "isnot" else "ILIKE" if op == "contains" else "NOT ILIKE" if op == "notcontains" else "=" + + +def __is_negation_operator(op): + return op in ("!=", "NOT ILIKE") + + +def __reverse_sql_operator(op): + return "=" if op == "!=" else "!=" if op == "=" else "ILIKE" if op == "NOT ILIKE" else "NOT ILIKE" + + +def __get_sql_operator_multiple(op): + op = op.lower() + return " IN " if op == "is" else " NOT IN " + + +def __get_sql_operator_boolean(op): + op = op.lower() + return True if op == "true" else False + + +def __get_sql_value_multiple(values): + if isinstance(values, tuple): + return values + return tuple([v for v in values]) + + +@dev.timed +def search2_pg(data, project_id, user_id, favorite_only=False, errors_only=False, error_status="ALL", + count_only=False, issue=None): + sessions = [] + generic_args = {"startDate": data['startDate'], "endDate": data['endDate'], + "projectId": project_id, + "userId": user_id} + with pg_client.PostgresClient() as cur: + extra_constraints = [ + cur.mogrify("s.project_id = %(project_id)s", {"project_id": project_id}), + cur.mogrify("s.duration IS NOT NULL", {}) + ] + extra_from = "" + fav_only_join = "" + if favorite_only and not errors_only: + fav_only_join = "LEFT JOIN public.user_favorite_sessions AS fs ON fs.session_id = s.session_id" + extra_constraints.append(cur.mogrify("fs.user_id = %(userId)s", {"userId": user_id})) + events_query_part = "" + strict = True + + if len(data.get("events", [])) > 0: + events_query_from = [] + event_index = 0 + + for event in data["events"]: + # TODO: remove this when message_id is removed + seq_id = False + event_type = event["type"].upper() + if event.get("operator") is None: + event["operator"] = "is" + op = __get_sql_operator(event["operator"]) + is_not = False + if __is_negation_operator(op) and event_index > 0: + is_not = True + op = __reverse_sql_operator(op) + event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" + event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", + "main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s", + "ms.start_ts <= %(endDate)s"] + event_args = {"value": helper.string_to_sql_like_with_op(event['value'], op)} + if event_type not in list(events.SUPPORTED_TYPES.keys()) \ + or event.get("value") in [None, "", "*"] \ + and (event_type != events.event_type.ERROR.ui_type \ + or event_type != events.event_type.ERROR_IOS.ui_type): + continue + if event_type == events.event_type.CLICK.ui_type: + event_from = event_from % f"{events.event_type.CLICK.table} AS main " + event_where.append(f"main.{events.event_type.CLICK.column} {op} %(value)s") + + elif event_type == events.event_type.INPUT.ui_type: + event_from = event_from % f"{events.event_type.INPUT.table} AS main " + event_where.append(f"main.{events.event_type.INPUT.column} {op} %(value)s") + if len(event.get("custom", "")) > 0: + event_where.append("main.value ILIKE %(custom)s") + event_args["custom"] = helper.string_to_sql_like_with_op(event['custom'], "ILIKE") + elif event_type == events.event_type.LOCATION.ui_type: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + event_where.append(f"main.{events.event_type.LOCATION.column} {op} %(value)s") + elif event_type == events.event_type.CUSTOM.ui_type: + seq_id = True + event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " + event_where.append(f"main.{events.event_type.CUSTOM.column} {op} %(value)s") + elif event_type == events.event_type.REQUEST.ui_type: + seq_id = True + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + event_where.append(f"main.{events.event_type.REQUEST.column} {op} %(value)s") + elif event_type == events.event_type.GRAPHQL.ui_type: + event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " + event_where.append(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s") + elif event_type == events.event_type.STATEACTION.ui_type: + event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " + event_where.append(f"main.{events.event_type.STATEACTION.column} {op} %(value)s") + elif event_type == events.event_type.ERROR.ui_type: + if event.get("source") in [None, "*", ""]: + event["source"] = "js_exception" + event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" + if event.get("value") not in [None, "*", ""]: + event_where.append(f"(main1.message {op} %(value)s OR main1.name {op} %(value)s)") + if event.get("source") not in [None, "*", ""]: + event_where.append(f"main1.source = %(source)s") + event_args["source"] = event["source"] + elif event.get("source") not in [None, "*", ""]: + event_where.append(f"main1.source = %(source)s") + event_args["source"] = event["source"] + + # ----- IOS + elif event_type == events.event_type.CLICK_IOS.ui_type: + seq_id = True + event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " + event_where.append(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s") + + elif event_type == events.event_type.INPUT_IOS.ui_type: + seq_id = True + event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " + event_where.append(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s") + + if len(event.get("custom", "")) > 0: + event_where.append("main.value ILIKE %(custom)s") + event_args["custom"] = helper.string_to_sql_like_with_op(event['custom'], "ILIKE") + elif event_type == events.event_type.VIEW_IOS.ui_type: + seq_id = True + event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " + event_where.append(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s") + elif event_type == events.event_type.CUSTOM_IOS.ui_type: + seq_id = True + event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " + event_where.append(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s") + elif event_type == events.event_type.REQUEST_IOS.ui_type: + seq_id = True + event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " + event_where.append(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s") + elif event_type == events.event_type.ERROR_IOS.ui_type: + seq_id = True + event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" + if event.get("value") not in [None, "*", ""]: + event_where.append(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)") + + else: + continue + + event_index += 1 + if is_not: + event_from += f""" LEFT JOIN (SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) AS left_not USING (session_id)""" + event_where[-1] = "left_not.session_id ISNULL" + events_query_from.append(cur.mogrify(f"""\ + (SELECT + main.session_id, {'seq_index' if seq_id else 'message_id %%%% 2147483647 AS seq_index'}, timestamp, {event_index} AS funnel_step + FROM {event_from} + WHERE {" AND ".join(event_where)} + )\ + """, {**generic_args, **event_args}).decode('UTF-8')) + + if len(events_query_from) > 0: + events_query_part = f"""\ + SELECT + session_id, MIN(timestamp) AS first_event_ts, MAX(timestamp) AS last_event_ts + FROM + ({(" UNION ALL ").join(events_query_from)}) AS f_query + GROUP BY 1 + {"" if event_index < 2 else f"HAVING events.funnel(array_agg(funnel_step ORDER BY timestamp,seq_index ASC), {event_index})" if strict + else f"HAVING array_length(array_agg(DISTINCT funnel_step), 1) = {len(data['events'])}"} + {fav_only_join} + """ + else: + data["events"] = [] + + # --------------------------------------------------------------------------- + if "filters" in data: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + for f in data["filters"]: + if not isinstance(f.get("value"), list): + f["value"] = [f.get("value")] + if len(f["value"]) == 0 or f["value"][0] is None: + continue + filter_type = f["type"].upper() + f["value"] = __get_sql_value_multiple(f["value"]) + if filter_type == sessions_metas.meta_type.USERBROWSER: + op = __get_sql_operator_multiple(f["operator"]) + extra_constraints.append( + cur.mogrify(f's.user_browser {op} %(value)s', {"value": f["value"]})) + + elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: + op = __get_sql_operator_multiple(f["operator"]) + extra_constraints.append(cur.mogrify(f's.user_os {op} %(value)s', {"value": f["value"]})) + + elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: + op = __get_sql_operator_multiple(f["operator"]) + extra_constraints.append(cur.mogrify(f's.user_device {op} %(value)s', {"value": f["value"]})) + + elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: + op = __get_sql_operator_multiple(f["operator"]) + extra_constraints.append(cur.mogrify(f's.user_country {op} %(value)s', {"value": f["value"]})) + elif filter_type == "duration".upper(): + if len(f["value"]) > 0 and f["value"][0] is not None: + extra_constraints.append( + cur.mogrify("s.duration >= %(minDuration)s", {"minDuration": f["value"][0]})) + if len(f["value"]) > 1 and f["value"][1] is not None and f["value"][1] > 0: + extra_constraints.append( + cur.mogrify("s.duration <= %(maxDuration)s", {"maxDuration": f["value"][1]})) + elif filter_type == sessions_metas.meta_type.REFERRER: + # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" + extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" + op = __get_sql_operator_multiple(f["operator"]) + extra_constraints.append( + cur.mogrify(f"p.base_referrer {op} %(referrer)s", {"referrer": f["value"]})) + elif filter_type == events.event_type.METADATA.ui_type: + op = __get_sql_operator(f["operator"]) + if f.get("key") in meta_keys.keys(): + extra_constraints.append( + cur.mogrify(f"s.{metadata.index_to_colname(meta_keys[f['key']])} {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) + ) + elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + op = __get_sql_operator(f["operator"]) + extra_constraints.append( + cur.mogrify(f"s.user_id {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) + ) + elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, + sessions_metas.meta_type.USERANONYMOUSID_IOS]: + op = __get_sql_operator(f["operator"]) + extra_constraints.append( + cur.mogrify(f"s.user_anonymous_id {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) + ) + elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: + op = __get_sql_operator(f["operator"]) + extra_constraints.append( + cur.mogrify(f"s.rev_id {op} %(value)s", + {"value": helper.string_to_sql_like_with_op(f["value"][0], op)}) + ) + + # --------------------------------------------------------------------------- + + if data.get("startDate") is not None: + extra_constraints.append(cur.mogrify("s.start_ts >= %(startDate)s", {"startDate": data['startDate']})) + else: + data['startDate'] = None + if data.get("endDate") is not None: + extra_constraints.append(cur.mogrify("s.start_ts <= %(endDate)s", {"endDate": data['endDate']})) + else: + data['endDate'] = None + + if data.get('platform') is not None: + if data['platform'] == 'mobile': + extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") + elif data['platform'] == 'desktop': + extra_constraints.append( + b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") + + order = "DESC" + if data.get("order") is not None: + order = data["order"] + sort = 'session_id' + if data.get("sort") is not None and data["sort"] != "session_id": + sort += " " + order + "," + helper.key_to_snake_case(data["sort"]) + else: + sort = 'session_id' + + if errors_only: + extra_from += f" INNER JOIN {events.event_type.ERROR.table} AS er USING (session_id) INNER JOIN public.errors AS ser USING (error_id)" + extra_constraints.append(b"ser.source = 'js_exception'") + if error_status != "ALL": + extra_constraints.append(cur.mogrify("ser.status = %(status)s", {"status": error_status.lower()})) + if favorite_only: + extra_from += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" + extra_constraints.append(cur.mogrify("ufe.user_id = %(user_id)s", {"user_id": user_id})) + + extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] + if not favorite_only and not errors_only: + extra_from += """LEFT JOIN (SELECT user_id, session_id + FROM public.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + USING (session_id)""" + extra_join = "" + if issue is not None: + extra_join = cur.mogrify(""" + INNER JOIN LATERAL(SELECT TRUE FROM events_common.issues INNER JOIN public.issues AS p_issues USING (issue_id) + WHERE issues.session_id=f.session_id + AND p_issues.type=%(type)s + AND p_issues.context_string=%(contextString)s + AND timestamp >= f.first_event_ts + AND timestamp <= f.last_event_ts) AS issues ON(TRUE) + """, {"contextString": issue["contextString"], "type": issue["type"]}).decode('UTF-8') + + query_part = f"""\ + FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else "public.sessions AS s"} + {extra_join} + {"INNER JOIN public.sessions AS s USING(session_id)" if len(events_query_part) > 0 else ""} + {extra_from} + WHERE + + {" AND ".join(extra_constraints)}""" + + if errors_only: + main_query = cur.mogrify(f"""\ + SELECT DISTINCT er.error_id, ser.status, ser.parent_error_id, ser.payload, + COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite, + COALESCE((SELECT TRUE + FROM public.user_viewed_errors AS ve + WHERE er.error_id = ve.error_id + AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed + {query_part};""", + generic_args) + + elif count_only: + main_query = cur.mogrify(f"""\ + SELECT COUNT(DISTINCT s.session_id) AS count_sessions, COUNT(DISTINCT s.user_uuid) AS count_users + {query_part};""", + generic_args) + else: + main_query = cur.mogrify(f"""\ + SELECT * FROM + (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} + {query_part} + ORDER BY s.session_id desc) AS filtred_sessions + ORDER BY favorite DESC, issue_score DESC, {sort} {order};""", + generic_args) + + # print("--------------------") + # print(main_query) + + cur.execute(main_query) + + if count_only: + return helper.dict_to_camel_case(cur.fetchone()) + sessions = [] + total = cur.rowcount + row = cur.fetchone() + limit = 200 + while row is not None and len(sessions) < limit: + if row.get("favorite"): + limit += 1 + sessions.append(row) + row = cur.fetchone() + + if errors_only: + return sessions + if data.get("sort") is not None and data["sort"] != "session_id": + sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data["sort"])], + reverse=data.get("order", "DESC").upper() == "DESC") + return { + 'total': total, + 'sessions': helper.list_to_camel_case(sessions) + } + + +def search_by_metadata(tenant_id, user_id, m_key, m_value, project_id=None): + if project_id is None: + all_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) + else: + all_projects = [ + projects.get_project(tenant_id=tenant_id, project_id=int(project_id), include_last_session=False, + include_gdpr=False)] + + all_projects = {int(p["projectId"]): p["name"] for p in all_projects} + project_ids = list(all_projects.keys()) + + available_keys = metadata.get_keys_by_projects(project_ids) + for i in available_keys: + available_keys[i]["user_id"] = sessions_metas.meta_type.USERID + available_keys[i]["user_anonymous_id"] = sessions_metas.meta_type.USERANONYMOUSID + results = {} + for i in project_ids: + if m_key not in available_keys[i].values(): + available_keys.pop(i) + results[i] = {"total": 0, "sessions": [], "missingMetadata": True} + project_ids = list(available_keys.keys()) + if len(project_ids) > 0: + with pg_client.PostgresClient() as cur: + sub_queries = [] + for i in project_ids: + col_name = list(available_keys[i].keys())[list(available_keys[i].values()).index(m_key)] + sub_queries.append(cur.mogrify( + f"(SELECT COALESCE(COUNT(s.*)) AS count FROM public.sessions AS s WHERE s.project_id = %(id)s AND s.{col_name} = %(value)s) AS \"{i}\"", + {"id": i, "value": m_value}).decode('UTF-8')) + query = f"""SELECT {", ".join(sub_queries)};""" + cur.execute(query=query) + + rows = cur.fetchone() + + sub_queries = [] + for i in rows.keys(): + results[i] = {"total": rows[i], "sessions": [], "missingMetadata": False, "name": all_projects[int(i)]} + if rows[i] > 0: + col_name = list(available_keys[int(i)].keys())[list(available_keys[int(i)].values()).index(m_key)] + sub_queries.append( + cur.mogrify( + f"""( + SELECT * + FROM ( + SELECT DISTINCT ON(favorite_sessions.session_id, s.session_id) {SESSION_PROJECTION_COLS} + FROM public.sessions AS s LEFT JOIN (SELECT session_id + FROM public.user_favorite_sessions + WHERE user_favorite_sessions.user_id = %(userId)s + ) AS favorite_sessions USING (session_id) + WHERE s.project_id = %(id)s AND s.duration IS NOT NULL AND s.{col_name} = %(value)s + ) AS full_sessions + ORDER BY favorite DESC, issue_score DESC + LIMIT 10 + )""", + {"id": i, "value": m_value, "userId": user_id}).decode('UTF-8')) + if len(sub_queries) > 0: + cur.execute("\nUNION\n".join(sub_queries)) + rows = cur.fetchall() + for i in rows: + results[str(i["project_id"])]["sessions"].append(helper.dict_to_camel_case(i)) + return results + + +def search_by_issue(user_id, issue, project_id, start_date, end_date): + constraints = ["s.project_id = %(projectId)s", + "p_issues.context_string = %(issueContextString)s", + "p_issues.type = %(issueType)s"] + if start_date is not None: + constraints.append("start_ts >= %(startDate)s") + if end_date is not None: + constraints.append("start_ts <= %(endDate)s") + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""SELECT DISTINCT ON(favorite_sessions.session_id, s.session_id) {SESSION_PROJECTION_COLS} + FROM public.sessions AS s + INNER JOIN events_common.issues USING (session_id) + INNER JOIN public.issues AS p_issues USING (issue_id) + LEFT JOIN (SELECT user_id, session_id + FROM public.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + USING (session_id) + WHERE {" AND ".join(constraints)} + ORDER BY s.session_id DESC;""", + { + "issueContextString": issue["contextString"], + "issueType": issue["type"], "userId": user_id, + "projectId": project_id, + "startDate": start_date, + "endDate": end_date + })) + + rows = cur.fetchall() + return helper.list_to_camel_case(rows) + + +def get_favorite_sessions(project_id, user_id, include_viewed=False): + with pg_client.PostgresClient() as cur: + query_part = cur.mogrify(f"""\ + FROM public.sessions AS s + LEFT JOIN public.user_favorite_sessions AS fs ON fs.session_id = s.session_id + WHERE fs.user_id = %(userId)s""", + {"projectId": project_id, "userId": user_id} + ) + + extra_query = b"" + if include_viewed: + extra_query = cur.mogrify(""",\ + COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS viewed""", + {"projectId": project_id, "userId": user_id}) + + cur.execute(f"""\ + SELECT s.project_id, + s.session_id::text AS session_id, + s.user_uuid, + s.user_id, + s.user_agent, + s.user_os, + s.user_browser, + s.user_device, + s.user_country, + s.start_ts, + s.duration, + s.events_count, + s.pages_count, + s.errors_count, + TRUE AS favorite + {extra_query.decode('UTF-8')} + {query_part.decode('UTF-8')} + ORDER BY s.session_id + LIMIT 50;""") + + sessions = cur.fetchall() + return helper.list_to_camel_case(sessions) diff --git a/api/chalicelib/core/sessions_assignments.py b/api/chalicelib/core/sessions_assignments.py new file mode 100644 index 000000000..2b9c28d8f --- /dev/null +++ b/api/chalicelib/core/sessions_assignments.py @@ -0,0 +1,158 @@ +from chalicelib.utils.helper import environ as env +from chalicelib.utils import helper +from chalicelib.utils.TimeUTC import TimeUTC +from chalicelib.utils import pg_client +from chalicelib.core import integrations_manager, integration_base_issue +import json + + +def __get_saved_data(project_id, session_id, issue_id, tool): + with pg_client.PostgresClient() as cur: + query = cur.mogrify(f"""\ + SELECT * + FROM public.assigned_sessions + WHERE + session_id = %(session_id)s + AND issue_id = %(issue_id)s + AND provider = %(provider)s;\ + """, + {"session_id": session_id, "issue_id": issue_id, "provider": tool.lower()}) + cur.execute( + query + ) + return helper.dict_to_camel_case(cur.fetchone()) + + +def create_new_assignment(tenant_id, project_id, session_id, creator_id, assignee, description, title, issue_type, integration_project_id): + error, integration = integrations_manager.get_integration(tenant_id=tenant_id, user_id=creator_id) + if error is not None: + return error + + i = integration.get() + + if i is None: + return {"errors": [f"integration not found"]} + link = env["SITE_URL"] + f"/{project_id}/session/{session_id}" + description += f"\n> {link}" + try: + issue = integration.issue_handler.create_new_assignment(title=title, assignee=assignee, description=description, + issue_type=issue_type, + integration_project_id=integration_project_id) + except integration_base_issue.RequestException as e: + return integration_base_issue.proxy_issues_handler(e) + if issue is not None and "id" not in issue: + return {"errors": ["something went wrong while creating the issue"]} + with pg_client.PostgresClient() as cur: + query = cur.mogrify("""\ + INSERT INTO public.assigned_sessions(session_id, issue_id, created_by, provider,provider_data) + VALUES (%(session_id)s, %(issue_id)s, %(creator_id)s, %(provider)s,%(provider_data)s);\ + """, + {"session_id": session_id, "creator_id": creator_id, + "issue_id": issue["id"], "provider": integration.provider.lower(), + "provider_data": json.dumps({"integrationProjectId": integration_project_id})}) + cur.execute( + query + ) + issue["provider"] = integration.provider.lower() + return issue + + +def get_all(project_id, user_id): + available_integrations = integrations_manager.get_available_integrations(user_id=user_id) + no_integration = not any(available_integrations.values()) + if no_integration: + return [] + all_integrations = all(available_integrations.values()) + extra_query = ["sessions.project_id = %(project_id)s"] + if not all_integrations: + extra_query.append("provider IN %(providers)s") + with pg_client.PostgresClient() as cur: + query = cur.mogrify(f"""\ + SELECT assigned_sessions.* + FROM public.assigned_sessions + INNER JOIN public.sessions USING (session_id) + WHERE {" AND ".join(extra_query)};\ +""", + {"project_id": project_id, + "providers": tuple(d for d in available_integrations if available_integrations[d])}) + cur.execute( + query + ) + assignments = helper.list_to_camel_case(cur.fetchall()) + for a in assignments: + a["createdAt"] = TimeUTC.datetime_to_timestamp(a["createdAt"]) + return assignments + + +def get_by_session(tenant_id, user_id, project_id, session_id): + available_integrations = integrations_manager.get_available_integrations(user_id=user_id) + if not any(available_integrations.values()): + return [] + extra_query = ["session_id = %(session_id)s", "provider IN %(providers)s"] + with pg_client.PostgresClient() as cur: + query = cur.mogrify(f"""\ + SELECT * + FROM public.assigned_sessions + WHERE {" AND ".join(extra_query)};""", + {"session_id": session_id, + "providers": tuple([k for k in available_integrations if available_integrations[k]])}) + cur.execute( + query + ) + results = cur.fetchall() + issues = {} + for i in results: + if i["provider"] not in issues.keys(): + issues[i["provider"]] = [] + + issues[i["provider"]].append({"integrationProjectId": i["provider_data"]["integrationProjectId"], + "id": i["issue_id"]}) + results = [] + for tool in issues.keys(): + error, integration = integrations_manager.get_integration(tool=tool, tenant_id=tenant_id, user_id=user_id) + if error is not None: + return error + + i = integration.get() + if i is None: + print("integration not found") + continue + + r = integration.issue_handler.get_by_ids(saved_issues=issues[tool]) + print(r) + for i in r["issues"]: + i["provider"] = tool + results += r["issues"] + return results + + +def get(tenant_id, user_id, project_id, session_id, assignment_id): + error, integration = integrations_manager.get_integration(tenant_id=tenant_id, user_id=user_id) + if error is not None: + return error + l = __get_saved_data(project_id, session_id, assignment_id, tool=integration.provider) + if l is None: + return {"errors": ["issue not found"]} + i = integration.get() + if i is None: + return {"errors": ["integration not found"]} + r = integration.issue_handler.get(integration_project_id=l["providerData"]["integrationProjectId"], + assignment_id=assignment_id) + + r["provider"] = integration.provider.lower() + return r + + +def comment(tenant_id, user_id, project_id, session_id, assignment_id, message): + error, integration = integrations_manager.get_integration(tenant_id=tenant_id, user_id=user_id) + if error is not None: + return error + i = integration.get() + + if i is None: + return {"errors": [f"integration not found"]} + l = __get_saved_data(project_id, session_id, assignment_id, tool=integration.provider) + + return integration.issue_handler.comment(integration_project_id=l["providerData"]["integrationProjectId"], + assignment_id=assignment_id, + comment=message) diff --git a/api/chalicelib/core/sessions_favorite_viewed.py b/api/chalicelib/core/sessions_favorite_viewed.py new file mode 100644 index 000000000..1239156ab --- /dev/null +++ b/api/chalicelib/core/sessions_favorite_viewed.py @@ -0,0 +1,89 @@ +from chalicelib.utils import pg_client +from chalicelib.core import sessions + + +def add_favorite_session(project_id, user_id, session_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""\ + INSERT INTO public.user_favorite_sessions + (user_id, session_id) + VALUES + (%(userId)s,%(sessionId)s);""", + {"userId": user_id, "sessionId": session_id}) + ) + return sessions.get_by_id2_pg(project_id=project_id, session_id=session_id, user_id=user_id, full_data=False, + include_fav_viewed=True) + + +def remove_favorite_session(project_id, user_id, session_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""\ + DELETE FROM public.user_favorite_sessions + WHERE + user_id = %(userId)s + AND session_id = %(sessionId)s;""", + {"userId": user_id, "sessionId": session_id}) + ) + return sessions.get_by_id2_pg(project_id=project_id, session_id=session_id, user_id=user_id, full_data=False, + include_fav_viewed=True) + + +def add_viewed_session(project_id, user_id, session_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + INSERT INTO public.user_viewed_sessions + (user_id, session_id) + VALUES + (%(userId)s,%(sessionId)s);""", + {"userId": user_id, "sessionId": session_id}) + ) + + +def favorite_session(project_id, user_id, session_id): + if favorite_session_exists(user_id=user_id, session_id=session_id): + return remove_favorite_session(project_id=project_id, user_id=user_id, session_id=session_id) + + return add_favorite_session(project_id=project_id, user_id=user_id, session_id=session_id) + + +def view_session(project_id, user_id, session_id): + if viewed_session_exists(user_id=user_id, session_id=session_id): + return None + return add_viewed_session(project_id=project_id, user_id=user_id, session_id=session_id) + + +def favorite_session_exists(user_id, session_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT + session_id + FROM public.user_favorite_sessions + WHERE + user_id = %(userId)s + AND session_id = %(sessionId)s""", + {"userId": user_id, "sessionId": session_id}) + ) + r = cur.fetchone() + return r is not None + + +def viewed_session_exists(user_id, session_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT + session_id + FROM public.user_viewed_sessions + WHERE + user_id = %(userId)s + AND session_id = %(sessionId)s""", + {"userId": user_id, "sessionId": session_id}) + ) + r = cur.fetchone() + if r: + return True + return False diff --git a/api/chalicelib/core/sessions_metas.py b/api/chalicelib/core/sessions_metas.py new file mode 100644 index 000000000..a21b78783 --- /dev/null +++ b/api/chalicelib/core/sessions_metas.py @@ -0,0 +1,226 @@ +from chalicelib.utils import pg_client, helper +from chalicelib.utils.event_filter_definition import SupportedFilter + + +def get_key_values(project_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""\ + SELECT ARRAY_AGG(DISTINCT s.user_os + ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='web') AS {meta_type.USEROS}, + ARRAY_AGG(DISTINCT s.user_browser + ORDER BY s.user_browser) + FILTER ( WHERE s.user_browser IS NOT NULL AND s.platform='web') AS {meta_type.USERBROWSER}, + ARRAY_AGG(DISTINCT s.user_device + ORDER BY s.user_device) + FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='web') AS {meta_type.USERDEVICE}, + ARRAY_AGG(DISTINCT s.user_country + ORDER BY s.user_country) + FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='web')::text[] AS {meta_type.USERCOUNTRY}, + ARRAY_AGG(DISTINCT s.user_id + ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='web') AS {meta_type.USERID}, + ARRAY_AGG(DISTINCT s.user_anonymous_id + ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='web') AS {meta_type.USERANONYMOUSID}, + ARRAY_AGG(DISTINCT s.rev_id + ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='web') AS {meta_type.REVID}, + ARRAY_AGG(DISTINCT p.referrer + ORDER BY p.referrer) + FILTER ( WHERE p.referrer != '' ) AS {meta_type.REFERRER}, + + ARRAY_AGG(DISTINCT s.user_os + ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='ios' ) AS {meta_type.USEROS_IOS}, + ARRAY_AGG(DISTINCT s.user_device + ORDER BY s.user_device) + FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='ios') AS {meta_type.USERDEVICE}, + ARRAY_AGG(DISTINCT s.user_country + ORDER BY s.user_country) + FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='ios')::text[] AS {meta_type.USERCOUNTRY_IOS}, + ARRAY_AGG(DISTINCT s.user_id + ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='ios') AS {meta_type.USERID_IOS}, + ARRAY_AGG(DISTINCT s.user_anonymous_id + ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='ios') AS {meta_type.USERANONYMOUSID_IOS}, + ARRAY_AGG(DISTINCT s.rev_id + ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='ios') AS {meta_type.REVID_IOS} + FROM public.sessions AS s + LEFT JOIN events.pages AS p USING (session_id) + WHERE s.project_id = %(site_id)s;""", + {"site_id": project_id} + ) + ) + + row = cur.fetchone() + for k in row.keys(): + if row[k] is None: + row[k] = [] + elif len(row[k]) > 500: + row[k] = row[k][:500] + return helper.dict_to_CAPITAL_keys(row) + + +def get_top_key_values(project_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""\ + SELECT {",".join([f"ARRAY((SELECT value FROM public.autocomplete WHERE project_id = %(site_id)s AND type='{k}' GROUP BY value ORDER BY COUNT(*) DESC LIMIT %(limit)s)) AS {k}" for k in SUPPORTED_TYPES.keys()])};""", + {"site_id": project_id, "limit": 5} + ) + ) + + row = cur.fetchone() + return helper.dict_to_CAPITAL_keys(row) + + +def __generic_query(typename): + return f"""\ + SELECT value, type + FROM ((SELECT value, type + FROM public.autocomplete + WHERE + project_id = %(project_id)s + AND type ='{typename}' + AND value ILIKE %(svalue)s + ORDER BY value + LIMIT 5) + UNION + (SELECT value, type + FROM public.autocomplete + WHERE + project_id = %(project_id)s + AND type ='{typename}' + AND value ILIKE %(value)s + ORDER BY value + LIMIT 5)) AS met""" + + +def __generic_autocomplete(typename): + def f(project_id, text): + with pg_client.PostgresClient() as cur: + query = cur.mogrify(__generic_query(typename), + {"project_id": project_id, "value": helper.string_to_sql_like(text), + "svalue": helper.string_to_sql_like("^" + text)}) + + cur.execute(query) + rows = cur.fetchall() + return rows + + return f + + +class meta_type: + USEROS = "USEROS" + USERBROWSER = "USERBROWSER" + USERDEVICE = "USERDEVICE" + USERCOUNTRY = "USERCOUNTRY" + USERID = "USERID" + USERANONYMOUSID = "USERANONYMOUSID" + REFERRER = "REFERRER" + REVID = "REVID" + # IOS + USEROS_IOS = "USEROS_IOS" + USERDEVICE_IOS = "USERDEVICE_IOS" + USERCOUNTRY_IOS = "USERCOUNTRY_IOS" + USERID_IOS = "USERID_IOS" + USERANONYMOUSID_IOS = "USERANONYMOUSID_IOS" + REVID_IOS = "REVID_IOS" + + +SUPPORTED_TYPES = { + meta_type.USEROS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USEROS), + query=__generic_query(typename=meta_type.USEROS), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + meta_type.USERBROWSER: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERBROWSER), + query=__generic_query(typename=meta_type.USERBROWSER), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + meta_type.USERDEVICE: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERDEVICE), + query=__generic_query(typename=meta_type.USERDEVICE), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + meta_type.USERCOUNTRY: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERCOUNTRY), + query=__generic_query(typename=meta_type.USERCOUNTRY), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + meta_type.USERID: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERID), + query=__generic_query(typename=meta_type.USERID), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + meta_type.USERANONYMOUSID: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERANONYMOUSID), + query=__generic_query(typename=meta_type.USERANONYMOUSID), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + meta_type.REVID: SupportedFilter(get=__generic_autocomplete(typename=meta_type.REVID), + query=__generic_query(typename=meta_type.REVID), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + meta_type.REFERRER: SupportedFilter(get=__generic_autocomplete(typename=meta_type.REFERRER), + query=__generic_query(typename=meta_type.REFERRER), + value_limit=5, + starts_with="/", + starts_limit=5, + ignore_if_starts_with=[]), + # IOS + meta_type.USEROS_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USEROS_IOS), + query=__generic_query(typename=meta_type.USEROS_IOS), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + meta_type.USERDEVICE_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERDEVICE_IOS), + query=__generic_query(typename=meta_type.USERDEVICE_IOS), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + meta_type.USERCOUNTRY_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERCOUNTRY_IOS), + query=__generic_query(typename=meta_type.USERCOUNTRY_IOS), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + meta_type.USERID_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERID_IOS), + query=__generic_query(typename=meta_type.USERID_IOS), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + meta_type.USERANONYMOUSID_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERANONYMOUSID_IOS), + query=__generic_query(typename=meta_type.USERANONYMOUSID_IOS), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + meta_type.REVID_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.REVID_IOS), + query=__generic_query(typename=meta_type.REVID_IOS), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + +} + + +def search(text, meta_type, project_id): + rows = [] + if meta_type.upper() not in list(SUPPORTED_TYPES.keys()): + return {"errors": ["unsupported type"]} + rows += SUPPORTED_TYPES[meta_type.upper()].get(project_id=project_id, text=text) + if meta_type.upper() + "_IOS" in list(SUPPORTED_TYPES.keys()): + rows += SUPPORTED_TYPES[meta_type.upper() + "_IOS"].get(project_id=project_id, text=text) + return {"data": rows} diff --git a/api/chalicelib/core/sessions_mobs.py b/api/chalicelib/core/sessions_mobs.py new file mode 100644 index 000000000..7c3e7fd0a --- /dev/null +++ b/api/chalicelib/core/sessions_mobs.py @@ -0,0 +1,25 @@ +from chalicelib.utils.helper import environ + +import boto3 + + +def get_web(sessionId): + return boto3.client('s3', region_name=environ["sessions_region"]).generate_presigned_url( + 'get_object', + Params={ + 'Bucket': environ["sessions_bucket"], + 'Key': sessionId + }, + ExpiresIn=100000 + ) + + +def get_ios(sessionId): + return boto3.client('s3', region_name=environ["ios_region"]).generate_presigned_url( + 'get_object', + Params={ + 'Bucket': environ["ios_bucket"], + 'Key': sessionId + }, + ExpiresIn=100000 + ) diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py new file mode 100644 index 000000000..8bcda04a7 --- /dev/null +++ b/api/chalicelib/core/significance.py @@ -0,0 +1,627 @@ +__author__ = "AZNAUROV David" +__maintainer__ = "KRAIEM Taha Yassine" + +from chalicelib.core import events, sessions_metas, metadata, sessions +from chalicelib.utils import dev + +""" +todo: remove LIMIT from the query +""" + +from typing import List +import math +import warnings +from collections import defaultdict + +from psycopg2.extras import RealDictRow +from chalicelib.utils import pg_client, helper + +SIGNIFICANCE_THRSH = 0.4 + +T_VALUES = {1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228, + 11: 2.201, 12: 2.179, 13: 2.160, 14: 2.145, 15: 2.13, 16: 2.120, 17: 2.110, 18: 2.101, 19: 2.093, 20: 2.086, + 21: 2.080, 22: 2.074, 23: 2.069, 25: 2.064, 26: 2.060, 27: 2.056, 28: 2.052, 29: 2.045, 30: 2.042} + + +@dev.timed +def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: + """ + Add minimal timestamp + :param filter_d: dict contains events&filters&... + :return: + """ + stages = filter_d["events"] + filters = filter_d.get("filters", []) + filter_issues = filter_d.get("issueTypes") + if filter_issues is None or len(filter_issues) == 0: + filter_issues = [] + stage_constraints = ["main.timestamp <= %(endTimestamp)s"] + first_stage_extra_constraints = ["s.project_id=%(project_id)s", "s.start_ts >= %(startTimestamp)s", + "s.start_ts <= %(endTimestamp)s"] + extra_from = "" + n_stages_query = [] + values = {} + if len(filters) > 0: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + for i, f in enumerate(filters): + if not isinstance(f.get("value"), list): + if isinstance(f.get("value"), tuple): + f["value"] = list(f.get("value")) + else: + f["value"] = [f.get("value")] + if len(f["value"]) == 0 or f["value"][0] is None: + continue + filter_type = f["type"].upper() + values[f"f_value_{i}"] = sessions.__get_sql_value_multiple(f["value"]) + if filter_type == sessions_metas.meta_type.USERBROWSER: + op = sessions.__get_sql_operator_multiple(f["operator"]) + first_stage_extra_constraints.append(f's.user_browser {op} %({f"f_value_{i}"})s') + + elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: + op = sessions.__get_sql_operator_multiple(f["operator"]) + first_stage_extra_constraints.append(f's.user_os {op} %({f"f_value_{i}"})s') + + elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: + op = sessions.__get_sql_operator_multiple(f["operator"]) + first_stage_extra_constraints.append(f's.user_device {op} %({f"f_value_{i}"})s') + + elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: + op = sessions.__get_sql_operator_multiple(f["operator"]) + first_stage_extra_constraints.append(f's.user_country {op} %({f"f_value_{i}"})s') + elif filter_type == "duration".upper(): + if len(f["value"]) > 0 and f["value"][0] is not None: + first_stage_extra_constraints.append(f's.duration >= %({f"f_value_{i}"})s') + values[f"f_value_{i}"] = f["value"][0] + if len(f["value"]) > 1 and f["value"][1] is not None and f["value"][1] > 0: + first_stage_extra_constraints.append('s.duration <= %({f"f_value_{i}"})s') + values[f"f_value_{i}"] = f["value"][1] + elif filter_type == sessions_metas.meta_type.REFERRER: + # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" + extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" + op = sessions.__get_sql_operator_multiple(f["operator"]) + first_stage_extra_constraints.append(f"p.base_referrer {op} %(referrer)s") + elif filter_type == events.event_type.METADATA.ui_type: + op = sessions.__get_sql_operator(f["operator"]) + if f.get("key") in meta_keys.keys(): + first_stage_extra_constraints.append( + f's.{metadata.index_to_colname(meta_keys[f["key"]])} {op} %({f"f_value_{i}"})s') + values[f"f_value_{i}"] = helper.string_to_sql_like_with_op(f["value"][0], op) + elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + op = sessions.__get_sql_operator(f["operator"]) + first_stage_extra_constraints.append(f's.user_id {op} %({f"f_value_{i}"})s') + values[f"f_value_{i}"] = helper.string_to_sql_like_with_op(f["value"][0], op) + elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, + sessions_metas.meta_type.USERANONYMOUSID_IOS]: + op = sessions.__get_sql_operator(f["operator"]) + first_stage_extra_constraints.append(f's.user_anonymous_id {op} %({f"f_value_{i}"})s') + values[f"f_value_{i}"] = helper.string_to_sql_like_with_op(f["value"][0], op) + elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: + op = sessions.__get_sql_operator(f["operator"]) + first_stage_extra_constraints.append(f's.rev_id {op} %({f"f_value_{i}"})s') + values[f"f_value_{i}"] = helper.string_to_sql_like_with_op(f["value"][0], op) + + for i, s in enumerate(stages): + if i == 0: + extra_from = ["INNER JOIN public.sessions AS s USING (session_id)"] + else: + extra_from = [] + if s.get("operator") is None: + s["operator"] = "is" + op = sessions.__get_sql_operator(s["operator"]) + event_type = s["type"].upper() + next_label = s["value"] + if event_type == events.event_type.CLICK.ui_type: + next_table = events.event_type.CLICK.table + next_col_name = events.event_type.CLICK.column + elif event_type == events.event_type.INPUT.ui_type: + next_table = events.event_type.INPUT.table + next_col_name = events.event_type.INPUT.column + elif event_type == events.event_type.LOCATION.ui_type: + next_table = events.event_type.LOCATION.table + next_col_name = events.event_type.LOCATION.column + elif event_type == events.event_type.CUSTOM.ui_type: + next_table = events.event_type.CUSTOM.table + next_col_name = events.event_type.CUSTOM.column + # IOS -------------- + elif event_type == events.event_type.CLICK_IOS.ui_type: + next_table = events.event_type.CLICK_IOS.table + next_col_name = events.event_type.CLICK_IOS.column + elif event_type == events.event_type.INPUT_IOS.ui_type: + next_table = events.event_type.INPUT_IOS.table + next_col_name = events.event_type.INPUT_IOS.column + elif event_type == events.event_type.VIEW_IOS.ui_type: + next_table = events.event_type.VIEW_IOS.table + next_col_name = events.event_type.VIEW_IOS.column + elif event_type == events.event_type.CUSTOM_IOS.ui_type: + next_table = events.event_type.CUSTOM_IOS.table + next_col_name = events.event_type.CUSTOM_IOS.column + else: + print("=================UNDEFINED") + continue + + values[f"value{i + 1}"] = helper.string_to_sql_like_with_op(next_label, op) + if sessions.__is_negation_operator(op) and i > 0: + op = sessions.__reverse_sql_operator(op) + main_condition = "left_not.session_id ISNULL" + extra_from.append(f"""LEFT JOIN LATERAL (SELECT session_id + FROM {next_table} AS s_main + WHERE s_main.{next_col_name} {op} %(value{i + 1})s + AND s_main.timestamp >= T{i}.stage{i}_timestamp + AND s_main.session_id = T1.session_id) AS left_not ON (TRUE)""") + else: + main_condition = f"""main.{next_col_name} {op} %(value{i + 1})s""" + n_stages_query.append(f""" + (SELECT main.session_id, + {"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp, + '{event_type}' AS type, + '{s["operator"]}' AS operator + FROM {next_table} AS main {" ".join(extra_from)} + WHERE main.timestamp >= {f"T{i}.stage{i}_timestamp" if i > 0 else "%(startTimestamp)s"} + {f"AND main.session_id=T1.session_id" if i > 0 else ""} + AND {main_condition} + {(" AND " + " AND ".join(stage_constraints)) if len(stage_constraints) > 0 else ""} + {(" AND " + " AND ".join(first_stage_extra_constraints)) if len(first_stage_extra_constraints) > 0 and i == 0 else ""} + GROUP BY main.session_id) + AS T{i + 1} {"USING (session_id)" if i > 0 else ""} + """) + if len(n_stages_query) == 0: + return [] + n_stages_query = " LEFT JOIN LATERAL ".join(n_stages_query) + n_stages_query += ") AS stages_t" + + n_stages_query = f""" + SELECT stages_and_issues_t.*,sessions.session_id, sessions.user_uuid FROM ( + SELECT * FROM ( + SELECT * FROM + {n_stages_query} + LEFT JOIN LATERAL + ( + SELECT * FROM + (SELECT ISE.session_id, + ISS.type as issue_type, + ISE.timestamp AS issue_timestamp, + ISS.context_string as issue_context, + ISS.issue_id as issue_id + FROM events_common.issues AS ISE INNER JOIN issues AS ISS USING (issue_id) + WHERE ISE.timestamp >= stages_t.stage1_timestamp + AND ISE.timestamp <= stages_t.stage{len(stages)}_timestamp + AND ISS.project_id=%(project_id)s + {"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""}) AS base_t + ) AS issues_t + USING (session_id)) AS stages_and_issues_t + inner join sessions USING(session_id); + """ + + # LIMIT 10000 + params = {"project_id": project_id, "startTimestamp": filter_d["startDate"], "endTimestamp": filter_d["endDate"], + "issueTypes": tuple(filter_issues), **values} + with pg_client.PostgresClient() as cur: + # print("---------------------------------------------------") + # print(cur.mogrify(n_stages_query, params)) + # print("---------------------------------------------------") + cur.execute(cur.mogrify(n_stages_query, params)) + rows = cur.fetchall() + return rows + + +def pearson_corr(x: list, y: list): + n = len(x) + if n != len(y): + raise ValueError(f'x and y must have the same length. Got {len(x)} and {len(y)} instead') + + if n < 2: + warnings.warn(f'x and y must have length at least 2. Got {n} instead') + return None, None, False + + # If an input is constant, the correlation coefficient is not defined. + if all(t == x[0] for t in x) or all(t == y[0] for t in y): + warnings.warn("An input array is constant; the correlation coefficent is not defined.") + return None, None, False + + if n == 2: + return math.copysign(1, x[1] - x[0]) * math.copysign(1, y[1] - y[0]), 1.0 + + xmean = sum(x) / len(x) + ymean = sum(y) / len(y) + + xm = [el - xmean for el in x] + ym = [el - ymean for el in y] + + normxm = math.sqrt((sum([xm[i] * xm[i] for i in range(len(xm))]))) + normym = math.sqrt((sum([ym[i] * ym[i] for i in range(len(ym))]))) + + threshold = 1e-8 + if normxm < threshold * abs(xmean) or normym < threshold * abs(ymean): + # If all the values in x (likewise y) are very close to the mean, + # the loss of precision that occurs in the subtraction xm = x - xmean + # might result in large errors in r. + warnings.warn("An input array is constant; the correlation coefficent is not defined.") + + r = sum( + i[0] * i[1] for i in zip([xm[i] / normxm for i in range(len(xm))], [ym[i] / normym for i in range(len(ym))])) + + # Presumably, if abs(r) > 1, then it is only some small artifact of floating point arithmetic. + # However, if r < 0, we don't care, as our problem is to find only positive correlations + r = max(min(r, 1.0), 0.0) + + # approximated confidence + if n < 31: + t_c = T_VALUES[n] + elif n < 50: + t_c = 2.02 + else: + t_c = 2 + if r >= 0.999: + confidence = 1 + else: + confidence = r * math.sqrt(n - 2) / math.sqrt(1 - r ** 2) + + if confidence > SIGNIFICANCE_THRSH: + return r, confidence, True + else: + return r, confidence, False + + +@dev.timed +def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_with_context, first_stage, last_stage): + """ + Returns two lists with binary values 0/1: + + transitions ::: if transited from the first stage to the last - 1 + else - 0 + errors ::: a dictionary where the keys are all unique issues (currently context-wise) + the values are lists + if an issue happened between the first stage to the last - 1 + else - 0 + + For a small task of calculating a total drop due to issues, + we need to disregard the issue type when creating the `errors`-like array. + The `all_errors` array can be obtained by logical OR statement applied to all errors by issue + The `transitions` array stays the same + """ + transitions = [] + n_sess_affected = 0 + errors = {} + for issue in all_issues_with_context: + split = issue.split('__^__') + errors[issue] = { + "errors": [], + "issue_type": split[0], + "context": split[1]} + + for row in rows: + t = 0 + first_ts = row[f'stage{first_stage}_timestamp'] + last_ts = row[f'stage{last_stage}_timestamp'] + if first_ts is None: + continue + elif first_ts is not None and last_ts is not None: + t = 1 + transitions.append(t) + + ic_present = False + for issue_type_with_context in errors: + ic = 0 + issue_type = errors[issue_type_with_context]["issue_type"] + context = errors[issue_type_with_context]["context"] + if row['issue_type'] is not None: + if last_ts is None or (first_ts < row['issue_timestamp'] < last_ts): + context_in_row = row['issue_context'] if row['issue_context'] is not None else '' + if issue_type == row['issue_type'] and context == context_in_row: + ic = 1 + ic_present = True + errors[issue_type_with_context]["errors"].append(ic) + + if ic_present and t: + n_sess_affected += 1 + + # def tuple_or(t: tuple): + # x = 0 + # for el in t: + # x |= el + # return x + def tuple_or(t: tuple): + for el in t: + if el > 0: + return 1 + return 0 + + errors = {key: errors[key]["errors"] for key in errors} + all_errors = [tuple_or(t) for t in zip(*errors.values())] + + return transitions, errors, all_errors, n_sess_affected + + +@dev.timed +def get_affected_users_for_all_issues(rows, first_stage, last_stage): + """ + + :param rows: + :param first_stage: + :param last_stage: + :return: + """ + affected_users = defaultdict(lambda: set()) + affected_sessions = defaultdict(lambda: set()) + contexts = defaultdict(lambda: None) + n_affected_users_dict = defaultdict(lambda: None) + n_affected_sessions_dict = defaultdict(lambda: None) + all_issues_with_context = set() + n_issues_dict = defaultdict(lambda: 0) + issues_by_session = defaultdict(lambda: 0) + + for row in rows: + + # check that the session has reached the first stage of subfunnel: + if row[f'stage{first_stage}_timestamp'] is None: + continue + + iss = row['issue_type'] + iss_ts = row['issue_timestamp'] + + # check that the issue exists and belongs to subfunnel: + if iss is not None and (row[f'stage{last_stage}_timestamp'] is None or + (row[f'stage{first_stage}_timestamp'] < iss_ts < row[f'stage{last_stage}_timestamp'])): + context_string = row['issue_context'] if row['issue_context'] is not None else '' + issue_with_context = iss + '__^__' + context_string + contexts[issue_with_context] = {"context": context_string, "id": row["issue_id"]} + all_issues_with_context.add(issue_with_context) + n_issues_dict[issue_with_context] += 1 + if row['user_uuid'] is not None: + affected_users[issue_with_context].add(row['user_uuid']) + + affected_sessions[issue_with_context].add(row['session_id']) + issues_by_session[row[f'session_id']] += 1 + + if len(affected_users) > 0: + n_affected_users_dict.update({ + iss: len(affected_users[iss]) for iss in affected_users + }) + if len(affected_sessions) > 0: + n_affected_sessions_dict.update({ + iss: len(affected_sessions[iss]) for iss in affected_sessions + }) + return all_issues_with_context, n_issues_dict, n_affected_users_dict, n_affected_sessions_dict, contexts + + +@dev.timed +def count_sessions(rows, n_stages): + session_counts = {i: set() for i in range(1, n_stages + 1)} + for ind, row in enumerate(rows): + for i in range(1, n_stages + 1): + if row[f"stage{i}_timestamp"] is not None: + session_counts[i].add(row[f"session_id"]) + session_counts = {i: len(session_counts[i]) for i in session_counts} + return session_counts + + +def count_users(rows, n_stages): + users_in_stages = defaultdict(lambda: set()) + + for ind, row in enumerate(rows): + for i in range(1, n_stages + 1): + if row[f"stage{i}_timestamp"] is not None: + users_in_stages[i].add(row["user_uuid"]) + + users_count = {i: len(users_in_stages[i]) for i in range(1, n_stages + 1)} + + return users_count + + +def get_stages(stages, rows): + n_stages = len(stages) + session_counts = count_sessions(rows, n_stages) + users_counts = count_users(rows, n_stages) + + stages_list = [] + for i, stage in enumerate(stages): + + drop = None + if i != 0: + if session_counts[i] == 0: + drop = 0 + elif session_counts[i] > 0: + drop = int(100 * (session_counts[i] - session_counts[i + 1]) / session_counts[i]) + + stages_list.append( + {"value": stage["value"], + "type": stage["type"], + "operator": stage["operator"], + "sessionsCount": session_counts[i + 1], + "drop_pct": drop, + "usersCount": users_counts[i + 1], + "dropDueToIssues": 0 + } + ) + return stages_list + + +@dev.timed +def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False): + """ + + :param stages: + :param rows: + :param first_stage: If it's a part of the initial funnel, provide a number of the first stage (starting from 1) + :param last_stage: If it's a part of the initial funnel, provide a number of the last stage (starting from 1) + :return: + """ + + n_stages = len(stages) + + if first_stage is None: + first_stage = 1 + if last_stage is None: + last_stage = n_stages + if last_stage > n_stages: + print("The number of the last stage provided is greater than the number of stages. Using n_stages instead") + last_stage = n_stages + + n_critical_issues = 0 + issues_dict = dict({"significant": [], + "insignificant": []}) + session_counts = count_sessions(rows, n_stages) + drop = session_counts[first_stage] - session_counts[last_stage] + + all_issues_with_context, n_issues_dict, affected_users_dict, affected_sessions, contexts = get_affected_users_for_all_issues( + rows, first_stage, last_stage) + transitions, errors, all_errors, n_sess_affected = get_transitions_and_issues_of_each_type(rows, + all_issues_with_context, + first_stage, last_stage) + + print("len(transitions) =", len(transitions)) + + if any(all_errors): + total_drop_corr, conf, is_sign = pearson_corr(transitions, all_errors) + if total_drop_corr is not None and drop is not None: + total_drop_due_to_issues = int(total_drop_corr * n_sess_affected) + else: + total_drop_due_to_issues = 0 + else: + total_drop_due_to_issues = 0 + + if drop_only: + return total_drop_due_to_issues + for issue in all_issues_with_context: + + if not any(errors[issue]): + continue + r, confidence, is_sign = pearson_corr(transitions, errors[issue]) + + if r is not None and drop is not None and is_sign: + lost_conversions = int(r * affected_sessions[issue]) + else: + lost_conversions = None + if r is None: + r = 0 + split = issue.split('__^__') + issues_dict['significant' if is_sign else 'insignificant'].append({ + "type": split[0], + "title": get_issue_title(split[0]), + "affected_sessions": affected_sessions[issue], + "unaffected_sessions": session_counts[1] - affected_sessions[issue], + "lost_conversions": lost_conversions, + "affected_users": affected_users_dict[issue], + "conversion_impact": round(r * 100), + "context_string": contexts[issue]["context"], + "issue_id": contexts[issue]["id"] + }) + + if is_sign: + n_critical_issues += n_issues_dict[issue] + + return n_critical_issues, issues_dict, total_drop_due_to_issues + + +@dev.timed +def get_top_insights(filter_d, project_id): + output = [] + stages = filter_d["events"] + # TODO: handle 1 stage alone + if len(stages) == 0: + print("no stages found") + return output, 0 + elif len(stages) == 1: + # TODO: count sessions, and users for single stage + output = [{ + "type": stages[0]["type"], + "value": stages[0]["value"], + "dropPercentage": None, + "operator": stages[0]["operator"], + "sessionsCount": 0, + "dropPct": 0, + "usersCount": 0, + "dropDueToIssues": 0 + + }] + counts = sessions.search2_pg(data=filter_d, project_id=project_id, user_id=None, count_only=True) + output[0]["sessionsCount"] = counts["countSessions"] + output[0]["usersCount"] = counts["countUsers"] + return output, 0 + # The result of the multi-stage query + rows = get_stages_and_events(filter_d=filter_d, project_id=project_id) + if len(rows) == 0: + return get_stages(stages, []), 0 + # Obtain the first part of the output + stages_list = get_stages(stages, rows) + # Obtain the second part of the output + total_drop_due_to_issues = get_issues(stages, rows, first_stage=filter_d.get("firstStage"), + last_stage=filter_d.get("lastStage"), drop_only=True) + return stages_list, total_drop_due_to_issues + + +@dev.timed +def get_issues_list(filter_d, project_id, first_stage=None, last_stage=None): + output = dict({'critical_issues_count': 0}) + stages = filter_d["events"] + # The result of the multi-stage query + rows = get_stages_and_events(filter_d=filter_d, project_id=project_id) + # print(json.dumps(rows[0],indent=4)) + # return + if len(rows) == 0: + return output + # Obtain the second part of the output + n_critical_issues, issues_dict, total_drop_due_to_issues = get_issues(stages, rows, first_stage=first_stage, + last_stage=last_stage) + output['total_drop_due_to_issues'] = total_drop_due_to_issues + # output['critical_issues_count'] = n_critical_issues + output = {**output, **issues_dict} + return output + + +def get_overview(filter_d, project_id, first_stage=None, last_stage=None): + output = dict() + stages = filter_d["events"] + # TODO: handle 1 stage alone + if len(stages) == 0: + return {"stages": [], + "criticalIssuesCount": 0} + elif len(stages) == 1: + # TODO: count sessions, and users for single stage + output["stages"] = [{ + "type": stages[0]["type"], + "value": stages[0]["value"], + "sessionsCount": None, + "dropPercentage": None, + "usersCount": None + }] + return output + # The result of the multi-stage query + rows = get_stages_and_events(filter_d=filter_d, project_id=project_id) + if len(rows) == 0: + return output + # Obtain the first part of the output + stages_list = get_stages(stages, rows) + + # Obtain the second part of the output + n_critical_issues, issues_dict, total_drop_due_to_issues = get_issues(stages, rows, first_stage=first_stage, + last_stage=last_stage) + + output['stages'] = stages_list + output['criticalIssuesCount'] = n_critical_issues + return output + + +def get_issue_title(issue_type): + return {'click_rage': "Click Rage", + 'dead_click': "Dead Click", + 'excessive_scrolling': "Excessive Scrolling", + 'bad_request': "Bad Request", + 'missing_resource': "Missing Image", + 'memory': "High Memory Usage", + 'cpu': "High CPU", + 'slow_resource': "Slow Resource", + 'slow_page_load': "Slow Page Performance", + 'crash': "Crash", + 'ml_cpu': "High CPU", + 'ml_memory': "High Memory Usage", + 'ml_dead_click': "Dead Click", + 'ml_click_rage': "Click Rage", + 'ml_mouse_thrashing': "Mouse Thrashing", + 'ml_excessive_scrolling': "Excessive Scrolling", + 'ml_slow_resources': "Slow Resource", + 'custom': "Custom Event", + 'js_exception': "Error", + 'custom_event_error': "Custom Error", + 'js_error': "Error"}.get(issue_type, issue_type) diff --git a/api/chalicelib/core/signup.py b/api/chalicelib/core/signup.py new file mode 100644 index 000000000..656d231ef --- /dev/null +++ b/api/chalicelib/core/signup.py @@ -0,0 +1,171 @@ +from chalicelib.utils import helper +from chalicelib.utils import pg_client +from chalicelib.core import users, telemetry +from chalicelib.utils import captcha +import json +from chalicelib.utils.TimeUTC import TimeUTC + + +def get_signed_ups(): + with pg_client.PostgresClient() as cur: + cur.execute("SELECT tenant_id, name FROM public.tenants;") + rows = cur.fetchall() + return helper.list_to_camel_case(rows) + + +def create_step1(data): + print(f"===================== SIGNUP STEP 1 AT {TimeUTC.to_human_readable(TimeUTC.now())} UTC") + errors = [] + + email = data.get("email") + print(f"=====================> {email}") + password = data.get("password") + + print("Verifying email validity") + email_exists = False + if email is None or len(email) < 5 or not helper.is_valid_email(email): + errors.append("Invalid email address.") + else: + print("Verifying email existance") + if users.email_exists(email): + # errors.append("Email address already in use.") + email_exists = True + if users.get_deleted_user_by_email(email) is not None: + # errors.append("Email address previously deleted.") + email_exists = True + + print("Verifying captcha") + if helper.allow_captcha() and not captcha.is_valid(data["g-recaptcha-response"]): + errors.append("Invalid captcha.") + + print("Verifying password validity") + if len(data["password"]) < 6: + errors.append("Password is too short, it must be at least 6 characters long.") + + print("Verifying fullname validity") + fullname = data.get("fullname") + if fullname is None or len(fullname) < 1 or not helper.is_alphabet_space_dash(fullname): + errors.append("Invalid full name.") + + print("Verifying company's name validity") + company_name = data.get("organizationName") + if company_name is None or len(company_name) < 1 or not helper.is_alphanumeric_space(company_name): + errors.append("invalid organization's name") + + print("Verifying project's name validity") + project_name = data.get("projectName") + if project_name is None or len(project_name) < 1: + project_name = "my first project" + if len(get_signed_ups()) > 0 and data.get("tenantId") is None: + errors.append("Tenant already exists, please select it from dropdown") + if len(errors) > 0: + print("==> error") + print(errors) + return {"errors": errors} + print("No errors detected") + params = { + "email": email, "password": password, + "fullname": fullname, + "projectName": project_name, + "data": json.dumps({"lastAnnouncementView": TimeUTC.now()}), + "organizationName": company_name, + "versionNumber": "0.0.0" + } + if data.get("tenantId") is not None: + update_user = """ + u AS ( + UPDATE public.users + SET name = %(fullname)s, deleted_at=NULL + WHERE email=%(email)s + RETURNING user_id,email, role, name + ) + UPDATE public.basic_authentication + SET password= crypt(%(password)s, gen_salt('bf', 12)) + WHERE user_id = (SELECT user_id FROM u)""" + insert_user = """ + a AS ( + UPDATE public.users + SET role='admin' + WHERE role ='owner' + ), + u AS ( + INSERT INTO public.users (email, role, name, data) + VALUES (%(email)s, 'owner', %(fullname)s,%(data)s) + RETURNING user_id,email,role,name + ) + INSERT INTO public.basic_authentication (user_id, password, generated_password) + VALUES ((SELECT user_id FROM u), crypt(%(password)s, gen_salt('bf', 12)), FALSE)""" + query = f"""\ + WITH t AS ( + UPDATE public.tenants + SET name = %(organizationName)s, + version_number = %(versionNumber)s + RETURNING api_key + ), + {update_user if email_exists else insert_user} + RETURNING (SELECT api_key FROM t) AS api_key,(SELECT project_id FROM projects LIMIT 1) AS project_id;""" + else: + query = f"""\ + WITH t AS ( + INSERT INTO public.tenants (name, version_number, edition) + VALUES (%(organizationName)s, %(versionNumber)s, 'fos') + RETURNING api_key + ), + u AS ( + INSERT INTO public.users (email, role, name, data) + VALUES (%(email)s, 'owner', %(fullname)s,%(data)s) + RETURNING user_id,email,role,name + ), + au AS (INSERT + INTO public.basic_authentication (user_id, password, generated_password) + VALUES ((SELECT user_id FROM u), crypt(%(password)s, gen_salt('bf', 12)), FALSE) + ) + INSERT INTO public.projects (name, active) + VALUES (%(projectName)s, TRUE) + RETURNING project_id, (SELECT api_key FROM t) AS api_key;""" + + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify(query, params)) + cur = cur.fetchone() + project_id = cur["project_id"] + api_key = cur["api_key"] + telemetry.new_client() + created_at = TimeUTC.now() + r = users.authenticate(email, password) + r["banner"] = False + r["limits"] = { + "teamMember": {"limit": 99, "remaining": 98, "count": 1}, + "projects": {"limit": 99, "remaining": 98, "count": 1}, + "metadata": [{ + "projectId": project_id, + "name": project_name, + "limit": 10, + "remaining": 10, + "count": 0 + }] + } + c = { + "tenantId": 1, + "name": company_name, + "apiKey": api_key, + "remainingTrial": 14, + "trialEnded": False, + "billingPeriodStartDate": created_at, + "hasActivePlan": True, + "projects": [ + { + "projectId": project_id, + "name": project_name, + "recorded": False, + "stackIntegrations": False, + "status": "red" + } + ] + } + return { + 'jwt': r.pop('jwt'), + 'data': { + "user": r, + "client": c, + } + } diff --git a/api/chalicelib/core/slack.py b/api/chalicelib/core/slack.py new file mode 100644 index 000000000..411be0567 --- /dev/null +++ b/api/chalicelib/core/slack.py @@ -0,0 +1,32 @@ +from datetime import datetime +from chalicelib.utils.helper import environ + +from chalicelib.core.collaboration_slack import Slack + + +def send(notification, destination): + if notification is None: + return + return Slack.send_text(tenant_id=notification["tenantId"], + webhook_id=destination, + text=notification["description"] \ + + f"\n<{environ['SITE_URL']}{notification['buttonUrl']}|{notification['buttonText']}>", + title=notification["title"], + title_link=notification["buttonUrl"], ) + + +def send_batch(notifications_list): + if notifications_list is None or len(notifications_list) == 0: + return + webhookId_map = {} + for n in notifications_list: + if n.get("destination") not in webhookId_map: + webhookId_map[n.get("destination")] = {"tenantId": n["notification"]["tenantId"], "batch": []} + webhookId_map[n.get("destination")]["batch"].append({"text": n["notification"]["description"] \ + + f"\n<{environ['SITE_URL']}{n['notification']['buttonUrl']}|{n['notification']['buttonText']}>", + "title": n["notification"]["title"], + "title_link": n["notification"]["buttonUrl"], + "ts": datetime.now().timestamp()}) + for batch in webhookId_map.keys(): + Slack.send_batch(tenant_id=webhookId_map[batch]["tenantId"], webhook_id=batch, + attachments=webhookId_map[batch]["batch"]) diff --git a/api/chalicelib/core/socket_ios.py b/api/chalicelib/core/socket_ios.py new file mode 100644 index 000000000..d925797fe --- /dev/null +++ b/api/chalicelib/core/socket_ios.py @@ -0,0 +1,22 @@ +import requests +from chalicelib.utils.helper import environ +from chalicelib.core import projects + + +def start_replay(project_id, session_id, device, os_version, mob_url): + r = requests.post(environ["IOS_MIDDLEWARE"] + "/replay", json={ + "projectId": project_id, + "projectKey": projects.get_project_key(project_id), + "sessionId": session_id, + "device": device, + "osVersion": os_version, + "mobUrl": mob_url + }) + if r.status_code != 200: + print("failed replay middleware") + print("status code: %s" % r.status_code) + print(r.text) + return r.text + result = r.json() + result["url"] = environ["IOS_MIDDLEWARE"] + return result diff --git a/api/chalicelib/core/sourcemaps.py b/api/chalicelib/core/sourcemaps.py new file mode 100644 index 000000000..c198b859b --- /dev/null +++ b/api/chalicelib/core/sourcemaps.py @@ -0,0 +1,157 @@ +from chalicelib.utils.helper import environ +from chalicelib.utils import helper + +from chalicelib.utils import s3 +import hashlib +from urllib.parse import urlparse + +from chalicelib.core import sourcemaps_parser + + +def __get_key(project_id, url): + u = urlparse(url) + new_url = u.scheme + "://" + u.netloc + u.path + return f"{project_id}/{hashlib.md5(new_url.encode()).hexdigest()}" + + +def presign_share_urls(project_id, urls): + results = [] + for u in urls: + results.append(s3.get_presigned_url_for_sharing(bucket=environ['sourcemaps_bucket'], expires_in=120, + key=__get_key(project_id, u), + check_exists=True)) + return results + + +def presign_upload_urls(project_id, urls): + results = [] + for u in urls: + results.append(s3.get_presigned_url_for_upload(bucket=environ['sourcemaps_bucket'], + expires_in=1800, + key=__get_key(project_id, u))) + return results + + +def __format_frame_old(f): + if f.get("context") is None: + f["context"] = [] + else: + f["context"] = [[f["line"], f["context"]]] + url = f.pop("url") + f["absPath"] = url + f["filename"] = urlparse(url).path + f["lineNo"] = f.pop("line") + f["colNo"] = f.pop("column") + f["function"] = f.pop("func") + return f + + +def __frame_is_valid(f): + return "columnNumber" in f and \ + "lineNumber" in f and \ + "fileName" in f + + +def __format_frame(f): + f["context"] = [] # no context by default + if "source" in f: f.pop("source") + url = f.pop("fileName") + f["absPath"] = url + f["filename"] = urlparse(url).path + f["lineNo"] = f.pop("lineNumber") + f["colNo"] = f.pop("columnNumber") + f["function"] = f.pop("functionName") if "functionName" in f else None + return f + + +def format_payload(p, truncate_to_first=False): + if type(p) is list: + return [__format_frame(f) for f in (p[:1] if truncate_to_first else p) if __frame_is_valid(f)] + if type(p) is dict: + stack = p.get("stack", []) + return [__format_frame_old(f) for f in (stack[:1] if truncate_to_first else stack)] + return [] + + +def get_traces_group(project_id, payload): + frames = format_payload(payload) + + results = [{}] * len(frames) + payloads = {} + all_exists = True + for i, u in enumerate(frames): + key = __get_key(project_id, u["absPath"]) # use filename instead? + if key not in payloads: + file_exists = s3.exists(environ['sourcemaps_bucket'], key) + all_exists = all_exists and file_exists + if not file_exists: + print(f"{u['absPath']} sourcemap (key '{key}') doesn't exist in S3") + payloads[key] = None + else: + payloads[key] = [] + results[i] = dict(u) + results[i]["frame"] = dict(u) + if payloads[key] is not None: + payloads[key].append({"resultIndex": i, + "position": {"line": u["lineNo"], "column": u["colNo"]}, + "frame": dict(u)}) + for key in payloads.keys(): + if payloads[key] is None: + continue + key_results = sourcemaps_parser.get_original_trace(key=key, positions=[o["position"] for o in payloads[key]]) + for i, r in enumerate(key_results): + res_index = payloads[key][i]["resultIndex"] + # function name search by frontend lib is better than sourcemaps' one in most cases + if results[res_index].get("function") is not None: + r["function"] = results[res_index]["function"] + r["frame"] = payloads[key][i]["frame"] + results[res_index] = r + return fetch_missed_contexts(results), all_exists + + +def get_js_cache_path(fullURL): + p = urlparse(fullURL) + return p.scheme + '/' + p.netloc + p.path # TODO (Also in go assets library): What if URL with query? (like versions) + + +MAX_COLUMN_OFFSET = 60 + + +def fetch_missed_contexts(frames): + source_cache = {} + for i in range(len(frames)): + if len(frames[i]["context"]) != 0: + continue + if frames[i]["frame"]["absPath"] in source_cache: + file = source_cache[frames[i]["frame"]["absPath"]] + else: + file = s3.get_file(environ['js_cache_bucket'], get_js_cache_path(frames[i]["frame"]["absPath"])) + if file is None: + print( + f"File {get_js_cache_path(frames[i]['frame']['absPath'])} not found in {environ['js_cache_bucket']}") + source_cache[frames[i]["frame"]["absPath"]] = file + if file is None: + continue + lines = file.split("\n") + + if frames[i]["lineNo"] is None: + print("no original-source found for frame in sourcemap results") + frames[i] = frames[i]["frame"] + frames[i]["originalMapping"] = False + + l = frames[i]["lineNo"] - 1 # starts from 1 + c = frames[i]["colNo"] - 1 # starts from 1 + if len(lines) == 1: + print(f"minified asset") + l = frames[i]["frame"]["lineNo"] - 1 # starts from 1 + c = frames[i]["frame"]["colNo"] - 1 # starts from 1 + elif l >= len(lines): + print(f"line number {l} greater than file length {len(lines)}") + continue + + line = lines[l] + offset = c - MAX_COLUMN_OFFSET + if offset < 0: # if the line is shirt + offset = 0 + frames[i]["context"].append([frames[i]["lineNo"], line[offset: c + MAX_COLUMN_OFFSET + 1]]) + return frames diff --git a/api/chalicelib/core/sourcemaps_parser.py b/api/chalicelib/core/sourcemaps_parser.py new file mode 100644 index 000000000..cb0463d55 --- /dev/null +++ b/api/chalicelib/core/sourcemaps_parser.py @@ -0,0 +1,22 @@ +import requests + +from chalicelib.utils.helper import environ + + +def get_original_trace(key, positions): + payload = { + "key": key, + "positions": positions, + "padding": 5, + "bucket": environ['sourcemaps_bucket'], + "bucket_config": { + "aws_access_key_id": environ["sourcemaps_bucket_key"], + "aws_secret_access_key": environ["sourcemaps_bucket_secret"], + "aws_region": environ["sourcemaps_bucket_region"] + } + } + r = requests.post(environ["sourcemaps"], json=payload) + if r.status_code != 200: + return {} + + return r.json() diff --git a/api/chalicelib/core/telemetry.py b/api/chalicelib/core/telemetry.py new file mode 100644 index 000000000..362550553 --- /dev/null +++ b/api/chalicelib/core/telemetry.py @@ -0,0 +1,43 @@ +from chalicelib.utils import pg_client +import requests + + +def process_data(data, edition='fos'): + return { + 'edition': edition, + 'tracking': data["opt_out"], + 'version': data["version_number"], + 'user_id': data["user_id"], + 'owner_email': None if data["opt_out"] else data["email"], + 'organization_name': None if data["opt_out"] else data["name"], + 'users_count': data["t_users"], + 'projects_count': data["t_projects"], + 'sessions_count': data["t_sessions"], + 'integrations_count': data["t_integrations"] + } + + +def compute(): + with pg_client.PostgresClient() as cur: + cur.execute( + f"""UPDATE public.tenants + SET t_integrations = COALESCE((SELECT COUNT(DISTINCT provider) FROM public.integrations) + + (SELECT COUNT(*) FROM public.webhooks WHERE type = 'slack') + + (SELECT COUNT(*) FROM public.jira_cloud), 0), + t_projects=COALESCE((SELECT COUNT(*) FROM public.projects WHERE deleted_at ISNULL), 0), + t_sessions=COALESCE((SELECT COUNT(*) FROM public.sessions), 0), + t_users=COALESCE((SELECT COUNT(*) FROM public.users WHERE deleted_at ISNULL), 0) + RETURNING *,(SELECT email FROM public.users WHERE role='owner' LIMIT 1);""" + ) + data = cur.fetchone() + requests.post('https://parrot.asayer.io/os/telemetry', json=process_data(data)) + + +def new_client(): + with pg_client.PostgresClient() as cur: + cur.execute( + f"""SELECT *, + (SELECT email FROM public.users WHERE role='owner' LIMIT 1) AS email + FROM public.tenants;""") + data = cur.fetchone() + requests.post('https://parrot.asayer.io/os/signup', json=process_data(data)) diff --git a/api/chalicelib/core/tenants.py b/api/chalicelib/core/tenants.py new file mode 100644 index 000000000..f047dcffa --- /dev/null +++ b/api/chalicelib/core/tenants.py @@ -0,0 +1,83 @@ +from chalicelib.utils import pg_client +from chalicelib.utils import helper +from chalicelib.core import users + + +def get_by_tenant_id(tenant_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""SELECT + tenant_id, + name, + api_key + created_at, + edition, + version_number, + opt_out + FROM public.tenants + LIMIT 1;""", + {"tenantId": tenant_id}) + ) + return helper.dict_to_camel_case(cur.fetchone()) + + +def get_by_api_key(api_key): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""SELECT + 1 AS tenant_id, + name, + created_at + FROM public.tenants + WHERE api_key = %(api_key)s + LIMIT 1;""", + {"api_key": api_key}) + ) + return helper.dict_to_camel_case(cur.fetchone()) + + +def generate_new_api_key(tenant_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""UPDATE public.tenants + SET api_key=generate_api_key(20) + RETURNING api_key;""", + {"tenant_id": tenant_id}) + ) + return helper.dict_to_camel_case(cur.fetchone()) + + +def edit_client(tenant_id, changes): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""\ + UPDATE public.tenants + SET {", ".join([f"{helper.key_to_snake_case(k)} = %({k})s" for k in changes.keys()])} + RETURNING name, opt_out;""", + {"tenantId": tenant_id, **changes}) + ) + return helper.dict_to_camel_case(cur.fetchone()) + + +def update(tenant_id, user_id, data): + admin = users.get(user_id=user_id, tenant_id=tenant_id) + + if not admin["admin"] and not admin["superAdmin"]: + return {"error": "unauthorized"} + if "name" not in data and "optOut" not in data: + return {"errors": ["please provide 'name' of 'optOut' attribute for update"]} + changes = {} + if "name" in data: + changes["name"] = data["name"] + if "optOut" in data: + changes["optOut"] = data["optOut"] + return edit_client(tenant_id=tenant_id, changes=changes) + + +def get_tenants(): + with pg_client.PostgresClient() as cur: + cur.execute(f"SELECT name FROM public.tenants") + return helper.list_to_camel_case(cur.fetchall()) diff --git a/api/chalicelib/core/users.py b/api/chalicelib/core/users.py new file mode 100644 index 000000000..819cb4c12 --- /dev/null +++ b/api/chalicelib/core/users.py @@ -0,0 +1,463 @@ +import json + +from chalicelib.core import authorizers + +from chalicelib.utils import helper +from chalicelib.utils import pg_client +from chalicelib.utils import dev +from chalicelib.utils.TimeUTC import TimeUTC +from chalicelib.utils.helper import environ + +from chalicelib.core import tenants + + +def create_new_member(email, password, admin, name, owner=False): + with pg_client.PostgresClient() as cur: + query = cur.mogrify(f"""\ + WITH u AS ( + INSERT INTO public.users (email, role, name, data) + VALUES (%(email)s, %(role)s, %(name)s, %(data)s) + RETURNING user_id,email,role,name,appearance + ), + au AS (INSERT + INTO public.basic_authentication (user_id, password, generated_password) + VALUES ((SELECT user_id FROM u), crypt(%(password)s, gen_salt('bf', 12)), TRUE)) + SELECT u.user_id AS id, + u.email, + u.role, + u.name, + TRUE AS change_password, + (CASE WHEN u.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, + (CASE WHEN u.role = 'admin' THEN TRUE ELSE FALSE END) AS admin, + (CASE WHEN u.role = 'member' THEN TRUE ELSE FALSE END) AS member, + u.appearance + FROM u;""", + {"email": email, "password": password, + "role": "owner" if owner else "admin" if admin else "member", "name": name, + "data": json.dumps({"lastAnnouncementView": TimeUTC.now()})}) + cur.execute( + query + ) + return helper.dict_to_camel_case(cur.fetchone()) + + +def restore_member(user_id, email, password, admin, name, owner=False): + with pg_client.PostgresClient() as cur: + query = cur.mogrify(f"""\ + UPDATE public.users + SET name= %(name)s, + role = %(role)s, + deleted_at= NULL, + created_at = timezone('utc'::text, now()), + api_key= generate_api_key(20) + WHERE user_id=%(user_id)s + RETURNING user_id AS id, + email, + role, + name, + TRUE AS change_password, + (CASE WHEN role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, + (CASE WHEN role = 'admin' THEN TRUE ELSE FALSE END) AS admin, + (CASE WHEN role = 'member' THEN TRUE ELSE FALSE END) AS member, + appearance;""", + {"user_id": user_id, "email": email, + "role": "owner" if owner else "admin" if admin else "member", "name": name}) + cur.execute( + query + ) + result = helper.dict_to_camel_case(cur.fetchone()) + query = cur.mogrify("""\ + UPDATE public.basic_authentication + SET password= crypt(%(password)s, gen_salt('bf', 12)), + generated_password= TRUE, + token=NULL, + token_requested_at=NULL + WHERE user_id=%(user_id)s;""", + {"user_id": user_id, "password": password}) + cur.execute( + query + ) + + return result + + +def update(tenant_id, user_id, changes): + AUTH_KEYS = ["password", "generatedPassword", "token"] + if len(changes.keys()) == 0: + return None + + sub_query_users = [] + sub_query_bauth = [] + for key in changes.keys(): + if key in AUTH_KEYS: + if key == "password": + sub_query_bauth.append("password = crypt(%(password)s, gen_salt('bf', 12))") + sub_query_bauth.append("changed_at = timezone('utc'::text, now())") + elif key == "token": + if changes[key] is not None: + sub_query_bauth.append("token = %(token)s") + sub_query_bauth.append("token_requested_at = timezone('utc'::text, now())") + else: + sub_query_bauth.append("token = NULL") + sub_query_bauth.append("token_requested_at = NULL") + else: + sub_query_bauth.append(f"{helper.key_to_snake_case(key)} = %({key})s") + else: + if key == "appearance": + sub_query_users.append(f"appearance = %(appearance)s::jsonb") + changes["appearance"] = json.dumps(changes[key]) + else: + sub_query_users.append(f"{helper.key_to_snake_case(key)} = %({key})s") + + with pg_client.PostgresClient() as cur: + if len(sub_query_users) > 0: + cur.execute( + cur.mogrify(f"""\ + UPDATE public.users + SET {" ,".join(sub_query_users)} + FROM public.basic_authentication + WHERE users.user_id = %(user_id)s + AND users.user_id = basic_authentication.user_id + RETURNING users.user_id AS id, + users.email, + users.role, + users.name, + basic_authentication.generated_password AS change_password, + (CASE WHEN users.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, + (CASE WHEN users.role = 'admin' THEN TRUE ELSE FALSE END) AS admin, + (CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member, + users.appearance;""", + {"user_id": user_id, **changes}) + ) + if len(sub_query_bauth) > 0: + cur.execute( + cur.mogrify(f"""\ + UPDATE public.basic_authentication + SET {" ,".join(sub_query_bauth)} + FROM public.users AS users + WHERE basic_authentication.user_id = %(user_id)s + AND users.user_id = basic_authentication.user_id + RETURNING users.user_id AS id, + users.email, + users.role, + users.name, + basic_authentication.generated_password AS change_password, + (CASE WHEN users.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, + (CASE WHEN users.role = 'admin' THEN TRUE ELSE FALSE END) AS admin, + (CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member, + users.appearance;""", + {"user_id": user_id, **changes}) + ) + + return helper.dict_to_camel_case(cur.fetchone()) + + +def create_member(tenant_id, user_id, data): + admin = get(tenant_id=tenant_id, user_id=user_id) + if not admin["admin"] and not admin["superAdmin"]: + return {"errors": ["unauthorized"]} + if data.get("userId") is not None: + return {"errors": ["please use POST/PUT /client/members/{memberId} for update"]} + user = get_by_email_only(email=data["email"]) + if user: + return {"errors": ["user already exists"]} + name = data.get("name", None) + if name is not None and not helper.is_alphabet_latin_space(name): + return {"errors": ["invalid user name"]} + if name is None: + name = data["email"] + temp_pass = helper.generate_salt()[:8] + user = get_deleted_user_by_email(email=data["email"]) + if user is not None: + new_member = restore_member(email=data["email"], password=temp_pass, + admin=data.get("admin", False), name=name, user_id=user["userId"]) + else: + new_member = create_new_member(email=data["email"], password=temp_pass, + admin=data.get("admin", False), name=name) + + helper.async_post(environ['email_basic'] % 'member_invitation', + { + "email": data["email"], + "userName": data["email"], + "tempPassword": temp_pass, + "clientId": tenants.get_by_tenant_id(tenant_id)["name"], + "senderName": admin["name"] + }) + return {"data": new_member} + + +def get(user_id, tenant_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""SELECT + users.user_id AS id, + email, + role, + name, + basic_authentication.generated_password, + (CASE WHEN role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, + (CASE WHEN role = 'admin' THEN TRUE ELSE FALSE END) AS admin, + (CASE WHEN role = 'member' THEN TRUE ELSE FALSE END) AS member, + appearance, + api_key + FROM public.users LEFT JOIN public.basic_authentication ON users.user_id=basic_authentication.user_id + WHERE + users.user_id = %(userId)s + AND deleted_at IS NULL + LIMIT 1;""", + {"userId": user_id}) + ) + r = cur.fetchone() + return helper.dict_to_camel_case(r, ignore_keys=["appearance"]) + + +def generate_new_api_key(user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""UPDATE public.users + SET api_key=generate_api_key(20) + WHERE + users.user_id = %(userId)s + AND deleted_at IS NULL + RETURNING api_key;""", + {"userId": user_id}) + ) + r = cur.fetchone() + return helper.dict_to_camel_case(r) + + +def edit(user_id_to_update, tenant_id, changes, editor_id): + ALLOW_EDIT = ["name", "email", "admin", "appearance"] + user = get(user_id=user_id_to_update, tenant_id=tenant_id) + if editor_id != user_id_to_update or "admin" in changes and changes["admin"] != user["admin"]: + admin = get(tenant_id=tenant_id, user_id=editor_id) + if not admin["superAdmin"] and not admin["admin"]: + return {"errors": ["unauthorized"]} + + keys = list(changes.keys()) + for k in keys: + if k not in ALLOW_EDIT: + changes.pop(k) + keys = list(changes.keys()) + + if len(keys) > 0: + if "email" in keys and changes["email"] != user["email"]: + if email_exists(changes["email"]): + return {"errors": ["email already exists."]} + if get_deleted_user_by_email(changes["email"]) is not None: + return {"errors": ["email previously deleted."]} + if "admin" in keys: + changes["role"] = "admin" if changes.pop("admin") else "member" + if len(changes.keys()) > 0: + updated_user = update(tenant_id=tenant_id, user_id=user_id_to_update, changes=changes) + + return {"data": updated_user} + return {"data": user} + + +def get_by_email_only(email): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""SELECT + users.user_id AS id, + 1 AS tenant_id, + users.email, + users.role, + users.name, + basic_authentication.generated_password, + (CASE WHEN users.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, + (CASE WHEN users.role = 'admin' THEN TRUE ELSE FALSE END) AS admin, + (CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member + FROM public.users LEFT JOIN public.basic_authentication ON users.user_id=basic_authentication.user_id + WHERE + users.email = %(email)s + AND users.deleted_at IS NULL;""", + {"email": email}) + ) + r = cur.fetchall() + return helper.list_to_camel_case(r) + + +def get_by_email_reset(email, reset_token): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""SELECT + users.user_id AS id, + 1 AS tenant_id, + users.email, + users.role, + users.name, + basic_authentication.generated_password, + (CASE WHEN users.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, + (CASE WHEN users.role = 'admin' THEN TRUE ELSE FALSE END) AS admin, + (CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member + FROM public.users LEFT JOIN public.basic_authentication ON users.user_id=basic_authentication.user_id + WHERE + users.email = %(email)s + AND basic_authentication.token =%(token)s + AND users.deleted_at IS NULL""", + {"email": email, "token": reset_token}) + ) + r = cur.fetchone() + return helper.dict_to_camel_case(r) + + +def get_members(tenant_id): + with pg_client.PostgresClient() as cur: + cur.execute( + f"""SELECT + users.user_id AS id, + users.email, + users.role, + users.name, + basic_authentication.generated_password, + (CASE WHEN users.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, + (CASE WHEN users.role = 'admin' THEN TRUE ELSE FALSE END) AS admin, + (CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member + FROM public.users LEFT JOIN public.basic_authentication ON users.user_id=basic_authentication.user_id + WHERE users.deleted_at IS NULL + ORDER BY name, id""" + ) + r = cur.fetchall() + if len(r): + return helper.list_to_camel_case(r) + + return [] + + +def delete_member(user_id, tenant_id, id_to_delete): + if user_id == id_to_delete: + return {"errors": ["unauthorized, cannot delete self"]} + + admin = get(user_id=user_id, tenant_id=tenant_id) + if admin["member"]: + return {"errors": ["unauthorized"]} + + to_delete = get(user_id=id_to_delete, tenant_id=tenant_id) + if to_delete is None: + return {"errors": ["not found"]} + + if to_delete["superAdmin"]: + return {"errors": ["cannot delete super admin"]} + + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify(f"""UPDATE public.users + SET deleted_at = timezone('utc'::text, now()) + WHERE user_id=%(user_id)s;""", + {"user_id": id_to_delete})) + return {"data": get_members(tenant_id=tenant_id)} + + +def change_password(tenant_id, user_id, email, old_password, new_password): + item = get(tenant_id=tenant_id, user_id=user_id) + if item is None: + return {"errors": ["access denied"]} + if old_password == new_password: + return {"errors": ["old and new password are the same"]} + auth = authenticate(email, old_password, for_change_password=True) + if auth is None: + return {"errors": ["wrong password"]} + changes = {"password": new_password, "generatedPassword": False} + return {"data": update(tenant_id=tenant_id, user_id=user_id, changes=changes), + "jwt": authenticate(email, new_password)["jwt"]} + + +def email_exists(email): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""SELECT + count(user_id) + FROM public.users + WHERE + email = %(email)s + AND deleted_at IS NULL + LIMIT 1;""", + {"email": email}) + ) + r = cur.fetchone() + return r["count"] > 0 + + +def get_deleted_user_by_email(email): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"""SELECT + * + FROM public.users + WHERE + email = %(email)s + AND deleted_at NOTNULL + LIMIT 1;""", + {"email": email}) + ) + r = cur.fetchone() + return helper.dict_to_camel_case(r) + + +def auth_exists(user_id, tenant_id, jwt_iat, jwt_aud): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + f"SELECT user_id AS id,jwt_iat, changed_at FROM public.users INNER JOIN public.basic_authentication USING(user_id) WHERE user_id = %(userId)s AND deleted_at IS NULL LIMIT 1;", + {"userId": user_id}) + ) + r = cur.fetchone() + return r is not None \ + and r.get("jwt_iat") is not None \ + and (abs(jwt_iat - TimeUTC.datetime_to_timestamp(r["jwt_iat"]) // 1000) <= 1 \ + or (jwt_aud.startswith("plugin") \ + and (r["changed_at"] is None \ + or jwt_iat >= (TimeUTC.datetime_to_timestamp(r["changed_at"]) // 1000))) + ) + + +@dev.timed +def authenticate(email, password, for_change_password=False, for_plugin=False): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + f"""SELECT + users.user_id AS id, + 1 AS tenant_id, + users.role, + users.name, + basic_authentication.generated_password AS change_password, + (CASE WHEN users.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, + (CASE WHEN users.role = 'admin' THEN TRUE ELSE FALSE END) AS admin, + (CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member, + users.appearance + FROM public.users INNER JOIN public.basic_authentication USING(user_id) + WHERE users.email = %(email)s + AND basic_authentication.password = crypt(%(password)s, basic_authentication.password) + AND basic_authentication.user_id = (SELECT su.user_id FROM public.users AS su WHERE su.email=%(email)s AND su.deleted_at IS NULL LIMIT 1) + LIMIT 1;""", + {"email": email, "password": password}) + + cur.execute(query) + r = cur.fetchone() + + if r is not None: + if for_change_password: + return True + r = helper.dict_to_camel_case(r, ignore_keys=["appearance"]) + query = cur.mogrify( + f"""UPDATE public.users + SET jwt_iat = timezone('utc'::text, now()) + WHERE user_id = %(user_id)s + RETURNING jwt_iat;""", + {"user_id": r["id"]}) + cur.execute(query) + return { + "jwt": authorizers.generate_jwt(r['id'], r['tenantId'], + TimeUTC.datetime_to_timestamp(cur.fetchone()["jwt_iat"]), + aud=f"plugin:{helper.get_stage_name()}" if for_plugin else f"front:{helper.get_stage_name()}"), + "email": email, + **r + } + return None diff --git a/api/chalicelib/core/webhook.py b/api/chalicelib/core/webhook.py new file mode 100644 index 000000000..99a3b0569 --- /dev/null +++ b/api/chalicelib/core/webhook.py @@ -0,0 +1,178 @@ +from chalicelib.utils import pg_client, helper +from chalicelib.utils.TimeUTC import TimeUTC +import requests + + +def get_by_id(webhook_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + SELECT + w.* + FROM public.webhooks AS w + where w.webhook_id =%(webhook_id)s AND deleted_at ISNULL;""", + {"webhook_id": webhook_id}) + ) + w = helper.dict_to_camel_case(cur.fetchone()) + if w: + w["createdAt"] = TimeUTC.datetime_to_timestamp(w["createdAt"]) + return w + + +def get(tenant_id, webhook_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + SELECT + w.* + FROM public.webhooks AS w + where w.webhook_id =%(webhook_id)s AND deleted_at ISNULL;""", + {"webhook_id": webhook_id}) + ) + w = helper.dict_to_camel_case(cur.fetchone()) + if w: + w["createdAt"] = TimeUTC.datetime_to_timestamp(w["createdAt"]) + return w + + +def get_by_type(tenant_id, webhook_type): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + SELECT + w.webhook_id AS id,w.webhook_id,w.endpoint,w.auth_header,w.type,w.index,w.name,w.created_at + FROM public.webhooks AS w + WHERE w.type =%(type)s AND deleted_at ISNULL;""", + {"type": webhook_type}) + ) + webhooks = helper.list_to_camel_case(cur.fetchall()) + for w in webhooks: + w["createdAt"] = TimeUTC.datetime_to_timestamp(w["createdAt"]) + return webhooks + + +def get_by_tenant(tenant_id, replace_none=False): + with pg_client.PostgresClient() as cur: + cur.execute("""\ + SELECT + w.* + FROM public.webhooks AS w + WHERE deleted_at ISNULL;""" + ) + all = helper.list_to_camel_case(cur.fetchall()) + if replace_none: + for w in all: + w["createdAt"] = TimeUTC.datetime_to_timestamp(w["createdAt"]) + for k in w.keys(): + if w[k] is None: + w[k] = '' + else: + for w in all: + w["createdAt"] = TimeUTC.datetime_to_timestamp(w["createdAt"]) + return all + + +def update(tenant_id, webhook_id, changes, replace_none=False): + allow_update = ["name", "index", "authHeader", "endpoint"] + with pg_client.PostgresClient() as cur: + sub_query = [f"{helper.key_to_snake_case(k)} = %({k})s" for k in changes.keys() if k in allow_update] + cur.execute( + cur.mogrify(f"""\ + UPDATE public.webhooks + SET {','.join(sub_query)} + WHERE webhook_id =%(id)s AND deleted_at ISNULL + RETURNING *;""", + {"id": webhook_id, **changes}) + ) + w = helper.dict_to_camel_case(cur.fetchone()) + w["createdAt"] = TimeUTC.datetime_to_timestamp(w["createdAt"]) + if replace_none: + for k in w.keys(): + if w[k] is None: + w[k] = '' + return w + + +def add(tenant_id, endpoint, auth_header=None, webhook_type='webhook', name="", replace_none=False): + with pg_client.PostgresClient() as cur: + query = cur.mogrify("""\ + INSERT INTO public.webhooks(endpoint,auth_header,type,name) + VALUES (%(endpoint)s, %(auth_header)s, %(type)s,%(name)s) + RETURNING *;""", + {"endpoint": endpoint, "auth_header": auth_header, + "type": webhook_type, "name": name}) + cur.execute( + query + ) + w = helper.dict_to_camel_case(cur.fetchone()) + w["createdAt"] = TimeUTC.datetime_to_timestamp(w["createdAt"]) + if replace_none: + for k in w.keys(): + if w[k] is None: + w[k] = '' + return w + + +def add_edit(tenant_id, data, replace_none=None): + if "webhookId" in data: + return update(tenant_id=tenant_id, webhook_id=data["webhookId"], + changes={"endpoint": data["endpoint"], + "authHeader": None if "authHeader" not in data else data["authHeader"], + "name": data["name"] if "name" in data else ""}, replace_none=replace_none) + else: + return add(tenant_id=tenant_id, + endpoint=data["endpoint"], + auth_header=None if "authHeader" not in data else data["authHeader"], + name=data["name"] if "name" in data else "", replace_none=replace_none) + + +def delete(tenant_id, webhook_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + UPDATE public.webhooks + SET deleted_at = (now() at time zone 'utc') + WHERE webhook_id =%(id)s AND deleted_at ISNULL + RETURNING *;""", + {"id": webhook_id}) + ) + return {"data": {"state": "success"}} + + +def trigger_batch(data_list): + webhooks_map = {} + for w in data_list: + if w["destination"] not in webhooks_map: + webhooks_map[w["destination"]] = get_by_id(webhook_id=w["destination"]) + __trigger(hook=webhooks_map[w["destination"]], data=w["data"]) + + +def __trigger(hook, data): + if hook["type"] == 'webhook': + headers = {} + if hook["authHeader"] is not None and len(hook["authHeader"]) > 0: + headers = {"Authorization": hook["authHeader"]} + + # body = { + # "webhookId": hook["id"], + # "createdAt": TimeUTC.now(), + # "event": event, + # "data": data + # } + + r = requests.post(url=hook["endpoint"], json=data, headers=headers) + if r.status_code != 200: + print("=======> webhook: something went wrong") + print(r) + print(r.status_code) + print(r.text) + return + response = None + try: + response = r.json() + except: + try: + response = r.text + except: + print("no response found") + return response diff --git a/api/chalicelib/core/weekly_report.py b/api/chalicelib/core/weekly_report.py new file mode 100644 index 000000000..23ff97446 --- /dev/null +++ b/api/chalicelib/core/weekly_report.py @@ -0,0 +1,233 @@ +from chalicelib.utils import pg_client, helper +from chalicelib.utils.helper import environ +from chalicelib.utils.helper import get_issue_title + +LOWEST_BAR_VALUE = 3 + + +def get_config(user_id): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify("""\ + SELECT users.weekly_report + FROM public.users + WHERE users.deleted_at ISNULL AND users.user_id=%(user_id)s + LIMIT 1;""", {"user_id": user_id})) + result = cur.fetchone() + return helper.dict_to_camel_case(result) + + +def edit_config(user_id, weekly_report): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify("""\ + UPDATE public.users + SET weekly_report= %(weekly_report)s + WHERE users.deleted_at ISNULL + AND users.user_id=%(user_id)s + RETURNING weekly_report;""", {"user_id": user_id, "weekly_report": weekly_report})) + result = cur.fetchone() + return helper.dict_to_camel_case(result) + + +def cron(): + with pg_client.PostgresClient() as cur: + cur.execute("""\ + SELECT project_id, + name AS project_name, + users.emails AS emails, + TO_CHAR(DATE_TRUNC('day', now()) - INTERVAL '1 week', 'Mon. DDth, YYYY') AS period_start, + TO_CHAR(DATE_TRUNC('day', now()), 'Mon. DDth, YYYY') AS period_end, + COALESCE(week_0_issues.count, 0) AS this_week_issues_count, + COALESCE(week_1_issues.count, 0) AS past_week_issues_count, + COALESCE(month_1_issues.count, 0) AS past_month_issues_count + FROM public.projects + INNER JOIN LATERAL ( + SELECT sessions.project_id + FROM public.sessions + WHERE sessions.project_id = projects.project_id + AND start_ts >= (EXTRACT(EPOCH FROM now() - INTERVAL '3 days') * 1000)::BIGINT + LIMIT 1) AS recently_active USING (project_id) + INNER JOIN LATERAL ( + SELECT COALESCE(ARRAY_AGG(email), '{}') AS emails + FROM public.users + WHERE users.tenant_id = projects.tenant_id + AND users.deleted_at ISNULL + AND users.weekly_report + ) AS users ON (TRUE) + LEFT JOIN LATERAL ( + SELECT COUNT(issues.*) AS count + FROM events_common.issues + INNER JOIN public.sessions USING (session_id) + WHERE sessions.project_id = projects.project_id + AND issues.timestamp >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + ) AS week_0_issues ON (TRUE) + LEFT JOIN LATERAL ( + SELECT COUNT(issues.*) AS count + FROM events_common.issues + INNER JOIN public.sessions USING (session_id) + WHERE sessions.project_id = projects.project_id + AND issues.timestamp <= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + AND issues.timestamp >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '2 week') * 1000)::BIGINT + ) AS week_1_issues ON (TRUE) + LEFT JOIN LATERAL ( + SELECT COUNT(issues.*) AS count + FROM events_common.issues + INNER JOIN public.sessions USING (session_id) + WHERE sessions.project_id = projects.project_id + AND issues.timestamp <= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + AND issues.timestamp >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '5 week') * 1000)::BIGINT + ) AS month_1_issues ON (TRUE) + WHERE projects.deleted_at ISNULL;""") + projects_data = cur.fetchall() + for p in projects_data: + print(f"checking {p['project_name']} : {p['project_id']}") + if len(p["emails"]) == 0 \ + or p["this_week_issues_count"] + p["past_week_issues_count"] + p["past_month_issues_count"] == 0: + print('ignore') + continue + print("valid") + p["past_week_issues_evolution"] = helper.__decimal_limit( + helper.__progress(p["this_week_issues_count"], p["past_week_issues_count"]), 1) + p["past_month_issues_evolution"] = helper.__decimal_limit( + helper.__progress(p["this_week_issues_count"], p["past_month_issues_count"]), 1) + cur.execute(cur.mogrify(""" + SELECT LEFT(TO_CHAR(timestamp_i, 'Dy'),1) AS day_short, + TO_CHAR(timestamp_i, 'Mon. DD, YYYY') AS day_long, + ( + SELECT COUNT(*) + FROM events_common.issues INNER JOIN public.issues USING (issue_id) + WHERE project_id = %(project_id)s + AND timestamp >= (EXTRACT(EPOCH FROM timestamp_i) * 1000)::BIGINT + AND timestamp <= (EXTRACT(EPOCH FROM timestamp_i + INTERVAL '1 day') * 1000)::BIGINT + ) AS issues_count + FROM generate_series( + DATE_TRUNC('day', now()) - INTERVAL '7 days', + DATE_TRUNC('day', now()) - INTERVAL '1 day', + '1 day'::INTERVAL + ) AS timestamp_i + ORDER BY timestamp_i;""", {"project_id": p["project_id"]})) + days_partition = cur.fetchall() + max_days_partition = max(x['issues_count'] for x in days_partition) + for d in days_partition: + if max_days_partition <= 0: + d["value"] = LOWEST_BAR_VALUE + else: + d["value"] = d["issues_count"] * 100 / max_days_partition + d["value"] = d["value"] if d["value"] > LOWEST_BAR_VALUE else LOWEST_BAR_VALUE + cur.execute(cur.mogrify("""\ + SELECT type, COUNT(*) AS count + FROM events_common.issues INNER JOIN public.issues USING (issue_id) + WHERE project_id = %(project_id)s + AND timestamp >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '7 days') * 1000)::BIGINT + GROUP BY type + ORDER BY count DESC, type + LIMIT 4;""", {"project_id": p["project_id"]})) + issues_by_type = cur.fetchall() + max_issues_by_type = sum(i["count"] for i in issues_by_type) + for i in issues_by_type: + i["type"] = get_issue_title(i["type"]) + if max_issues_by_type <= 0: + i["value"] = LOWEST_BAR_VALUE + else: + i["value"] = i["count"] * 100 / max_issues_by_type + cur.execute(cur.mogrify("""\ + SELECT TO_CHAR(timestamp_i, 'Dy') AS day_short, + TO_CHAR(timestamp_i, 'Mon. DD, YYYY') AS day_long, + COALESCE((SELECT JSONB_AGG(sub) + FROM ( + SELECT type, COUNT(*) AS count + FROM events_common.issues + INNER JOIN public.issues USING (issue_id) + WHERE project_id = %(project_id)s + AND timestamp >= (EXTRACT(EPOCH FROM timestamp_i) * 1000)::BIGINT + AND timestamp <= (EXTRACT(EPOCH FROM timestamp_i + INTERVAL '1 day') * 1000)::BIGINT + GROUP BY type + ORDER BY count + ) AS sub), '[]'::JSONB) AS partition + FROM generate_series( + DATE_TRUNC('day', now()) - INTERVAL '7 days', + DATE_TRUNC('day', now()) - INTERVAL '1 day', + '1 day'::INTERVAL + ) AS timestamp_i + GROUP BY timestamp_i + ORDER BY timestamp_i;""", {"project_id": p["project_id"]})) + issues_breakdown_by_day = cur.fetchall() + for i in issues_breakdown_by_day: + i["sum"] = sum(x["count"] for x in i["partition"]) + for j in i["partition"]: + j["type"] = get_issue_title(j["type"]) + max_days_partition = max(i["sum"] for i in issues_breakdown_by_day) + for i in issues_breakdown_by_day: + for j in i["partition"]: + if max_days_partition <= 0: + j["value"] = LOWEST_BAR_VALUE + else: + j["value"] = j["count"] * 100 / max_days_partition + j["value"] = j["value"] if j["value"] > LOWEST_BAR_VALUE else LOWEST_BAR_VALUE + cur.execute(cur.mogrify(""" + SELECT type, + COUNT(*) AS issue_count, + COUNT(DISTINCT session_id) AS sessions_count, + (SELECT COUNT(DISTINCT sessions.session_id) + FROM public.sessions + INNER JOIN events_common.issues AS sci USING (session_id) + INNER JOIN public.issues AS si USING (issue_id) + WHERE si.project_id = %(project_id)s + AND sessions.project_id = %(project_id)s + AND sessions.start_ts <= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + AND sessions.start_ts >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '2 weeks') * 1000)::BIGINT + AND si.type = mi.type + AND sessions.duration IS NOT NULL + ) AS last_week_sessions_count, + (SELECT COUNT(DISTINCT sci.session_id) + FROM public.sessions + INNER JOIN events_common.issues AS sci USING (session_id) + INNER JOIN public.issues AS si USING (issue_id) + WHERE si.project_id = %(project_id)s + AND sessions.project_id = %(project_id)s + AND sessions.start_ts <= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + AND sessions.start_ts >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '5 weeks') * 1000)::BIGINT + AND si.type = mi.type + AND sessions.duration IS NOT NULL + ) AS last_month_sessions_count + FROM events_common.issues + INNER JOIN public.issues AS mi USING (issue_id) + INNER JOIN public.sessions USING (session_id) + WHERE mi.project_id = %(project_id)s AND sessions.project_id = %(project_id)s AND sessions.duration IS NOT NULL + AND sessions.start_ts >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + GROUP BY type + ORDER BY issue_count DESC;""", {"project_id": p["project_id"]})) + issues_breakdown_list = cur.fetchall() + if len(issues_breakdown_list) > 4: + others = {"type": "Others", + "sessions_count": sum(i["sessions_count"] for i in issues_breakdown_list[4:]), + "issue_count": sum(i["issue_count"] for i in issues_breakdown_list[4:]), + "last_week_sessions_count": sum( + i["last_week_sessions_count"] for i in issues_breakdown_list[4:]), + "last_month_sessions_count": sum( + i["last_month_sessions_count"] for i in issues_breakdown_list[4:])} + issues_breakdown_list = issues_breakdown_list[:4] + issues_breakdown_list.append(others) + for i in issues_breakdown_list: + i["type"] = get_issue_title(i["type"]) + i["last_week_sessions_evolution"] = helper.__decimal_limit( + helper.__progress(i["sessions_count"], i["last_week_sessions_count"]), 1) + i["last_month_sessions_evolution"] = helper.__decimal_limit( + helper.__progress(i["sessions_count"], i["last_month_sessions_count"]), 1) + i["sessions_count"] = f'{i["sessions_count"]:,}' + keep_types = [i["type"] for i in issues_breakdown_list] + for i in issues_breakdown_by_day: + keep = [] + for j in i["partition"]: + if j["type"] in keep_types: + keep.append(j) + i["partition"] = keep + helper.async_post(environ['email_funnel'] % "weekly_report2", + {"email": p.pop("emails"), + "data": { + **p, + "days_partition": days_partition, + "issues_by_type": issues_by_type, + "issues_breakdown_by_day": issues_breakdown_by_day, + "issues_breakdown_list": issues_breakdown_list + } + }) diff --git a/api/chalicelib/utils/TimeUTC.py b/api/chalicelib/utils/TimeUTC.py new file mode 100644 index 000000000..4b19c45ba --- /dev/null +++ b/api/chalicelib/utils/TimeUTC.py @@ -0,0 +1,115 @@ +from datetime import datetime, timedelta +from calendar import monthrange +import pytz + + +class TimeUTC: + MS_MINUTE = 60 * 1000 + MS_HOUR = MS_MINUTE * 60 + MS_DAY = MS_HOUR * 24 + MS_MONTH = MS_DAY * 30 + MS_MONTH_TRUE = monthrange(datetime.now(pytz.utc).astimezone(pytz.utc).year, + datetime.now(pytz.utc).astimezone(pytz.utc).month)[1] * MS_DAY + RANGE_VALUE = None + + @staticmethod + def midnight(delta_days=0): + return int((datetime.now(pytz.utc) + timedelta(delta_days)) \ + .replace(hour=0, minute=0, second=0, microsecond=0) \ + .astimezone(pytz.utc).timestamp() * 1000) + + @staticmethod + def __now(delta_days=0, delta_minutes=0, delta_seconds=0): + return (datetime.now(pytz.utc) + timedelta(days=delta_days, minutes=delta_minutes, seconds=delta_seconds)) \ + .astimezone(pytz.utc) + + @staticmethod + def now(delta_days=0, delta_minutes=0, delta_seconds=0): + return int(TimeUTC.__now(delta_days=delta_days, delta_minutes=delta_minutes, + delta_seconds=delta_seconds).timestamp() * 1000) + + @staticmethod + def month_start(delta_month=0): + month = TimeUTC.__now().month + delta_month + return int(datetime.now(pytz.utc) \ + .replace(year=TimeUTC.__now().year + ((-12 + month) // 12 if month % 12 <= 0 else month // 12), + month=12 + month % 12 if month % 12 <= 0 else month % 12 if month > 12 else month, + day=1, + hour=0, minute=0, + second=0, + microsecond=0) \ + .astimezone(pytz.utc).timestamp() * 1000) + + @staticmethod + def year_start(delta_year=0): + return int(datetime.now(pytz.utc) \ + .replace(year=TimeUTC.__now().year + delta_year, month=1, day=1, hour=0, minute=0, second=0, + microsecond=0) \ + .astimezone(pytz.utc).timestamp() * 1000) + + @staticmethod + def custom(year=None, month=None, day=None, hour=None, minute=None): + args = locals() + return int(datetime.now(pytz.utc) \ + .replace(**{key: args[key] for key in args if args[key] is not None}, second=0, microsecond=0) \ + .astimezone(pytz.utc).timestamp() * 1000) + + @staticmethod + def future(delta_day, delta_hour, delta_minute, minutes_period=None, start=None): + this_time = TimeUTC.__now() + if delta_day == -1: + if this_time.hour < delta_hour or this_time.hour == delta_hour and this_time.minute < delta_minute: + return TimeUTC.custom(hour=delta_hour, minute=delta_minute) + + return TimeUTC.custom(day=TimeUTC.__now(1).day, hour=delta_hour, minute=delta_minute) + elif delta_day > -1: + if this_time.weekday() < delta_day or this_time.weekday() == delta_day and ( + this_time.hour < delta_hour or this_time.hour == delta_hour and this_time.minute < delta_minute): + return TimeUTC.custom(day=TimeUTC.__now(delta_day - this_time.weekday()).day, hour=delta_hour, + minute=delta_minute) + + return TimeUTC.custom(day=TimeUTC.__now(7 + delta_day - this_time.weekday()).day, hour=delta_hour, + minute=delta_minute) + if start is not None: + return start + minutes_period * 60 * 1000 + + return TimeUTC.now(delta_minutes=minutes_period) + + @staticmethod + def from_ms_timestamp(ts): + return datetime.fromtimestamp(ts // 1000, pytz.utc) + + @staticmethod + def to_human_readable(ts, fmt='%Y-%m-%d %H:%M:%S UTC'): + return datetime.utcfromtimestamp(ts // 1000).strftime(fmt) + + @staticmethod + def human_to_timestamp(ts, pattern): + return int(datetime.strptime(ts, pattern).timestamp() * 1000) + + @staticmethod + def datetime_to_timestamp(date): + if date is None: + return None + return int(datetime.timestamp(date) * 1000) + + @staticmethod + def get_start_end_from_range(range_value): + range_value = range_value.upper() + if TimeUTC.RANGE_VALUE is None: + this_instant = TimeUTC.now() + TimeUTC.RANGE_VALUE = { + "TODAY": {"start": TimeUTC.midnight(), "end": this_instant}, + "YESTERDAY": {"start": TimeUTC.midnight(delta_days=-1), "end": TimeUTC.midnight()}, + "LAST_7_DAYS": {"start": TimeUTC.midnight(delta_days=-7), "end": this_instant}, + "LAST_30_DAYS": {"start": TimeUTC.midnight(delta_days=-30), "end": this_instant}, + "THIS_MONTH": {"start": TimeUTC.month_start(), "end": this_instant}, + "LAST_MONTH": {"start": TimeUTC.month_start(delta_month=-1), "end": TimeUTC.month_start()}, + "THIS_YEAR": {"start": TimeUTC.year_start(), "end": this_instant}, + "CUSTOM_RANGE": {"start": TimeUTC.midnight(delta_days=-7), "end": this_instant} # Default is 7 days + } + return TimeUTC.RANGE_VALUE[range_value]["start"], TimeUTC.RANGE_VALUE[range_value]["end"] + + @staticmethod + def get_utc_offset(): + return int((datetime.now(pytz.utc).now() - datetime.now(pytz.utc).replace(tzinfo=None)).total_seconds() * 1000) diff --git a/api/chalicelib/utils/__init__.py b/api/chalicelib/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/chalicelib/utils/args_transformer.py b/api/chalicelib/utils/args_transformer.py new file mode 100644 index 000000000..5611df054 --- /dev/null +++ b/api/chalicelib/utils/args_transformer.py @@ -0,0 +1,10 @@ +def int_arg(x): + return int(x) if x is not None else int(0) + + +def float_arg(x): + return float(x) if x is not None else float(0) + + +def string(x): + return x diff --git a/api/chalicelib/utils/captcha.py b/api/chalicelib/utils/captcha.py new file mode 100644 index 000000000..f499a6da9 --- /dev/null +++ b/api/chalicelib/utils/captcha.py @@ -0,0 +1,24 @@ +from chalicelib.utils.helper import environ as env +import requests +from chalicelib.utils import helper + + +def __get_captcha_config(): + return env["captcha_server"], env["captcha_key"] + + +def is_valid(response): + if not helper.allow_captcha(): + print("!! Captcha is disabled") + return True + url, secret = __get_captcha_config() + r = requests.post(url=url, data={"secret": secret, "response": response}) + if r.status_code != 200: + print("something went wrong") + print(r) + print(r.status_code) + print(r.text) + return + r = r.json() + print(r) + return r["success"] diff --git a/api/chalicelib/utils/dev.py b/api/chalicelib/utils/dev.py new file mode 100644 index 000000000..3b350ce7e --- /dev/null +++ b/api/chalicelib/utils/dev.py @@ -0,0 +1,28 @@ +from functools import wraps +from time import time +import inspect +from chalicelib.utils import helper + + +def timed(f): + @wraps(f) + def wrapper(*args, **kwds): + if not helper.TRACK_TIME: + return f(*args, **kwds) + start = time() + result = f(*args, **kwds) + elapsed = time() - start + if inspect.stack()[1][3] == "_view_func": + print("DEBUG: %s: took %d s to finish" % (f.__name__, elapsed)) + else: + call_stack = [i[3] for i in inspect.stack()[1:] if i[3] != "wrapper"] + call_stack = [c for c in call_stack if + c not in ['__init__', '__call__', 'finish_request', 'process_request_thread', + 'handle_request', '_generic_handle', 'handle', '_bootstrap_inner', 'run', + '_bootstrap', '_main_rest_api_handler', '_user_handler', + '_get_view_function_response', 'wrapped_event', 'handle_one_request', + '_global_error_handler', 'asayer_middleware']] + print("DEBUG: %s > %s took %d s to finish" % (" > ".join(call_stack), f.__name__, elapsed)) + return result + + return wrapper diff --git a/api/chalicelib/utils/email_handler.py b/api/chalicelib/utils/email_handler.py new file mode 100644 index 000000000..2de35e616 --- /dev/null +++ b/api/chalicelib/utils/email_handler.py @@ -0,0 +1,90 @@ +import base64 +import re +from email.header import Header +from email.mime.image import MIMEImage +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +from chalicelib.utils import helper, smtp +from chalicelib.utils.helper import environ + + +def __get_subject(subject): + return subject if helper.is_production() else f"{helper.get_stage_name()}: {subject}" + + +def __get_html_from_file(source, formatting_variables): + if formatting_variables is None: + formatting_variables = {} + formatting_variables["frontend_url"] = environ["SITE_URL"] + with open(source, "r") as body: + BODY_HTML = body.read() + if formatting_variables is not None and len(formatting_variables.keys()) > 0: + BODY_HTML = re.sub(r"%(?![(])", "%%", BODY_HTML) + BODY_HTML = BODY_HTML % {**formatting_variables} + return BODY_HTML + + +def __replace_images(HTML): + pattern_holder = re.compile(r'') + return HTML, mime_img + + +def send_html(BODY_HTML, SUBJECT, recipient, bcc=None): + BODY_HTML, mime_img = __replace_images(BODY_HTML) + if not isinstance(recipient, list): + recipient = [recipient] + msg = MIMEMultipart() + msg['Subject'] = Header(__get_subject(SUBJECT), 'utf-8') + msg['From'] = environ["EMAIL_FROM"] + msg['To'] = "" + body = MIMEText(BODY_HTML.encode('utf-8'), 'html', "utf-8") + msg.attach(body) + for m in mime_img: + msg.attach(m) + + with smtp.SMTPClient() as s: + for r in recipient: + msg.replace_header("To", r) + r = [r] + if bcc is not None and len(bcc) > 0: + r += [bcc] + try: + print(f"Email sending to: {r}") + s.sendmail(msg['FROM'], r, msg.as_string().encode('ascii')) + except Exception as e: + print("!!! Email error!") + print(e) + + +def send_text(recipients, text, subject): + with smtp.SMTPClient() as s: + msg = MIMEMultipart() + msg['Subject'] = Header(__get_subject(subject), 'utf-8') + msg['From'] = environ["EMAIL_FROM"] + msg['To'] = ", ".join(recipients) + body = MIMEText(text) + msg.attach(body) + try: + s.sendmail(msg['FROM'], recipients, msg.as_string().encode('ascii')) + except Exception as e: + print("!! Text-email failed: " + subject), + print(e) + + +def __escape_text_html(text): + return text.replace("@", "@").replace(".", ".").replace("=", "=") diff --git a/api/chalicelib/utils/email_helper.py b/api/chalicelib/utils/email_helper.py new file mode 100644 index 000000000..3e5565f26 --- /dev/null +++ b/api/chalicelib/utils/email_helper.py @@ -0,0 +1,119 @@ +from chalicelib.utils.TimeUTC import TimeUTC +from chalicelib.utils.email_handler import __get_html_from_file, send_html, __escape_text_html + + +def send_team_invitation(recipient, user_name, temp_password, client_id, sender_name): + BODY_HTML = __get_html_from_file("chalicelib/utils/html/invitation.html", + formatting_variables={"userName": __escape_text_html(user_name), + "password": temp_password, "clientId": client_id, + "sender": sender_name}) + SUBJECT = "Welcome to Asayer" + send_html(BODY_HTML, SUBJECT, recipient) + + +def send_reset_code(recipient, reset_code): + BODY_HTML = __get_html_from_file("chalicelib/utils/html/reset_password.html", + formatting_variables={"code": reset_code}) + SUBJECT = "Password recovery" + send_html(BODY_HTML, SUBJECT, recipient) + + +def send_assign_session(recipient, message, link): + BODY_HTML = __get_html_from_file("chalicelib/utils/html/assignment.html", + formatting_variables={"message": message, + "now": TimeUTC.to_human_readable(TimeUTC.now()), + "link": link}) + SUBJECT = "assigned session" + send_html(BODY_HTML, SUBJECT, recipient) + + +def alert_email(recipients, subject, data): + BODY_HTML = __get_html_from_file("chalicelib/utils/html/alert_notification.html", formatting_variables=data) + send_html(BODY_HTML=BODY_HTML, SUBJECT=subject, recipient=recipients) + + +def __get_color(idx): + return "#3EAAAF" if idx == 0 else "#77C3C7" if idx == 1 else "#9ED4D7" if idx == 2 else "#99d59a" + + +def weekly_report2(recipients, data): + data["o_tr_u"] = "" + data["o_tr_d"] = "" + for d in data["days_partition"]: + data[ + "o_tr_u"] += f""" + + + + + +
 
+ """ + data[ + "o_tr_d"] += f"""{d["day_short"]}""" + + data[ + "past_week_issues_status"] = f' 0 else "decrease"}.png" width="15px" height="10px" style="font-weight:300;vertical-align:middle">' + data["week_decision"] = "More" if data["past_week_issues_evolution"] > 0 else "Fewer" + data["past_week_issues_evolution"] = abs(data["past_week_issues_evolution"]) + data[ + "past_month_issues_status"] = f' 0 else "decrease"}.png" width="15px" height="10px" style="font-weight:300;vertical-align:middle">' + data["month_decision"] = "More" if data["past_month_issues_evolution"] > 0 else "Fewer" + data["past_month_issues_evolution"] = abs(data["past_month_issues_evolution"]) + data["progress_legend"] = [] + data["progress_tr"] = "" + for idx, i in enumerate(data["issues_by_type"]): + color = __get_color(idx) + data["progress_legend"].append( + f""" + {i["count"]}{i["type"]} + """) + data[ + "progress_tr"] += f' ' + + data["progress_legend"] = '' + "".join( + data["progress_legend"]) + "" + data["breakdown_list"] = "" + color_breakdown = {} + data["breakdown_list_other"] = "" + for idx, i in enumerate(data["issues_breakdown_list"]): + if idx < len(data["issues_breakdown_list"]) - 1 or i["type"].lower() != "others": + color = __get_color(idx) + color_breakdown[i["type"]] = color + data["breakdown_list"] += f""" +   {i["type"]} + {i["sessions_count"]} + 0 else "decrease"}.png" width="10px" height="7px" style="font-weight:300;vertical-align:middle;margin-right: 3px;"> {abs(i["last_week_sessions_evolution"])}% + 0 else "decrease"}.png" width="10px" height="7px" style="font-weight:300;vertical-align:middle;margin-right: 3px;"> {abs(i["last_month_sessions_evolution"])}% + """ + else: + data["breakdown_list_other"] = f""" + +   {i["type"]} + {i["sessions_count"]} + 0 else "decrease"}.png" width="10px" height="7px" style="font-weight:300;vertical-align:middle;margin-right: 3px;"> {abs(i["last_week_sessions_evolution"])}% + 0 else "decrease"}.png" width="10px" height="7px" style="font-weight:300;vertical-align:middle;margin-right: 3px;"> {abs(i["last_month_sessions_evolution"])}% + + """ + data["b_tr_u"] = "" + data["b_tr_d"] = "" + for i in data["issues_breakdown_by_day"]: + data[ + "b_tr_d"] += f""" + {i["day_short"]} + """ + if len(i["partition"]) > 0: + sup_partition = "" + for j in i["partition"]: + sup_partition += f'' + else: + sup_partition = '' + data[ + "b_tr_u"] += f""" + + {sup_partition} +
+ """ + BODY_HTML = __get_html_from_file("chalicelib/utils/html/Project-Weekly-Report.html", formatting_variables=data) + SUBJECT = "Asayer Project Weekly Report" + send_html(BODY_HTML=BODY_HTML, SUBJECT=SUBJECT, recipient=recipients) diff --git a/api/chalicelib/utils/event_filter_definition.py b/api/chalicelib/utils/event_filter_definition.py new file mode 100644 index 000000000..4c132cb13 --- /dev/null +++ b/api/chalicelib/utils/event_filter_definition.py @@ -0,0 +1,15 @@ +class Event: + def __init__(self, ui_type, table, column): + self.ui_type = ui_type + self.table = table + self.column = column + + +class SupportedFilter: + def __init__(self, get, query, value_limit, starts_with, starts_limit, ignore_if_starts_with): + self.get = get + self.query = query + self.valueLimit = value_limit + self.startsWith = starts_with + self.startsLimit = starts_limit + self.ignoreIfStartsWith = ignore_if_starts_with diff --git a/api/chalicelib/utils/github_client_v3.py b/api/chalicelib/utils/github_client_v3.py new file mode 100644 index 000000000..a86b721aa --- /dev/null +++ b/api/chalicelib/utils/github_client_v3.py @@ -0,0 +1,133 @@ +import requests +from datetime import datetime + + +class github_formatters: + + @staticmethod + def get_timestamp(ts): + ts = ts[:-1] + pattern = '%Y-%m-%dT%H:%M:%S' + creation = datetime.strptime(ts, pattern) + return int(creation.timestamp() * 1000) + + @staticmethod + def label(label): + return { + 'id': label["id"], + 'name': label["name"], + 'description': label["description"], + 'color': label["color"] + } + + @staticmethod + def comment(comment): + return { + 'id': str(comment["id"]), + 'message': comment["body"], + 'author': str(github_formatters.user(comment["user"])["id"]), + 'createdAt': github_formatters.get_timestamp(comment["created_at"]) + } + + @staticmethod + def issue(issue): + labels = [github_formatters.label(l) for l in issue["labels"]] + result = { + 'id': str(issue["number"]), + 'creator': str(github_formatters.user(issue["user"])["id"]), + 'assignees': [str(github_formatters.user(a)["id"]) for a in issue["assignees"]], + 'title': issue["title"], + 'description': issue["body"], + 'status': issue["state"], + 'createdAt': github_formatters.get_timestamp(issue["created_at"]), + 'closed': issue["closed_at"] is not None, + 'commentsCount': issue["comments"], + 'issueType': [str(l["id"]) for l in labels if l["name"].lower() != "asayer"], + 'labels': [l["name"] for l in labels] + } + return result + + @staticmethod + def user(user): + if not user: + return None + result = { + 'id': user["id"], + 'name': user["login"], + 'avatarUrls': {'24x24': user["avatar_url"]}, + 'email': "" + } + return result + + @staticmethod + def team_to_dict(team): + if not team: + return None + + result = {'id': team.id, 'name': team.name, 'members_count': team.members_count} + return result + + @staticmethod + def repo(repo): + if not repo: + return None + return { + "id": str(repo["id"]), + "name": repo["name"], + "description": repo["description"], + "creator": str(repo["owner"]["id"]) + } + + @staticmethod + def organization(org): + if not org: + return None + return { + "id": org["id"], + "name": org["login"], + "description": org["description"], + "avatarUrls": {'24x42': org["avatar_url"]} + } + + +def get_response_links(response): + links = {} + if "Link" in response.headers: + link_headers = response.headers["Link"].split(", ") + for link_header in link_headers: + (url, rel) = link_header.split("; ") + url = url[1:-1] + rel = rel[5:-1] + links[rel] = url + return links + + +class githubV3Request: + __base = "https://api.github.com" + + def __init__(self, token): + self.__token = token + + def __get_request_header(self): + return {"Accept": "application/vnd.github.v3+json", 'Authorization': f'token {self.__token}'} + + def get(self, url, params={}): + results = [] + params = {"per_page": 100, **params} + pages = {"next": f"{self.__base}{url}", "last": ""} + while len(pages.keys()) > 0 and pages["next"] != pages["last"]: + response = requests.get(pages["next"], headers=self.__get_request_header(), params=params) + pages = get_response_links(response) + result = response.json() + if response.status_code != 200: + print("!-------- error") + print(result) + raise Exception(result["message"]) + if isinstance(result, dict): + return result + results += result + return results + + def post(self, url, body): + response = requests.post(f"{self.__base}{url}", headers=self.__get_request_header(), json=body) + return response.json() diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py new file mode 100644 index 000000000..5bb809dd8 --- /dev/null +++ b/api/chalicelib/utils/helper.py @@ -0,0 +1,380 @@ +import math +import random +import re +import string + +import requests + +local_prefix = 'local-' +from os import environ, path + +import json + + +def get_version_number(): + return environ["version"] + + +def get_stage_name(): + stage = environ["stage"] + return stage[len(local_prefix):] if stage.startswith(local_prefix) else stage + + +def is_production(): + return get_stage_name() == "production" + + +def is_staging(): + return get_stage_name() == "staging" + + +def is_onprem(): + return not is_production() and not is_staging() + + +def is_local(): + return environ["stage"].startswith(local_prefix) + + +def generate_salt(): + return "".join(random.choices(string.hexdigits, k=36)) + + +def remove_empty_none_values(dictionary): + aux = {} + for key in dictionary.keys(): + if dictionary[key] is not None: + if isinstance(dictionary[key], dict): + aux[key] = remove_empty_none_values(dictionary[key]) + elif not isinstance(dictionary[key], str) or len(dictionary[key]) > 0: + aux[key] = dictionary[key] + return aux + + +def unique_ordered_list(array): + uniq = [] + [uniq.append(x) for x in array if x not in uniq] + return uniq + + +def unique_unordered_list(array): + return list(set(array)) + + +def list_to_camel_case(items, flatten=False): + for i in range(len(items)): + if flatten: + items[i] = flatten_nested_dicts(items[i]) + items[i] = dict_to_camel_case(items[i]) + + return items + + +def dict_to_camel_case(variable, delimiter='_', ignore_keys=[]): + if variable is None: + return None + if isinstance(variable, str): + return variable + elif isinstance(variable, dict): + aux = {} + for key in variable.keys(): + if key in ignore_keys: + aux[key] = variable[key] + elif isinstance(variable[key], dict): + aux[key_to_camel_case(key, delimiter)] = dict_to_camel_case(variable[key]) + elif isinstance(variable[key], list): + aux[key_to_camel_case(key, delimiter)] = list_to_camel_case(variable[key]) + else: + aux[key_to_camel_case(key, delimiter)] = variable[key] + return aux + else: + return variable + + +def dict_to_CAPITAL_keys(variable): + if variable is None: + return None + if isinstance(variable, str): + return variable.upper() + elif isinstance(variable, dict): + aux = {} + for key in variable.keys(): + if isinstance(variable[key], dict): + aux[key.upper()] = dict_to_CAPITAL_keys(variable[key]) + else: + aux[key.upper()] = variable[key] + return aux + else: + return variable + + +def variable_to_snake_case(variable, delimiter='_', split_number=False): + if isinstance(variable, str): + return key_to_snake_case(variable, delimiter, split_number) + elif isinstance(variable, dict): + aux = {} + for key in variable.keys(): + if isinstance(variable[key], dict): + aux[key_to_snake_case(key, delimiter, split_number)] = variable_to_snake_case(variable[key], delimiter, + split_number) + else: + aux[key_to_snake_case(key, delimiter, split_number)] = variable[key] + return aux + else: + return variable + + +def key_to_camel_case(snake_str, delimiter='_'): + if snake_str.startswith(delimiter): + snake_str = snake_str[1:] + components = snake_str.split(delimiter) + return components[0] + ''.join(x.title() for x in components[1:]) + + +def key_to_snake_case(name, delimiter='_', split_number=False): + s1 = re.sub('(.)([A-Z][a-z]+)', fr'\1{delimiter}\2', name) + return re.sub('([a-z])([A-Z0-9])' if split_number else '([a-z0-9])([A-Z])', fr'\1{delimiter}\2', s1).lower() + + +TRACK_TIME = True + + +def __sbool_to_bool(value): + if value is None or not isinstance(value, str): + return False + return value.lower() in ["true", "yes", "1"] + + +def allow_cron(): + return "allowCron" not in environ or __sbool_to_bool(environ["allowCron"]) + + +def allow_captcha(): + return environ.get("captcha_server") is not None and environ.get("captcha_key") is not None \ + and len(environ["captcha_server"]) > 0 and len(environ["captcha_key"]) > 0 + + +def allow_sentry(): + return "sentry" not in environ or __sbool_to_bool(environ["sentry"]) + + +def async_post(endpoint, data): + data["auth"] = environ["async_Token"] + try: + requests.post(endpoint, timeout=1, json=data) + except requests.exceptions.ReadTimeout: + pass + + +def string_to_sql_like(value): + value = re.sub(' +', ' ', value) + value = value.replace("*", "%") + if value.startswith("^"): + value = value[1:] + elif not value.startswith("%"): + value = '%' + value + + if value.endswith("$"): + value = value[:-1] + elif not value.endswith("%"): + value = value + '%' + # value = value.replace(" ", "%") + return value + + +def string_to_sql_like_with_op(value, op): + if isinstance(value, list) and len(value) > 0: + _value = value[0] + else: + _value = value + if _value is None: + return _value + if op.lower() != 'ilike': + return _value.replace("%", "%%") + _value = _value.replace("*", "%") + if _value.startswith("^"): + _value = _value[1:] + elif not _value.startswith("%"): + _value = '%' + _value + + if _value.endswith("$"): + _value = _value[:-1] + elif not _value.endswith("%"): + _value = _value + '%' + return _value.replace("%", "%%") + + +def is_valid_email(email): + return re.match(r"[^@]+@[^@]+\.[^@]+", email) is not None + + +def is_valid_http_url(url): + regex = re.compile( + r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + + return re.match(regex, url) is not None + + +def is_valid_url(url): + regex = re.compile( + # r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + + return re.match(regex, url) is not None + + +def is_alphabet_space(word): + r = re.compile("^[a-zA-Z ]*$") + return r.match(word) is not None + + +def is_alphabet_latin_space(word): + r = re.compile("^[a-zA-Z\u00C0-\u00D6\u00D8-\u00f6\u00f8-\u00ff\s ]*$") + return r.match(word) is not None + + +def is_alphabet_space_dash(word): + r = re.compile("^[a-zA-Z -]*$") + return r.match(word) is not None + + +def is_alphanumeric_space(word): + r = re.compile("^[a-zA-Z0-9._\- ]*$") + return r.match(word) is not None + + +def merge_lists_by_key(l1, l2, key): + merged = {} + for item in l1 + l2: + if item[key] in merged: + merged[item[key]].update(item) + else: + merged[item[key]] = item + return [val for (_, val) in merged.items()] + + +def flatten_nested_dicts(obj): + if obj is None: + return None + result = {} + for key in obj.keys(): + if isinstance(obj[key], dict): + result = {**result, **flatten_nested_dicts(obj[key])} + else: + result[key] = obj[key] + return result + + +def delete_keys_from_dict(d, to_delete): + if isinstance(to_delete, str): + to_delete = [to_delete] + if isinstance(d, dict): + for single_to_delete in set(to_delete): + if single_to_delete in d: + del d[single_to_delete] + for k, v in d.items(): + delete_keys_from_dict(v, to_delete) + elif isinstance(d, list): + for i in d: + delete_keys_from_dict(i, to_delete) + return d + + +def explode_widget(data, key=None): + result = [] + for k in data.keys(): + if k.endswith("Progress") or k == "chart": + continue + result.append({"key": key_to_snake_case(k) if key is None else key, "data": {"value": data[k]}}) + if k + "Progress" in data: + result[-1]["data"]["progress"] = data[k + "Progress"] + if "chart" in data: + result[-1]["data"]["chart"] = [] + for c in data["chart"]: + result[-1]["data"]["chart"].append({"timestamp": c["timestamp"], "value": c[k]}) + return result + + +TEMP_PATH = "./" if is_local() else "/tmp/" + + +def get_issue_title(issue_type): + return {'click_rage': "Click Rage", + 'dead_click': "Dead Click", + 'excessive_scrolling': "Excessive Scrolling", + 'bad_request': "Bad Request", + 'missing_resource': "Missing Image", + 'memory': "High Memory Usage", + 'cpu': "High CPU", + 'slow_resource': "Slow Resource", + 'slow_page_load': "Slow Page", + 'crash': "Crash", + 'ml_cpu': "High CPU", + 'ml_memory': "High Memory Usage", + 'ml_dead_click': "Dead Click", + 'ml_click_rage': "Click Rage", + 'ml_mouse_thrashing': "Mouse Thrashing", + 'ml_excessive_scrolling': "Excessive Scrolling", + 'ml_slow_resources': "Slow Resource", + 'custom': "Custom Event", + 'js_exception': "Error", + 'custom_event_error': "Custom Error", + 'js_error': "Error"}.get(issue_type, issue_type) + + +def __progress(old_val, new_val): + return ((old_val - new_val) / new_val) * 100 if new_val > 0 else 0 if old_val == 0 else 100 + + +def __decimal_limit(value, limit): + factor = pow(10, limit) + value = math.floor(value * factor) + if value % factor == 0: + return value // factor + return value / factor + + +def is_free_open_source_edition(): + return __sbool_to_bool(environ.get("isFOS")) + + +def is_enterprise_edition(): + return __sbool_to_bool(environ.get("isEE")) + + +stag_config_file = f"chalicelib/.configs/{environ['stage']}.json" +if not path.isfile(stag_config_file): + print("!! stage config file not found, using .chalice/config.json only") +else: + print("!! stage config file found, merging with priority to .chalice/config.json") + with open(stag_config_file) as json_file: + config = json.load(json_file) + environ = {**config, **environ} + +if (is_free_open_source_edition() or is_enterprise_edition()) and environ.get("config_file"): + if not path.isfile(environ.get("config_file")): + print("!! config file not found, using default environment") + else: + with open(environ.get("config_file")) as json_file: + config = json.load(json_file) + environ = {**environ, **config} + + +def get_internal_project_id(project_id64): + if project_id64 < 0x10000000000000 or project_id64 >= 0x20000000000000: + return None + + project_id64 = (project_id64 - 0x10000000000000) * 4212451012670231 & 0xfffffffffffff + if project_id64 > 0xffffffff: + return None + project_id = int(project_id64) + return project_id diff --git a/api/chalicelib/utils/html/Project-Weekly-Report.html b/api/chalicelib/utils/html/Project-Weekly-Report.html new file mode 100644 index 000000000..bb7b7a38e --- /dev/null +++ b/api/chalicelib/utils/html/Project-Weekly-Report.html @@ -0,0 +1,170 @@ + + +
+
+ + + + + + + +
+
+
+ + + + + + +
+ + + + + + +
%(project_name)s | Weekly Report
%(period_start)s - %(period_end)s
+
+ +
+
+
+ + +
+ +
+ + + + + +
+

287 Issues Occured This Week

+
+ + + + + + +
+ + + %(o_tr_u)s + + + %(o_tr_d)s + +
+ 		+ + + + + + + + + +
+
%(past_week_issues_status)s %(past_week_issues_evolution)s%
+ 		+
%(past_month_issues_status)s %(past_month_issues_evolution)s%
+
+  %(week_decision)s than past 7 days + + %(month_decision)s than past 30 days +
+
+ +
+ +
+ +
+ + + + + + + + + + + + +
+

Issues by Type

+
+ + + %(progress_legend)s + +
+
+ + + %(progress_tr)s + +
+
+ + +
+ +
+ +
+ + + +

Issues Breakdown

+ + + + %(b_tr_u)s + + + %(b_tr_d)s + +
+ + + + + + + + + + + + %(breakdown_list)s + + %(breakdown_list_other)s +
TypeSessionsPast 7 DaysPast 30 Days
+ + +
+ +
+ + +
+ +
+ + \ No newline at end of file diff --git a/api/chalicelib/utils/html/alert_notification.html b/api/chalicelib/utils/html/alert_notification.html new file mode 100644 index 000000000..36f7a7614 --- /dev/null +++ b/api/chalicelib/utils/html/alert_notification.html @@ -0,0 +1,60 @@ + + + + + + + +
+
+ + + + + + + + + + + + + + + + + + +
+
+ Asayer +
+
+

+ New alert!

+

+ %(message)s

+

+ See metrics for more details.

+ + +
+
+
+

+ Sent with ♡ from Asayer © 2021 - All rights reserved.

+ https://asayer.io/ +

+ +
+
+ +
+
+ + + + + diff --git a/api/chalicelib/utils/html/assignment.html b/api/chalicelib/utils/html/assignment.html new file mode 100644 index 000000000..a6eacbfbf --- /dev/null +++ b/api/chalicelib/utils/html/assignment.html @@ -0,0 +1,556 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/api/chalicelib/utils/html/img/asayer-logo.png b/api/chalicelib/utils/html/img/asayer-logo.png new file mode 100644 index 000000000..56b14d369 Binary files /dev/null and b/api/chalicelib/utils/html/img/asayer-logo.png differ diff --git a/api/chalicelib/utils/html/img/header.png b/api/chalicelib/utils/html/img/header.png new file mode 100644 index 000000000..0108751f4 Binary files /dev/null and b/api/chalicelib/utils/html/img/header.png differ diff --git a/api/chalicelib/utils/html/img/icn-facebook.png b/api/chalicelib/utils/html/img/icn-facebook.png new file mode 100644 index 000000000..5df55e55f Binary files /dev/null and b/api/chalicelib/utils/html/img/icn-facebook.png differ diff --git a/api/chalicelib/utils/html/img/icn-linkedin.png b/api/chalicelib/utils/html/img/icn-linkedin.png new file mode 100644 index 000000000..7de49669c Binary files /dev/null and b/api/chalicelib/utils/html/img/icn-linkedin.png differ diff --git a/api/chalicelib/utils/html/img/icn-medium.png b/api/chalicelib/utils/html/img/icn-medium.png new file mode 100644 index 000000000..dfaafcbb2 Binary files /dev/null and b/api/chalicelib/utils/html/img/icn-medium.png differ diff --git a/api/chalicelib/utils/html/img/icn-twitter.png b/api/chalicelib/utils/html/img/icn-twitter.png new file mode 100644 index 000000000..a74363513 Binary files /dev/null and b/api/chalicelib/utils/html/img/icn-twitter.png differ diff --git a/api/chalicelib/utils/html/img/illustrations/img-no-signal.png b/api/chalicelib/utils/html/img/illustrations/img-no-signal.png new file mode 100644 index 000000000..9723375d8 Binary files /dev/null and b/api/chalicelib/utils/html/img/illustrations/img-no-signal.png differ diff --git a/api/chalicelib/utils/html/img/illustrations/img-welcome.png b/api/chalicelib/utils/html/img/illustrations/img-welcome.png new file mode 100644 index 000000000..6239ced39 Binary files /dev/null and b/api/chalicelib/utils/html/img/illustrations/img-welcome.png differ diff --git a/api/chalicelib/utils/html/img/laptop.png b/api/chalicelib/utils/html/img/laptop.png new file mode 100644 index 000000000..a54d0df7f Binary files /dev/null and b/api/chalicelib/utils/html/img/laptop.png differ diff --git a/api/chalicelib/utils/html/img/weekly/arrow-decrease.png b/api/chalicelib/utils/html/img/weekly/arrow-decrease.png new file mode 100644 index 000000000..7264a1eec Binary files /dev/null and b/api/chalicelib/utils/html/img/weekly/arrow-decrease.png differ diff --git a/api/chalicelib/utils/html/img/weekly/arrow-increase.png b/api/chalicelib/utils/html/img/weekly/arrow-increase.png new file mode 100644 index 000000000..175d8db5b Binary files /dev/null and b/api/chalicelib/utils/html/img/weekly/arrow-increase.png differ diff --git a/api/chalicelib/utils/html/img/weekly/asayer-logo.png b/api/chalicelib/utils/html/img/weekly/asayer-logo.png new file mode 100644 index 000000000..b7440013c Binary files /dev/null and b/api/chalicelib/utils/html/img/weekly/asayer-logo.png differ diff --git a/api/chalicelib/utils/html/invitation.html b/api/chalicelib/utils/html/invitation.html new file mode 100644 index 000000000..9c8c1448f --- /dev/null +++ b/api/chalicelib/utils/html/invitation.html @@ -0,0 +1,595 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/api/chalicelib/utils/html/reset_password.html b/api/chalicelib/utils/html/reset_password.html new file mode 100644 index 000000000..68bb21e63 --- /dev/null +++ b/api/chalicelib/utils/html/reset_password.html @@ -0,0 +1,545 @@ + + + + + + + + + + + + + + + + + + + + + + + """ + data[ + "o_tr_d"] += f"""""" + + data[ + "past_week_issues_status"] = f' 0 else "decrease"}.png" width="15px" height="10px" style="font-weight:300;vertical-align:middle">' + data["week_decision"] = "More" if data["past_week_issues_evolution"] > 0 else "Fewer" + data["past_week_issues_evolution"] = abs(data["past_week_issues_evolution"]) + data[ + "past_month_issues_status"] = f' 0 else "decrease"}.png" width="15px" height="10px" style="font-weight:300;vertical-align:middle">' + data["month_decision"] = "More" if data["past_month_issues_evolution"] > 0 else "Fewer" + data["past_month_issues_evolution"] = abs(data["past_month_issues_evolution"]) + data["progress_legend"] = [] + data["progress_tr"] = "" + for idx, i in enumerate(data["issues_by_type"]): + color = __get_color(idx) + data["progress_legend"].append( + f"""""") + data[ + "progress_tr"] += f'' + + data["progress_legend"] = '' + "".join( + data["progress_legend"]) + "" + data["breakdown_list"] = "" + color_breakdown = {} + data["breakdown_list_other"] = "" + for idx, i in enumerate(data["issues_breakdown_list"]): + if idx < len(data["issues_breakdown_list"]) - 1 or i["type"].lower() != "others": + color = __get_color(idx) + color_breakdown[i["type"]] = color + data["breakdown_list"] += f""" + + + + + """ + else: + data["breakdown_list_other"] = f""" + + + + + + + """ + data["b_tr_u"] = "" + data["b_tr_d"] = "" + for i in data["issues_breakdown_by_day"]: + data[ + "b_tr_d"] += f"""""" + if len(i["partition"]) > 0: + sup_partition = "" + for j in i["partition"]: + sup_partition += f'' + else: + sup_partition = '' + data[ + "b_tr_u"] += f"""""" + BODY_HTML = __get_html_from_file("chalicelib/utils/html/Project-Weekly-Report.html", formatting_variables=data) + SUBJECT = "Asayer Project Weekly Report" + send_html(BODY_HTML=BODY_HTML, SUBJECT=SUBJECT, recipient=recipients) diff --git a/ee/api/chalicelib/utils/event_filter_definition.py b/ee/api/chalicelib/utils/event_filter_definition.py new file mode 100644 index 000000000..4c132cb13 --- /dev/null +++ b/ee/api/chalicelib/utils/event_filter_definition.py @@ -0,0 +1,15 @@ +class Event: + def __init__(self, ui_type, table, column): + self.ui_type = ui_type + self.table = table + self.column = column + + +class SupportedFilter: + def __init__(self, get, query, value_limit, starts_with, starts_limit, ignore_if_starts_with): + self.get = get + self.query = query + self.valueLimit = value_limit + self.startsWith = starts_with + self.startsLimit = starts_limit + self.ignoreIfStartsWith = ignore_if_starts_with diff --git a/ee/api/chalicelib/utils/github_client_v3.py b/ee/api/chalicelib/utils/github_client_v3.py new file mode 100644 index 000000000..a86b721aa --- /dev/null +++ b/ee/api/chalicelib/utils/github_client_v3.py @@ -0,0 +1,133 @@ +import requests +from datetime import datetime + + +class github_formatters: + + @staticmethod + def get_timestamp(ts): + ts = ts[:-1] + pattern = '%Y-%m-%dT%H:%M:%S' + creation = datetime.strptime(ts, pattern) + return int(creation.timestamp() * 1000) + + @staticmethod + def label(label): + return { + 'id': label["id"], + 'name': label["name"], + 'description': label["description"], + 'color': label["color"] + } + + @staticmethod + def comment(comment): + return { + 'id': str(comment["id"]), + 'message': comment["body"], + 'author': str(github_formatters.user(comment["user"])["id"]), + 'createdAt': github_formatters.get_timestamp(comment["created_at"]) + } + + @staticmethod + def issue(issue): + labels = [github_formatters.label(l) for l in issue["labels"]] + result = { + 'id': str(issue["number"]), + 'creator': str(github_formatters.user(issue["user"])["id"]), + 'assignees': [str(github_formatters.user(a)["id"]) for a in issue["assignees"]], + 'title': issue["title"], + 'description': issue["body"], + 'status': issue["state"], + 'createdAt': github_formatters.get_timestamp(issue["created_at"]), + 'closed': issue["closed_at"] is not None, + 'commentsCount': issue["comments"], + 'issueType': [str(l["id"]) for l in labels if l["name"].lower() != "asayer"], + 'labels': [l["name"] for l in labels] + } + return result + + @staticmethod + def user(user): + if not user: + return None + result = { + 'id': user["id"], + 'name': user["login"], + 'avatarUrls': {'24x24': user["avatar_url"]}, + 'email': "" + } + return result + + @staticmethod + def team_to_dict(team): + if not team: + return None + + result = {'id': team.id, 'name': team.name, 'members_count': team.members_count} + return result + + @staticmethod + def repo(repo): + if not repo: + return None + return { + "id": str(repo["id"]), + "name": repo["name"], + "description": repo["description"], + "creator": str(repo["owner"]["id"]) + } + + @staticmethod + def organization(org): + if not org: + return None + return { + "id": org["id"], + "name": org["login"], + "description": org["description"], + "avatarUrls": {'24x42': org["avatar_url"]} + } + + +def get_response_links(response): + links = {} + if "Link" in response.headers: + link_headers = response.headers["Link"].split(", ") + for link_header in link_headers: + (url, rel) = link_header.split("; ") + url = url[1:-1] + rel = rel[5:-1] + links[rel] = url + return links + + +class githubV3Request: + __base = "https://api.github.com" + + def __init__(self, token): + self.__token = token + + def __get_request_header(self): + return {"Accept": "application/vnd.github.v3+json", 'Authorization': f'token {self.__token}'} + + def get(self, url, params={}): + results = [] + params = {"per_page": 100, **params} + pages = {"next": f"{self.__base}{url}", "last": ""} + while len(pages.keys()) > 0 and pages["next"] != pages["last"]: + response = requests.get(pages["next"], headers=self.__get_request_header(), params=params) + pages = get_response_links(response) + result = response.json() + if response.status_code != 200: + print("!-------- error") + print(result) + raise Exception(result["message"]) + if isinstance(result, dict): + return result + results += result + return results + + def post(self, url, body): + response = requests.post(f"{self.__base}{url}", headers=self.__get_request_header(), json=body) + return response.json() diff --git a/ee/api/chalicelib/utils/helper.py b/ee/api/chalicelib/utils/helper.py new file mode 100644 index 000000000..5bb809dd8 --- /dev/null +++ b/ee/api/chalicelib/utils/helper.py @@ -0,0 +1,380 @@ +import math +import random +import re +import string + +import requests + +local_prefix = 'local-' +from os import environ, path + +import json + + +def get_version_number(): + return environ["version"] + + +def get_stage_name(): + stage = environ["stage"] + return stage[len(local_prefix):] if stage.startswith(local_prefix) else stage + + +def is_production(): + return get_stage_name() == "production" + + +def is_staging(): + return get_stage_name() == "staging" + + +def is_onprem(): + return not is_production() and not is_staging() + + +def is_local(): + return environ["stage"].startswith(local_prefix) + + +def generate_salt(): + return "".join(random.choices(string.hexdigits, k=36)) + + +def remove_empty_none_values(dictionary): + aux = {} + for key in dictionary.keys(): + if dictionary[key] is not None: + if isinstance(dictionary[key], dict): + aux[key] = remove_empty_none_values(dictionary[key]) + elif not isinstance(dictionary[key], str) or len(dictionary[key]) > 0: + aux[key] = dictionary[key] + return aux + + +def unique_ordered_list(array): + uniq = [] + [uniq.append(x) for x in array if x not in uniq] + return uniq + + +def unique_unordered_list(array): + return list(set(array)) + + +def list_to_camel_case(items, flatten=False): + for i in range(len(items)): + if flatten: + items[i] = flatten_nested_dicts(items[i]) + items[i] = dict_to_camel_case(items[i]) + + return items + + +def dict_to_camel_case(variable, delimiter='_', ignore_keys=[]): + if variable is None: + return None + if isinstance(variable, str): + return variable + elif isinstance(variable, dict): + aux = {} + for key in variable.keys(): + if key in ignore_keys: + aux[key] = variable[key] + elif isinstance(variable[key], dict): + aux[key_to_camel_case(key, delimiter)] = dict_to_camel_case(variable[key]) + elif isinstance(variable[key], list): + aux[key_to_camel_case(key, delimiter)] = list_to_camel_case(variable[key]) + else: + aux[key_to_camel_case(key, delimiter)] = variable[key] + return aux + else: + return variable + + +def dict_to_CAPITAL_keys(variable): + if variable is None: + return None + if isinstance(variable, str): + return variable.upper() + elif isinstance(variable, dict): + aux = {} + for key in variable.keys(): + if isinstance(variable[key], dict): + aux[key.upper()] = dict_to_CAPITAL_keys(variable[key]) + else: + aux[key.upper()] = variable[key] + return aux + else: + return variable + + +def variable_to_snake_case(variable, delimiter='_', split_number=False): + if isinstance(variable, str): + return key_to_snake_case(variable, delimiter, split_number) + elif isinstance(variable, dict): + aux = {} + for key in variable.keys(): + if isinstance(variable[key], dict): + aux[key_to_snake_case(key, delimiter, split_number)] = variable_to_snake_case(variable[key], delimiter, + split_number) + else: + aux[key_to_snake_case(key, delimiter, split_number)] = variable[key] + return aux + else: + return variable + + +def key_to_camel_case(snake_str, delimiter='_'): + if snake_str.startswith(delimiter): + snake_str = snake_str[1:] + components = snake_str.split(delimiter) + return components[0] + ''.join(x.title() for x in components[1:]) + + +def key_to_snake_case(name, delimiter='_', split_number=False): + s1 = re.sub('(.)([A-Z][a-z]+)', fr'\1{delimiter}\2', name) + return re.sub('([a-z])([A-Z0-9])' if split_number else '([a-z0-9])([A-Z])', fr'\1{delimiter}\2', s1).lower() + + +TRACK_TIME = True + + +def __sbool_to_bool(value): + if value is None or not isinstance(value, str): + return False + return value.lower() in ["true", "yes", "1"] + + +def allow_cron(): + return "allowCron" not in environ or __sbool_to_bool(environ["allowCron"]) + + +def allow_captcha(): + return environ.get("captcha_server") is not None and environ.get("captcha_key") is not None \ + and len(environ["captcha_server"]) > 0 and len(environ["captcha_key"]) > 0 + + +def allow_sentry(): + return "sentry" not in environ or __sbool_to_bool(environ["sentry"]) + + +def async_post(endpoint, data): + data["auth"] = environ["async_Token"] + try: + requests.post(endpoint, timeout=1, json=data) + except requests.exceptions.ReadTimeout: + pass + + +def string_to_sql_like(value): + value = re.sub(' +', ' ', value) + value = value.replace("*", "%") + if value.startswith("^"): + value = value[1:] + elif not value.startswith("%"): + value = '%' + value + + if value.endswith("$"): + value = value[:-1] + elif not value.endswith("%"): + value = value + '%' + # value = value.replace(" ", "%") + return value + + +def string_to_sql_like_with_op(value, op): + if isinstance(value, list) and len(value) > 0: + _value = value[0] + else: + _value = value + if _value is None: + return _value + if op.lower() != 'ilike': + return _value.replace("%", "%%") + _value = _value.replace("*", "%") + if _value.startswith("^"): + _value = _value[1:] + elif not _value.startswith("%"): + _value = '%' + _value + + if _value.endswith("$"): + _value = _value[:-1] + elif not _value.endswith("%"): + _value = _value + '%' + return _value.replace("%", "%%") + + +def is_valid_email(email): + return re.match(r"[^@]+@[^@]+\.[^@]+", email) is not None + + +def is_valid_http_url(url): + regex = re.compile( + r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + + return re.match(regex, url) is not None + + +def is_valid_url(url): + regex = re.compile( + # r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + + return re.match(regex, url) is not None + + +def is_alphabet_space(word): + r = re.compile("^[a-zA-Z ]*$") + return r.match(word) is not None + + +def is_alphabet_latin_space(word): + r = re.compile("^[a-zA-Z\u00C0-\u00D6\u00D8-\u00f6\u00f8-\u00ff\s ]*$") + return r.match(word) is not None + + +def is_alphabet_space_dash(word): + r = re.compile("^[a-zA-Z -]*$") + return r.match(word) is not None + + +def is_alphanumeric_space(word): + r = re.compile("^[a-zA-Z0-9._\- ]*$") + return r.match(word) is not None + + +def merge_lists_by_key(l1, l2, key): + merged = {} + for item in l1 + l2: + if item[key] in merged: + merged[item[key]].update(item) + else: + merged[item[key]] = item + return [val for (_, val) in merged.items()] + + +def flatten_nested_dicts(obj): + if obj is None: + return None + result = {} + for key in obj.keys(): + if isinstance(obj[key], dict): + result = {**result, **flatten_nested_dicts(obj[key])} + else: + result[key] = obj[key] + return result + + +def delete_keys_from_dict(d, to_delete): + if isinstance(to_delete, str): + to_delete = [to_delete] + if isinstance(d, dict): + for single_to_delete in set(to_delete): + if single_to_delete in d: + del d[single_to_delete] + for k, v in d.items(): + delete_keys_from_dict(v, to_delete) + elif isinstance(d, list): + for i in d: + delete_keys_from_dict(i, to_delete) + return d + + +def explode_widget(data, key=None): + result = [] + for k in data.keys(): + if k.endswith("Progress") or k == "chart": + continue + result.append({"key": key_to_snake_case(k) if key is None else key, "data": {"value": data[k]}}) + if k + "Progress" in data: + result[-1]["data"]["progress"] = data[k + "Progress"] + if "chart" in data: + result[-1]["data"]["chart"] = [] + for c in data["chart"]: + result[-1]["data"]["chart"].append({"timestamp": c["timestamp"], "value": c[k]}) + return result + + +TEMP_PATH = "./" if is_local() else "/tmp/" + + +def get_issue_title(issue_type): + return {'click_rage': "Click Rage", + 'dead_click': "Dead Click", + 'excessive_scrolling': "Excessive Scrolling", + 'bad_request': "Bad Request", + 'missing_resource': "Missing Image", + 'memory': "High Memory Usage", + 'cpu': "High CPU", + 'slow_resource': "Slow Resource", + 'slow_page_load': "Slow Page", + 'crash': "Crash", + 'ml_cpu': "High CPU", + 'ml_memory': "High Memory Usage", + 'ml_dead_click': "Dead Click", + 'ml_click_rage': "Click Rage", + 'ml_mouse_thrashing': "Mouse Thrashing", + 'ml_excessive_scrolling': "Excessive Scrolling", + 'ml_slow_resources': "Slow Resource", + 'custom': "Custom Event", + 'js_exception': "Error", + 'custom_event_error': "Custom Error", + 'js_error': "Error"}.get(issue_type, issue_type) + + +def __progress(old_val, new_val): + return ((old_val - new_val) / new_val) * 100 if new_val > 0 else 0 if old_val == 0 else 100 + + +def __decimal_limit(value, limit): + factor = pow(10, limit) + value = math.floor(value * factor) + if value % factor == 0: + return value // factor + return value / factor + + +def is_free_open_source_edition(): + return __sbool_to_bool(environ.get("isFOS")) + + +def is_enterprise_edition(): + return __sbool_to_bool(environ.get("isEE")) + + +stag_config_file = f"chalicelib/.configs/{environ['stage']}.json" +if not path.isfile(stag_config_file): + print("!! stage config file not found, using .chalice/config.json only") +else: + print("!! stage config file found, merging with priority to .chalice/config.json") + with open(stag_config_file) as json_file: + config = json.load(json_file) + environ = {**config, **environ} + +if (is_free_open_source_edition() or is_enterprise_edition()) and environ.get("config_file"): + if not path.isfile(environ.get("config_file")): + print("!! config file not found, using default environment") + else: + with open(environ.get("config_file")) as json_file: + config = json.load(json_file) + environ = {**environ, **config} + + +def get_internal_project_id(project_id64): + if project_id64 < 0x10000000000000 or project_id64 >= 0x20000000000000: + return None + + project_id64 = (project_id64 - 0x10000000000000) * 4212451012670231 & 0xfffffffffffff + if project_id64 > 0xffffffff: + return None + project_id = int(project_id64) + return project_id diff --git a/ee/api/chalicelib/utils/html/Project-Weekly-Report.html b/ee/api/chalicelib/utils/html/Project-Weekly-Report.html new file mode 100644 index 000000000..bb7b7a38e --- /dev/null +++ b/ee/api/chalicelib/utils/html/Project-Weekly-Report.html @@ -0,0 +1,170 @@ + + +
+
+
+ + + + + + +
+
+
+ + + + + + +
+ + + + + + +
%(project_name)s | Weekly Report
%(period_start)s - %(period_end)s
+
+ +
+
+
+ + +
+ +
+ + + + + +
+

287 Issues Occured This Week

+
+ + + + + + +
+ + + %(o_tr_u)s + + + %(o_tr_d)s + +
+ 		+ + + + + + + + + +
+
%(past_week_issues_status)s %(past_week_issues_evolution)s%
+ 		+
%(past_month_issues_status)s %(past_month_issues_evolution)s%
+
+  %(week_decision)s than past 7 days + + %(month_decision)s than past 30 days +
+
+ +
+ +
+ +
+ + + + + + + + + + + + +
+

Issues by Type

+
+ + + %(progress_legend)s + +
+
+ + + %(progress_tr)s + +
+
+ + +
+ +
+ +
+ + + +

Issues Breakdown

+ + + + %(b_tr_u)s + + + %(b_tr_d)s + +
+ + + + + + + + + + + + %(breakdown_list)s + + %(breakdown_list_other)s +
TypeSessionsPast 7 DaysPast 30 Days
+ + +
+ +
+ + +
+ + + + \ No newline at end of file diff --git a/ee/api/chalicelib/utils/html/alert_notification.html b/ee/api/chalicelib/utils/html/alert_notification.html new file mode 100644 index 000000000..36f7a7614 --- /dev/null +++ b/ee/api/chalicelib/utils/html/alert_notification.html @@ -0,0 +1,60 @@ + + + + + + + +
+
+ + + + + + + + + + + + + + + + + + +
+
+ Asayer +
+
+

+ New alert!

+

+ %(message)s

+

+ See metrics for more details.

+ + +
+
+
+

+ Sent with ♡ from Asayer © 2021 - All rights reserved.

+ https://asayer.io/ +

+ +
+
+ +
+
+ + + + + diff --git a/ee/api/chalicelib/utils/html/assignment.html b/ee/api/chalicelib/utils/html/assignment.html new file mode 100644 index 000000000..a6eacbfbf --- /dev/null +++ b/ee/api/chalicelib/utils/html/assignment.html @@ -0,0 +1,556 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ee/api/chalicelib/utils/html/img/asayer-logo.png b/ee/api/chalicelib/utils/html/img/asayer-logo.png new file mode 100644 index 000000000..56b14d369 Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/asayer-logo.png differ diff --git a/ee/api/chalicelib/utils/html/img/header.png b/ee/api/chalicelib/utils/html/img/header.png new file mode 100644 index 000000000..0108751f4 Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/header.png differ diff --git a/ee/api/chalicelib/utils/html/img/icn-facebook.png b/ee/api/chalicelib/utils/html/img/icn-facebook.png new file mode 100644 index 000000000..5df55e55f Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/icn-facebook.png differ diff --git a/ee/api/chalicelib/utils/html/img/icn-linkedin.png b/ee/api/chalicelib/utils/html/img/icn-linkedin.png new file mode 100644 index 000000000..7de49669c Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/icn-linkedin.png differ diff --git a/ee/api/chalicelib/utils/html/img/icn-medium.png b/ee/api/chalicelib/utils/html/img/icn-medium.png new file mode 100644 index 000000000..dfaafcbb2 Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/icn-medium.png differ diff --git a/ee/api/chalicelib/utils/html/img/icn-twitter.png b/ee/api/chalicelib/utils/html/img/icn-twitter.png new file mode 100644 index 000000000..a74363513 Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/icn-twitter.png differ diff --git a/ee/api/chalicelib/utils/html/img/illustrations/img-no-signal.png b/ee/api/chalicelib/utils/html/img/illustrations/img-no-signal.png new file mode 100644 index 000000000..9723375d8 Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/illustrations/img-no-signal.png differ diff --git a/ee/api/chalicelib/utils/html/img/illustrations/img-welcome.png b/ee/api/chalicelib/utils/html/img/illustrations/img-welcome.png new file mode 100644 index 000000000..6239ced39 Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/illustrations/img-welcome.png differ diff --git a/ee/api/chalicelib/utils/html/img/laptop.png b/ee/api/chalicelib/utils/html/img/laptop.png new file mode 100644 index 000000000..a54d0df7f Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/laptop.png differ diff --git a/ee/api/chalicelib/utils/html/img/weekly/arrow-decrease.png b/ee/api/chalicelib/utils/html/img/weekly/arrow-decrease.png new file mode 100644 index 000000000..7264a1eec Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/weekly/arrow-decrease.png differ diff --git a/ee/api/chalicelib/utils/html/img/weekly/arrow-increase.png b/ee/api/chalicelib/utils/html/img/weekly/arrow-increase.png new file mode 100644 index 000000000..175d8db5b Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/weekly/arrow-increase.png differ diff --git a/ee/api/chalicelib/utils/html/img/weekly/asayer-logo.png b/ee/api/chalicelib/utils/html/img/weekly/asayer-logo.png new file mode 100644 index 000000000..b7440013c Binary files /dev/null and b/ee/api/chalicelib/utils/html/img/weekly/asayer-logo.png differ diff --git a/ee/api/chalicelib/utils/html/invitation.html b/ee/api/chalicelib/utils/html/invitation.html new file mode 100644 index 000000000..9c8c1448f --- /dev/null +++ b/ee/api/chalicelib/utils/html/invitation.html @@ -0,0 +1,595 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ee/api/chalicelib/utils/html/reset_password.html b/ee/api/chalicelib/utils/html/reset_password.html new file mode 100644 index 000000000..68bb21e63 --- /dev/null +++ b/ee/api/chalicelib/utils/html/reset_password.html @@ -0,0 +1,545 @@ + + + + + + + + + + + + + + + + + + + + + + +