jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #88 from openreplay/dev

Browse source 
v1.2.0
This commit is contained in:
Shekar Siri 2021-07-14 21:21:27 +05:30 • committed by GitHub
commit 7a4a1cd11f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
224 changed files with 9018 additions and 2367 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/api-ee.yaml vendored
View file

@ -47,7 +47,7 @@ jobs:
sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.EE_MINIO_SECRET_KEY }}\" #g" vars.yaml
sed -i "s#domain_name.*#domain_name: \"foss.openreplay.com\" #g" vars.yaml
sed -i "s#kubeconfig.*#kubeconfig_path: ${KUBECONFIG}#g" vars.yaml
sed -i "s/tag:.*/tag: \"$IMAGE_TAG\"/g" vars.yaml
sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml
bash kube-install.sh --app chalice
env:
DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }}

2
.github/workflows/api.yaml vendored
View file

@ -47,7 +47,7 @@ jobs:
sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.OSS_MINIO_SECRET_KEY }}\" #g" vars.yaml
sed -i "s#domain_name.*#domain_name: \"foss.openreplay.com\" #g" vars.yaml
sed -i "s#kubeconfig.*#kubeconfig_path: ${KUBECONFIG}#g" vars.yaml
sed -i "s/tag:.*/tag: \"$IMAGE_TAG\"/g" app/chalice.yaml
sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml
bash kube-install.sh --app chalice
env:
DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}

64
.github/workflows/utilities.yaml vendored Normal file
View file

@ -0,0 +1,64 @@
# This action will push the utilities changes to aws
on:
push:
branches:
- dev
paths:
- utilities/**
name: Build and Deploy Utilities
jobs:
deploy:
name: Deploy
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
with:
# We need to diff with old commit
# to see which workers got changed.
fetch-depth: 2
- name: Docker login
run: |
docker login ${{ secrets.OSS_REGISTRY_URL }} -u ${{ secrets.OSS_DOCKER_USERNAME }} -p "${{ secrets.OSS_REGISTRY_TOKEN }}"
- uses: azure/k8s-set-context@v1
with:
method: kubeconfig
kubeconfig: ${{ secrets.OSS_KUBECONFIG }} # Use content of kubeconfig in secret.
id: setcontext
- name: Building and Pusing api image
id: build-image
env:
DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}
IMAGE_TAG: ${{ github.sha }}
ENVIRONMENT: staging
run: |
cd utilities
PUSH_IMAGE=1 bash build.sh
- name: Deploy to kubernetes
run: |
cd scripts/helm/
sed -i "s#minio_access_key.*#minio_access_key: \"${{ secrets.OSS_MINIO_ACCESS_KEY }}\" #g" vars.yaml
sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.OSS_MINIO_SECRET_KEY }}\" #g" vars.yaml
sed -i "s#domain_name.*#domain_name: \"foss.openreplay.com\" #g" vars.yaml
sed -i "s#kubeconfig.*#kubeconfig_path: ${KUBECONFIG}#g" vars.yaml
sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml
bash kube-install.sh --app utilities
env:
DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}
IMAGE_TAG: ${{ github.sha }}
ENVIRONMENT: staging
# - name: Debug Job
# if: ${{ failure() }}
# uses: mxschmitt/action-tmate@v3
# env:
# DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}
# IMAGE_TAG: ${{ github.sha }}
# ENVIRONMENT: staging
#

2
.github/workflows/workers.yaml vendored
View file

@ -70,7 +70,7 @@ jobs:
sed -i "s#kubeconfig.*#kubeconfig_path: ${KUBECONFIG}#g" vars.yaml
for image in $(cat ../../backend/images_to_build.txt);
do
sed -i "s/tag:.*/tag: \"$IMAGE_TAG\"/g" app/${image}.yaml
sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml
# Deploy command
bash kube-install.sh --app $image
done

5
api/.chalice/config.json
View file

@ -30,9 +30,10 @@
"sessions_bucket": "mobs",
"sessions_region": "us-east-1",
"put_S3_TTL": "20",
"sourcemaps_reader": "http://127.0.0.1:3000/",
"sourcemaps_reader": "http://utilities-openreplay.app.svc.cluster.local:9000/sourcemaps",
"sourcemaps_bucket": "sourcemaps",
"js_cache_bucket": "sessions-assets",
"peers": "http://utilities-openreplay.app.svc.cluster.local:9000/assist/peers",
"async_Token": "",
"EMAIL_HOST": "",
"EMAIL_PORT": "587",
@ -51,7 +52,7 @@
"S3_HOST": "",
"S3_KEY": "",
"S3_SECRET": "",
"version_number": "1.0.0"
"version_number": "1.2.0"
},
"lambda_timeout": 150,
"lambda_memory_size": 400,

8
api/Dockerfile
View file

@ -4,14 +4,6 @@ WORKDIR /work
COPY . .
RUN pip install -r requirements.txt -t ./vendor --upgrade
RUN pip install chalice==1.22.2
# Installing Nodejs
RUN apt update && apt install -y curl && \
curl -fsSL https://deb.nodesource.com/setup_12.x | bash - && \
apt install -y nodejs && \
apt remove --purge -y curl && \
rm -rf /var/lib/apt/lists/* && \
cd sourcemaps_reader && \
npm install
# Add Tini
# Startup daemon

2
api/build.sh
View file

@ -27,6 +27,8 @@ function build_api(){
docker build -f ./Dockerfile --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/chalice:${git_sha1} .
[[ $PUSH_IMAGE -eq 1 ]] && {
docker push ${DOCKER_REPO:-'local'}/chalice:${git_sha1}
docker tag ${DOCKER_REPO:-'local'}/chalice:${git_sha1} ${DOCKER_REPO:-'local'}/chalice:latest
docker push ${DOCKER_REPO:-'local'}/chalice:latest
}
}

9
api/chalicelib/blueprints/bp_core.py
View file

@ -10,7 +10,8 @@ from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assig
log_tool_elasticsearch, log_tool_datadog, \
log_tool_stackdriver, reset_password, sessions_favorite_viewed, \
log_tool_cloudwatch, log_tool_sentry, log_tool_sumologic, log_tools, errors, sessions, \
log_tool_newrelic, announcements, log_tool_bugsnag, weekly_report, integration_jira_cloud, integration_github
log_tool_newrelic, announcements, log_tool_bugsnag, weekly_report, integration_jira_cloud, integration_github, \
assist
from chalicelib.core.collaboration_slack import Slack
from chalicelib.utils import email_helper
@ -875,3 +876,9 @@ def all_issue_types(context):
@app.route('/{projectId}/flows', methods=['GET', 'PUT', 'POST', 'DELETE'])
def removed_endpoints(projectId=None, context=None):
return Response(body={"errors": ["Endpoint no longer available"]}, status_code=410)
@app.route('/{projectId}/assist/sessions', methods=['GET'])
def sessions_live(projectId, context):
data = assist.get_live_sessions(projectId)
return {'data': data}

59
api/chalicelib/blueprints/subs/bp_dashboard.py
View file

@ -130,8 +130,9 @@ def get_network_widget(projectId, context):
@app.route('/{projectId}/dashboard/{widget}/search', methods=['GET'])
def get_dashboard_autocomplete(projectId, widget, context):
params = app.current_request.query_params
if params is None:
if params is None or params.get('q') is None or len(params.get('q')) == 0:
return {"data": []}
params['q'] = '^' + params['q']
if widget in ['performance']:
data = dashboard.search(params.get('q', ''), params.get('type', ''), project_id=projectId,
@ -547,59 +548,3 @@ def get_dashboard_group(projectId, context):
*helper.explode_widget(dashboard.get_avg_cpu(project_id=projectId, **{**data, **args})),
*helper.explode_widget(dashboard.get_avg_fps(project_id=projectId, **{**data, **args})),
]}
@app.route('/{projectId}/dashboard/errors_crashes', methods=['GET', 'POST'])
def get_dashboard_group(projectId, context):
data = app.current_request.json_body
if data is None:
data = {}
params = app.current_request.query_params
args = dashboard.dashboard_args(params)
return {"data": [
{"key": "errors",
"data": dashboard.get_errors(project_id=projectId, **{**data, **args})},
{"key": "errors_trend",
"data": dashboard.get_errors_trend(project_id=projectId, **{**data, **args})},
{"key": "crashes",
"data": dashboard.get_crashes(project_id=projectId, **{**data, **args})},
{"key": "domains_errors",
"data": dashboard.get_domains_errors(project_id=projectId, **{**data, **args})},
{"key": "errors_per_domains",
"data": dashboard.get_errors_per_domains(project_id=projectId, **{**data, **args})},
{"key": "calls_errors",
"data": dashboard.get_calls_errors(project_id=projectId, **{**data, **args})},
{"key": "errors_per_type",
"data": dashboard.get_errors_per_type(project_id=projectId, **{**data, **args})},
{"key": "impacted_sessions_by_js_errors",
"data": dashboard.get_impacted_sessions_by_js_errors(project_id=projectId, **{**data, **args})}
]}
@app.route('/{projectId}/dashboard/resources', methods=['GET', 'POST'])
def get_dashboard_group(projectId, context):
data = app.current_request.json_body
if data is None:
data = {}
params = app.current_request.query_params
args = dashboard.dashboard_args(params)
return {"data": [
{"key": "slowest_images",
"data": dashboard.get_slowest_images(project_id=projectId, **{**data, **args})},
{"key": "missing_resources",
"data": dashboard.get_missing_resources_trend(project_id=projectId, **{**data, **args})},
{"key": "slowest_resources",
"data": dashboard.get_slowest_resources(project_id=projectId, type='all', **{**data, **args})},
{"key": "resources_loading_time",
"data": dashboard.get_resources_loading_time(project_id=projectId, **{**data, **args})},
{"key": "resources_by_party",
"data": dashboard.get_resources_by_party(project_id=projectId, **{**data, **args})},
{"key": "resource_type_vs_response_end",
"data": dashboard.resource_type_vs_response_end(project_id=projectId, **{**data, **args})},
{"key": "resources_vs_visually_complete",
"data": dashboard.get_resources_vs_visually_complete(project_id=projectId, **{**data, **args})},
{"key": "resources_count_by_type",
"data": dashboard.get_resources_count_by_type(project_id=projectId, **{**data, **args})}
]}

55
api/chalicelib/core/assist.py Normal file
View file

@ -0,0 +1,55 @@
from chalicelib.utils import pg_client, helper
from chalicelib.core import projects
import requests
from chalicelib.utils.helper import environ
SESSION_PROJECTION_COLS = """s.project_id,
s.session_id::text AS session_id,
s.user_uuid,
s.user_id,
s.user_agent,
s.user_os,
s.user_browser,
s.user_device,
s.user_device_type,
s.user_country,
s.start_ts,
s.user_anonymous_id,
s.platform
"""
def get_live_sessions(project_id):
project_key = projects.get_project_key(project_id)
connected_peers = requests.get(environ["peers"] + f"/{project_key}")
if connected_peers.status_code != 200:
print("!! issue with the peer-server")
print(connected_peers.text)
return []
connected_peers = connected_peers.json().get("data", [])
if len(connected_peers) == 0:
return []
connected_peers = tuple(connected_peers)
with pg_client.PostgresClient() as cur:
query = cur.mogrify(f"""\
SELECT {SESSION_PROJECTION_COLS}, %(project_key)s||'-'|| session_id AS peer_id
FROM public.sessions AS s
WHERE s.project_id = %(project_id)s
AND session_id IN %(connected_peers)s;""",
{"project_id": project_id, "connected_peers": connected_peers, "project_key": project_key})
cur.execute(query)
results = cur.fetchall()
return helper.list_to_camel_case(results)
def is_live(project_id, session_id, project_key=None):
if project_key is None:
project_key = projects.get_project_key(project_id)
connected_peers = requests.get(environ["peers"] + f"/{project_key}")
if connected_peers.status_code != 200:
print("!! issue with the peer-server")
print(connected_peers.text)
return False
connected_peers = connected_peers.json().get("data", [])
return session_id in connected_peers

903
api/chalicelib/core/dashboard.py
View file

File diff suppressed because it is too large Load diff

3
api/chalicelib/core/reset_password.py
View file

@ -38,8 +38,7 @@ def step2(data):
print("error: wrong email or reset code")
return {"errors": ["wrong email or reset code"]}
users.update(tenant_id=user["tenantId"], user_id=user["id"],
changes={"token": None, "password": data["password"], "generatedPassword": False,
"verifiedEmail": True})
changes={"token": None, "password": data["password"], "generatedPassword": False})
return {"data": {"state": "success"}}

7
api/chalicelib/core/sessions.py
View file

@ -1,6 +1,6 @@
from chalicelib.utils import pg_client, helper, dev
from chalicelib.core import events, sessions_metas, socket_ios, metadata, events_ios, \
sessions_mobs, issues, projects, errors, resources
sessions_mobs, issues, projects, errors, resources, assist
SESSION_PROJECTION_COLS = """s.project_id,
s.session_id::text AS session_id,
@ -54,7 +54,8 @@ def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_
f"""\
SELECT
s.*,
s.session_id::text AS session_id
s.session_id::text AS session_id,
(SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key
{"," if len(extra_query) > 0 else ""}{",".join(extra_query)}
{(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata._get_column_names()]) + ") AS project_metadata") if group_metadata else ''}
FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""}
@ -99,6 +100,8 @@ def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_
data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data)
data['issues'] = issues.get_by_session_id(session_id=session_id)
data['live'] = assist.is_live(project_id=project_id, session_id=session_id,
project_key=data["projectKey"])
return data
return None

2
api/chalicelib/utils/email_handler.py
View file

@ -10,7 +10,7 @@ from chalicelib.utils.helper import environ
def __get_subject(subject):
return subject if helper.is_production() else f"{helper.get_stage_name()}: {subject}"
return subject
def __get_html_from_file(source, formatting_variables):

2
api/chalicelib/utils/pg_client.py
View file

@ -27,7 +27,7 @@ class ORThreadedConnectionPool(psycopg2.pool.ThreadedConnectionPool):
try:
postgreSQL_pool = ORThreadedConnectionPool(20, 100, **PG_CONFIG)
postgreSQL_pool = ORThreadedConnectionPool(50, 100, **PG_CONFIG)
if (postgreSQL_pool):
print("Connection pool created successfully")
except (Exception, psycopg2.DatabaseError) as error:

7
api/db_changes.sql Normal file
View file

@ -0,0 +1,7 @@
BEGIN;
CREATE INDEX pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time>0;
CREATE INDEX pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time>0;
CREATE INDEX pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0;
CREATE INDEX pages_ttfb_idx ON events.pages (ttfb) WHERE ttfb > 0;
CREATE INDEX pages_time_to_interactive_idx ON events.pages (time_to_interactive) WHERE time_to_interactive > 0;
COMMIT;

3
api/entrypoint.sh
View file

@ -1,6 +1,3 @@
#!/bin/bash
cd sourcemaps_reader
nohup node server.js &> /tmp/sourcemaps_reader.log &
cd ..
python env_handler.py
chalice local --no-autoreload --host 0.0.0.0 --stage ${ENTERPRISE_BUILD}

11
api/sourcemaps_reader/.gitignore vendored
View file

@ -1,11 +0,0 @@
# package directories
node_modules
jspm_packages
# Serverless directories
.serverless/*.zip
node_modules/
.idea
test.js

15
api/sourcemaps_reader/README.md
View file

@ -1,15 +0,0 @@
# sourcemap-reader
Source Map Reader
# For SAAS:
to run local; put your test values in handler then run `node handler.js`
to deploy `sls deploy --stage [staging|prod|dev]`
# Requirements:
- nodeJS 12 or greater
# Install for OS:
```
npm install
node server.js
```

109
api/sourcemaps_reader/package-lock.json generated
View file

@ -1,109 +0,0 @@
{
"name": "sourcemap-reader",
"version": "1.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"aws-sdk": {
"version": "2.654.0",
"resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.654.0.tgz",
"integrity": "sha512-RAx/SJ74zAqBW1wyRxiHNflmrv50i35pu8kPxfMIJ418TJzIMt+LKgn55rTJgyUdUzKi+MC9XMY4n7IDtwj3HA==",
"requires": {
"buffer": "4.9.1",
"events": "1.1.1",
"ieee754": "1.1.13",
"jmespath": "0.15.0",
"querystring": "0.2.0",
"sax": "1.2.1",
"url": "0.10.3",
"uuid": "3.3.2",
"xml2js": "0.4.19"
},
"dependencies": {
"buffer": {
"version": "4.9.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
"integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
"requires": {
"base64-js": "^1.0.2",
"ieee754": "^1.1.4",
"isarray": "^1.0.0"
}
},
"events": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
"integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ="
},
"punycode": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
"integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
},
"url": {
"version": "0.10.3",
"resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
"integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=",
"requires": {
"punycode": "1.3.2",
"querystring": "0.2.0"
}
}
}
},
"base64-js": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
"integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
},
"ieee754": {
"version": "1.1.13",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
"integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
},
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"jmespath": {
"version": "0.15.0",
"resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
"integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc="
},
"querystring": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
"integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
},
"sax": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
"integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o="
},
"source-map": {
"version": "0.7.3",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz",
"integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ=="
},
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
},
"xml2js": {
"version": "0.4.19",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
"integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==",
"requires": {
"sax": ">=0.6.0",
"xmlbuilder": "~9.0.1"
}
},
"xmlbuilder": {
"version": "9.0.7",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz",
"integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0="
}
}
}

16
api/sourcemaps_reader/package.json
View file

@ -1,16 +0,0 @@
{
"name": "sourcemap-reader",
"version": "1.0.0",
"description": "",
"main": "handler.js",
"dependencies": {
"aws-sdk": "^2.654.0",
"source-map": "^0.7.3"
},
"devDependencies": {},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "Kraiem taha yassine <tahayk2@gmail.com>",
"license": "ISC"
}

38
api/sourcemaps_reader/server.js
View file

@ -1,38 +0,0 @@
const http = require('http');
const handler = require('./handler');
const hostname = '127.0.0.1';
const port = 3000;
const server = http.createServer((req, res) => {
if (req.method === 'POST') {
let data = '';
req.on('data', chunk => {
data += chunk;
});
req.on('end', function () {
data = JSON.parse(data);
console.log("Starting parser for: " + data.key);
// process.env = {...process.env, ...data.bucket_config};
handler.sourcemapReader(data)
.then((results) => {
res.statusCode = 200;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(results));
})
.catch((e) => {
console.error("Something went wrong");
console.error(e);
res.statusCode(500);
res.end(e);
});
})
} else {
res.statusCode = 405;
res.setHeader('Content-Type', 'text/plain');
res.end('Method Not Allowed');
}
});
server.listen(port, hostname, () => {
console.log(`Server running at http://${hostname}:${port}/`);
});

7
backend/Dockerfile
View file

@ -4,15 +4,11 @@ RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bas
WORKDIR /root
COPY go.mod .
COPY go.sum .
COPY . .
RUN go mod download
COPY . .
ARG SERVICE_NAME
RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o service -tags musl openreplay/backend/services/$SERVICE_NAME
FROM alpine
@ -39,6 +35,7 @@ ENV TZ=UTC \
AWS_REGION_WEB=eu-central-1 \
AWS_REGION_IOS=eu-west-1 \
AWS_REGION_ASSETS=eu-central-1 \
CACHE_ASSETS=false \
ASSETS_SIZE_LIMIT=6291456

1
backend/Dockerfile-all
View file

@ -35,6 +35,7 @@ ENV TZ=UTC \
AWS_REGION_WEB=eu-central-1 \
AWS_REGION_IOS=eu-west-1 \
AWS_REGION_ASSETS=eu-central-1 \
CACHE_ASSETS=true \
ASSETS_SIZE_LIMIT=6291456
RUN mkdir $FS_DIR

2
backend/pkg/intervals/intervals.go
View file

@ -1,6 +1,6 @@
package intervals
const EVENTS_COMMIT_INTERVAL = 1 * 60 * 1000
const EVENTS_COMMIT_INTERVAL = 30 * 1000
const HEARTBEAT_INTERVAL = 2 * 60 * 1000
const INTEGRATIONS_REQUEST_INTERVAL = 2 * 60 * 1000
const EVENTS_PAGE_EVENT_TIMEOUT = 2 * 60 * 1000

18
backend/pkg/url/assets/css.go
View file

@ -51,13 +51,27 @@ func ExtractURLsFromCSS(css string) []string {
return urls
}
func (r *Rewriter) RewriteCSS(sessionID uint64, baseurl string, css string) string {
func rewriteLinks(css string, rewrite func(rawurl string) string) string {
for _, idx := range cssUrlsIndex(css) {
f := idx[0]
t := idx[1]
rawurl, q := unquote(css[f:t])
// why exactly quote back?
css = css[:f] + q + r.RewriteURL(sessionID, baseurl, rawurl) + q + css[t:]
css = css[:f] + q + rewrite(rawurl) + q + css[t:]
}
return css
}
func ResolveCSS(baseURL string, css string) string {
css = rewriteLinks(css, func(rawurl string) string {
return ResolveURL(baseURL, rawurl)
})
return strings.Replace(css, ":hover", ".-asayer-hover", -1)
}
func (r *Rewriter) RewriteCSS(sessionID uint64, baseurl string, css string) string {
css = rewriteLinks(css, func(rawurl string) string {
return r.RewriteURL(sessionID, baseurl, rawurl)
})
return strings.Replace(css, ":hover", ".-asayer-hover", -1)
}

7
backend/pkg/url/assets/url.go
View file

@ -12,6 +12,9 @@ func getSessionKey(sessionID uint64) string {
}
func ResolveURL(baseurl string, rawurl string) string {
if !isRelativeCachable(rawurl) {
return rawurl
}
base, _ := url.ParseRequestURI(baseurl) // fn Only for base urls
u, _ := url.Parse(rawurl) // TODO: handle errors ?
if base == nil || u == nil {
@ -50,7 +53,7 @@ func GetFullCachableURL(baseURL string, relativeURL string) (string, bool) {
}
const ASAYER_QUERY_START = "ASAYER_QUERY_ESCtRT"
const OPENREPLAY_QUERY_START = "OPENREPLAY_QUERY"
func getCachePath(rawurl string) string {
u, _ := url.Parse(rawurl)
@ -59,7 +62,7 @@ func getCachePath(rawurl string) string {
if (s[len(s) - 1] != '/') {
s += "/"
}
s += ASAYER_QUERY_START + url.PathEscape(u.RawQuery)
s += OPENREPLAY_QUERY_START + url.PathEscape(u.RawQuery)
}
return s
}

1
backend/services/assets/cacher/cacher.go
View file

@ -62,6 +62,7 @@ func (c *cacher) cacheURL(requestURL string, sessionID uint64, depth byte, conte
req, _ := http.NewRequest("GET", requestURL, nil)
req.Header.Set("Cookie", "ABv=3;") // Hack for rueducommerce
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:31.0) Gecko/20100101 Firefox/31.0")
res, err := c.httpClient.Do(req)
if err != nil {
c.Errors <- errors.Wrap(err, context)

2
backend/services/assets/main.go
View file

@ -64,6 +64,8 @@ func main() {
log.Printf("Caught signal %v: terminating\n", sig)
consumer.Close()
os.Exit(0)
case err := <-cacher.Errors:
log.Printf("Error while caching: %v", err)
case <-tick:
cacher.UpdateTimeouts()
default:

18
backend/services/http/assets.go
View file

@ -6,7 +6,7 @@ import (
)
func sendAssetForCache(sessionID uint64, baseURL string, relativeURL string) {
if fullURL, cachable := assets.GetFullCachableURL(baseURL, relativeURL); cachable {
if fullURL, cacheable := assets.GetFullCachableURL(baseURL, relativeURL); cacheable {
producer.Produce(topicTrigger, sessionID, messages.Encode(&messages.AssetCache{
URL: fullURL,
}))
@ -17,4 +17,20 @@ func sendAssetsForCacheFromCSS(sessionID uint64, baseURL string, css string) {
for _, u := range assets.ExtractURLsFromCSS(css) { // TODO: in one shot with rewriting
sendAssetForCache(sessionID, baseURL, u)
}
}
func handleURL(sessionID uint64, baseURL string, url string) string {
if cacheAssets {
sendAssetForCache(sessionID, baseURL, url)
return rewriter.RewriteURL(sessionID, baseURL, url)
}
return assets.ResolveURL(baseURL, url)
}
func handleCSS(sessionID uint64, baseURL string, css string) string {
if cacheAssets {
sendAssetsForCacheFromCSS(sessionID, baseURL, css)
return rewriter.RewriteCSS(sessionID, baseURL, css)
}
return assets.ResolveCSS(baseURL, css)
}

12
backend/services/http/handlers.go
View file

@ -174,32 +174,28 @@ func pushMessagesSeparatelyHandler(w http.ResponseWriter, r *http.Request) {
switch m := msg.(type) {
case *SetNodeAttributeURLBased:
if m.Name == "src" || m.Name == "href" {
sendAssetForCache(sessionData.ID, m.BaseURL, m.Value)
msg = &SetNodeAttribute{
ID: m.ID,
Name: m.Name,
Value: rewriter.RewriteURL(sessionData.ID, m.BaseURL, m.Value),
Value: handleURL(sessionData.ID, m.BaseURL, m.Value),
}
} else if m.Name == "style" {
sendAssetsForCacheFromCSS(sessionData.ID, m.BaseURL, m.Value)
msg = &SetNodeAttribute{
ID: m.ID,
Name: m.Name,
Value: rewriter.RewriteCSS(sessionData.ID, m.BaseURL, m.Value),
Value: handleCSS(sessionData.ID, m.BaseURL, m.Value),
}
}
case *SetCSSDataURLBased:
sendAssetsForCacheFromCSS(sessionData.ID, m.BaseURL, m.Data)
msg = &SetCSSData{
ID: m.ID,
Data: rewriter.RewriteCSS(sessionData.ID, m.BaseURL, m.Data),
Data: handleCSS(sessionData.ID, m.BaseURL, m.Data),
}
case *CSSInsertRuleURLBased:
sendAssetsForCacheFromCSS(sessionData.ID, m.BaseURL, m.Rule)
msg = &CSSInsertRule{
ID: m.ID,
Index: m.Index,
Rule: rewriter.RewriteCSS(sessionData.ID, m.BaseURL, m.Rule),
Rule: handleCSS(sessionData.ID, m.BaseURL, m.Rule),
}
}

3
backend/services/http/main.go
View file

@ -35,6 +35,7 @@ var topicRaw string
var topicTrigger string
var topicAnalytics string
// var kafkaTopicEvents string
var cacheAssets bool
func main() {
log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile)
@ -52,6 +53,8 @@ func main() {
uaParser = uaparser.NewUAParser(env.String("UAPARSER_FILE"))
geoIP = geoip.NewGeoIP(env.String("MAXMINDDB_FILE"))
flaker = flakeid.NewFlaker(env.WorkerID())
cacheAssets = env.Bool("CACHE_ASSETS")
HTTP_PORT := env.String("HTTP_PORT")
server := &http.Server{

11
ee/api/.chalice/config.json
View file

@ -33,8 +33,9 @@
"sessions_bucket": "mobs",
"sessions_region": "us-east-1",
"put_S3_TTL": "20",
"sourcemaps_reader": "http://127.0.0.1:3000/",
"sourcemaps_reader": "http://utilities-openreplay.app.svc.cluster.local:9000/sourcemaps",
"sourcemaps_bucket": "sourcemaps",
"peers": "http://utilities-openreplay.app.svc.cluster.local:9000/assist/peers",
"js_cache_bucket": "sessions-assets",
"async_Token": "",
"EMAIL_HOST": "",
@ -54,7 +55,13 @@
"S3_HOST": "",
"S3_KEY": "",
"S3_SECRET": "",
"version_number": "1.0.0"
"version_number": "1.0.0",
"LICENSE_KEY": "",
"SAML2_MD_URL": "",
"idp_entityId": "",
"idp_sso_url": "",
"idp_x509cert": "",
"idp_sls_url": ""
},
"lambda_timeout": 150,
"lambda_memory_size": 400,

58
ee/api/.gitignore vendored
View file

@ -175,4 +175,60 @@ SUBNETS.json
chalicelib/.config
chalicelib/saas
README/*
Pipfile
Pipfile
/chalicelib/core/alerts.py
/chalicelib/core/announcements.py
/chalicelib/blueprints/bp_app_api.py
/chalicelib/blueprints/bp_core.py
/chalicelib/blueprints/bp_core_crons.py
/chalicelib/core/collaboration_slack.py
/chalicelib/core/errors_favorite_viewed.py
/chalicelib/core/events.py
/chalicelib/core/events_ios.py
/chalicelib/core/integration_base.py
/chalicelib/core/integration_base_issue.py
/chalicelib/core/integration_github.py
/chalicelib/core/integration_github_issue.py
/chalicelib/core/integration_jira_cloud.py
/chalicelib/core/integration_jira_cloud_issue.py
/chalicelib/core/integrations_manager.py
/chalicelib/core/issues.py
/chalicelib/core/jobs.py
/chalicelib/core/log_tool_bugsnag.py
/chalicelib/core/log_tool_cloudwatch.py
/chalicelib/core/log_tool_datadog.py
/chalicelib/core/log_tool_elasticsearch.py
/chalicelib/core/log_tool_newrelic.py
/chalicelib/core/log_tool_rollbar.py
/chalicelib/core/log_tool_sentry.py
/chalicelib/core/log_tool_stackdriver.py
/chalicelib/core/log_tool_sumologic.py
/chalicelib/core/sessions_assignments.py
/chalicelib/core/sessions_favorite_viewed.py
/chalicelib/core/sessions_metas.py
/chalicelib/core/sessions_mobs.py
/chalicelib/core/significance.py
/chalicelib/core/slack.py
/chalicelib/core/socket_ios.py
/chalicelib/core/sourcemaps.py
/chalicelib/core/sourcemaps_parser.py
/chalicelib/core/weekly_report.py
/chalicelib/saml
/chalicelib/utils/html/
/chalicelib/utils/__init__.py
/chalicelib/utils/args_transformer.py
/chalicelib/utils/captcha.py
/chalicelib/utils/dev.py
/chalicelib/utils/email_handler.py
/chalicelib/utils/email_helper.py
/chalicelib/utils/event_filter_definition.py
/chalicelib/utils/github_client_v3.py
/chalicelib/utils/helper.py
/chalicelib/utils/jira_client.py
/chalicelib/utils/metrics_helper.py
/chalicelib/utils/pg_client.py
/chalicelib/utils/s3.py
/chalicelib/utils/smtp.py
/chalicelib/utils/strings.py
/chalicelib/utils/TimeUTC.py

13
ee/api/app.py
View file

@ -11,7 +11,7 @@ from chalicelib.utils import helper
from chalicelib.utils import pg_client
from chalicelib.utils.helper import environ
from chalicelib.blueprints import bp_ee, bp_ee_crons
from chalicelib.blueprints import bp_ee, bp_ee_crons, bp_saml
app = Chalice(app_name='parrot')
app.debug = not helper.is_production() or helper.is_local()
@ -59,17 +59,11 @@ _overrides.chalice_app(app)
@app.middleware('http')
def or_middleware(event, get_response):
from chalicelib.ee import unlock
from chalicelib.core import unlock
if not unlock.is_valid():
return Response(body={"errors": ["expired license"]}, status_code=403)
if "{projectid}" in event.path.lower():
from chalicelib.ee import projects
print("==================================")
print(event.context["authorizer"].get("authorizer_identity"))
print(event.uri_params["projectId"])
print(projects.get_internal_project_id(event.uri_params["projectId"]))
print(event.context["authorizer"]["tenantId"])
print("==================================")
from chalicelib.core import projects
if event.context["authorizer"].get("authorizer_identity") == "api_key" \
and not projects.is_authorized(
project_id=projects.get_internal_project_id(event.uri_params["projectId"]),
@ -126,3 +120,4 @@ app.register_blueprint(bp_dashboard.app)
# Enterprise
app.register_blueprint(bp_ee.app)
app.register_blueprint(bp_ee_crons.app)
app.register_blueprint(bp_saml.app)

2
ee/api/chalicelib/blueprints/bp_authorizers.py
View file

@ -2,7 +2,7 @@ from chalice import Blueprint, AuthResponse
from chalicelib.utils import helper
from chalicelib.core import authorizers
from chalicelib.ee import users
from chalicelib.core import users
app = Blueprint(__name__)

27
ee/api/chalicelib/blueprints/bp_core_dynamic.py
View file

@ -3,19 +3,19 @@ from chalice import Blueprint, Response
from chalicelib import _overrides
from chalicelib.core import metadata, errors_favorite_viewed, slack, alerts, sessions, integration_github, \
integrations_manager
from chalicelib.utils import captcha
from chalicelib.utils import captcha, SAML2_helper
from chalicelib.utils import helper
from chalicelib.utils.helper import environ
from chalicelib.ee import tenants
from chalicelib.ee import signup
from chalicelib.ee import users
from chalicelib.ee import projects
from chalicelib.ee import errors
from chalicelib.ee import notifications
from chalicelib.ee import boarding
from chalicelib.ee import webhook
from chalicelib.ee import license
from chalicelib.core import tenants
from chalicelib.core import signup
from chalicelib.core import users
from chalicelib.core import projects
from chalicelib.core import errors
from chalicelib.core import notifications
from chalicelib.core import boarding
from chalicelib.core import webhook
from chalicelib.core import license
from chalicelib.core.collaboration_slack import Slack
app = Blueprint(__name__)
@ -41,6 +41,8 @@ def login():
return {
'errors': ['You’ve entered invalid Email or Password.']
}
elif "errors" in r:
return r
tenant_id = r.pop("tenantId")
# change this in open-source
@ -74,7 +76,8 @@ def get_account(context):
"metadata": metadata.get_remaining_metadata_with_count(context['tenantId'])
},
**license.get_status(context["tenantId"]),
"smtp": environ["EMAIL_HOST"] is not None and len(environ["EMAIL_HOST"]) > 0
"smtp": environ["EMAIL_HOST"] is not None and len(environ["EMAIL_HOST"]) > 0,
"saml2": SAML2_helper.is_saml2_available()
}
}
@ -350,6 +353,8 @@ def get_members(context):
@app.route('/client/members', methods=['PUT', 'POST'])
def add_member(context):
if SAML2_helper.is_saml2_available():
return {"errors": ["please use your SSO server to add teammates"]}
data = app.current_request.json_body
return users.create_member(tenant_id=context['tenantId'], user_id=context['userId'], data=data)

4
ee/api/chalicelib/blueprints/bp_core_dynamic_crons.py
View file

@ -4,8 +4,8 @@ from chalicelib.utils import helper
app = Blueprint(__name__)
_overrides.chalice_app(app)
from chalicelib.ee import telemetry
from chalicelib.ee import unlock
from chalicelib.core import telemetry
from chalicelib.core import unlock
# Run every day.

2
ee/api/chalicelib/blueprints/bp_ee.py
View file

@ -1,7 +1,7 @@
from chalice import Blueprint
from chalicelib import _overrides
from chalicelib.ee import unlock
from chalicelib.core import unlock
app = Blueprint(__name__)
_overrides.chalice_app(app)

188
ee/api/chalicelib/blueprints/bp_saml.py Normal file
View file

@ -0,0 +1,188 @@
from chalice import Blueprint
from chalicelib import _overrides
from chalicelib.utils.SAML2_helper import prepare_request, init_saml_auth
app = Blueprint(__name__)
_overrides.chalice_app(app)
from chalicelib.utils.helper import environ
from onelogin.saml2.auth import OneLogin_Saml2_Logout_Request
from onelogin.saml2.utils import OneLogin_Saml2_Utils
from chalice import Response
from chalicelib.core import users, tenants
@app.route("/saml2", methods=['GET'], authorizer=None)
def start_sso():
app.current_request.path = ''
req = prepare_request(request=app.current_request)
auth = init_saml_auth(req)
sso_built_url = auth.login()
return Response(
# status_code=301,
status_code=307,
body='',
headers={'Location': sso_built_url, 'Content-Type': 'text/plain'})
@app.route('/saml2/acs', methods=['POST'], content_types=['application/x-www-form-urlencoded'], authorizer=None)
def process_sso_assertion():
req = prepare_request(request=app.current_request)
session = req["cookie"]["session"]
request = req['request']
auth = init_saml_auth(req)
request_id = None
if 'AuthNRequestID' in session:
request_id = session['AuthNRequestID']
auth.process_response(request_id=request_id)
errors = auth.get_errors()
user_data = {}
if len(errors) == 0:
if 'AuthNRequestID' in session:
del session['AuthNRequestID']
user_data = auth.get_attributes()
# session['samlUserdata'] = user_data
# session['samlNameId'] = auth.get_nameid()
# session['samlNameIdFormat'] = auth.get_nameid_format()
# session['samlNameIdNameQualifier'] = auth.get_nameid_nq()
# session['samlNameIdSPNameQualifier'] = auth.get_nameid_spnq()
# session['samlSessionIndex'] = auth.get_session_index()
# session['samlSessionExpiration'] = auth.get_session_expiration()
# print('>>>>')
# print(session)
self_url = OneLogin_Saml2_Utils.get_self_url(req)
if 'RelayState' in request.form and self_url != request.form['RelayState']:
print("====>redirect")
return Response(
status_code=307,
body='',
headers={'Location': auth.redirect_to(request.form['RelayState']), 'Content-Type': 'text/plain'})
elif auth.get_settings().is_debug_active():
error_reason = auth.get_last_error_reason()
return {"errors": [error_reason]}
email = auth.get_nameid()
existing = users.get_by_email_only(auth.get_nameid())
internal_id = next(iter(user_data.get("internalId", [])), None)
if len(existing) == 0 or existing[0].get("origin") != 'saml':
tenant_key = user_data.get("tenantKey", [])
if len(tenant_key) == 0:
print("tenantKey not present in assertion")
return Response(
status_code=307,
body={"errors": ["tenantKey not present in assertion"]},
headers={'Location': auth.redirect_to(request.form['RelayState']), 'Content-Type': 'text/plain'})
else:
t = tenants.get_by_tenant_key(tenant_key[0])
if t is None:
return Response(
status_code=307,
body={"errors": ["Unknown tenantKey"]},
headers={'Location': auth.redirect_to(request.form['RelayState']), 'Content-Type': 'text/plain'})
if len(existing) == 0:
print("== new user ==")
users.create_sso_user(tenant_id=t['tenantId'], email=email, admin=True, origin='saml',
name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])),
internal_id=internal_id)
else:
existing = existing[0]
if existing.get("origin") != 'saml':
print("== migrating user to SAML ==")
users.update(tenant_id=t['tenantId'], user_id=existing["id"],
changes={"origin": 'saml', "internal_id": internal_id})
return users.authenticate_sso(email=email, internal_id=internal_id, exp=auth.get_session_expiration())
@app.route('/saml2/slo', methods=['GET'])
def process_slo_request(context):
req = prepare_request(request=app.current_request)
session = req["cookie"]["session"]
request = req['request']
auth = init_saml_auth(req)
name_id = session_index = name_id_format = name_id_nq = name_id_spnq = None
if 'samlNameId' in session:
name_id = session['samlNameId']
if 'samlSessionIndex' in session:
session_index = session['samlSessionIndex']
if 'samlNameIdFormat' in session:
name_id_format = session['samlNameIdFormat']
if 'samlNameIdNameQualifier' in session:
name_id_nq = session['samlNameIdNameQualifier']
if 'samlNameIdSPNameQualifier' in session:
name_id_spnq = session['samlNameIdSPNameQualifier']
users.change_jwt_iat(context["userId"])
return Response(
status_code=307,
body='',
headers={'Location': auth.logout(name_id=name_id, session_index=session_index, nq=name_id_nq,
name_id_format=name_id_format,
spnq=name_id_spnq), 'Content-Type': 'text/plain'})
@app.route('/saml2/sls', methods=['GET'], authorizer=None)
def process_sls_assertion():
req = prepare_request(request=app.current_request)
session = req["cookie"]["session"]
request = req['request']
auth = init_saml_auth(req)
request_id = None
if 'LogoutRequestID' in session:
request_id = session['LogoutRequestID']
def dscb():
session.clear()
url = auth.process_slo(request_id=request_id, delete_session_cb=dscb)
errors = auth.get_errors()
if len(errors) == 0:
if 'SAMLRequest' in req['get_data']:
logout_request = OneLogin_Saml2_Logout_Request(auth.get_settings(), req['get_data']['SAMLRequest'])
user_email = logout_request.get_nameid(auth.get_last_request_xml())
to_logout = users.get_by_email_only(user_email)
if len(to_logout) > 0:
to_logout = to_logout[0]['id']
users.change_jwt_iat(to_logout)
else:
print("Unknown user SLS-Request By IdP")
else:
print("Preprocessed SLS-Request by SP")
if url is not None:
return Response(
status_code=307,
body='',
headers={'Location': url, 'Content-Type': 'text/plain'})
return Response(
status_code=307,
body='',
headers={'Location': environ["SITE_URL"], 'Content-Type': 'text/plain'})
@app.route('/saml2/metadata', methods=['GET'], authorizer=None)
def saml2_metadata():
req = prepare_request(request=app.current_request)
auth = init_saml_auth(req)
settings = auth.get_settings()
metadata = settings.get_sp_metadata()
errors = settings.validate_metadata(metadata)
if len(errors) == 0:
return Response(
status_code=200,
body=metadata,
headers={'Content-Type': 'text/xml'})
else:
return Response(
status_code=500,
body=', '.join(errors))

2
ee/api/chalicelib/blueprints/subs/bp_dashboard.py
View file

@ -2,7 +2,7 @@ from chalice import Blueprint
from chalicelib.utils import helper
from chalicelib import _overrides
from chalicelib.ee import dashboard
from chalicelib.core import dashboard
from chalicelib.core import metadata

11
ee/api/chalicelib/core/authorizers.py
View file

@ -3,8 +3,8 @@ import jwt
from chalicelib.utils import helper
from chalicelib.utils.TimeUTC import TimeUTC
from chalicelib.ee import tenants
from chalicelib.ee import users
from chalicelib.core import tenants
from chalicelib.core import users
def jwt_authorizer(token):
@ -38,12 +38,13 @@ def jwt_context(context):
}
def generate_jwt(id, tenant_id, iat, aud):
def generate_jwt(id, tenant_id, iat, aud, exp=None):
token = jwt.encode(
payload={
"userId": id,
"tenantId": tenant_id,
"exp": iat // 1000 + int(environ["jwt_exp_delta_seconds"]) + TimeUTC.get_utc_offset() // 1000,
"exp": iat // 1000 + int(environ["jwt_exp_delta_seconds"]) + TimeUTC.get_utc_offset() // 1000 \
if exp is None else exp,
"iss": environ["jwt_issuer"],
"iat": iat // 1000,
"aud": aud
@ -58,4 +59,4 @@ def api_key_authorizer(token):
t = tenants.get_by_api_key(token)
if t is not None:
t["createdAt"] = TimeUTC.datetime_to_timestamp(t["createdAt"])
return t
return t

4
ee/api/chalicelib/ee/boarding.py → ee/api/chalicelib/core/boarding.py
View file

@ -1,8 +1,8 @@
from chalicelib.utils import pg_client
from chalicelib.core import log_tool_datadog, log_tool_stackdriver, log_tool_sentry
from chalicelib.ee import users
from chalicelib.ee import projects
from chalicelib.core import users
from chalicelib.core import projects
def get_state(tenant_id):

2
ee/api/chalicelib/ee/dashboard.py → ee/api/chalicelib/core/dashboard.py
View file

@ -5,7 +5,7 @@ from chalicelib.utils import pg_client
from chalicelib.utils import args_transformer
from chalicelib.utils import helper
from chalicelib.utils.TimeUTC import TimeUTC
from chalicelib.ee.utils import ch_client
from chalicelib.utils import ch_client
from math import isnan
from chalicelib.utils.metrics_helper import __get_step_size

4
ee/api/chalicelib/ee/errors.py → ee/api/chalicelib/core/errors.py
View file

@ -1,9 +1,9 @@
import json
from chalicelib.utils import pg_client, helper
from chalicelib.ee.utils import ch_client
from chalicelib.utils import ch_client
from chalicelib.core import sourcemaps, sessions
from chalicelib.ee import dashboard
from chalicelib.core import dashboard
from chalicelib.utils.TimeUTC import TimeUTC

2
ee/api/chalicelib/ee/license.py → ee/api/chalicelib/core/license.py
View file

@ -1,5 +1,5 @@
from chalicelib.utils import pg_client
from chalicelib.ee import unlock
from chalicelib.core import unlock
def get_status(tenant_id):

2
ee/api/chalicelib/core/metadata.py
View file

@ -1,6 +1,6 @@
from chalicelib.utils import pg_client, helper, dev
from chalicelib.ee import projects
from chalicelib.core import projects
import re

0
ee/api/chalicelib/ee/notifications.py → ee/api/chalicelib/core/notifications.py
View file

2
ee/api/chalicelib/ee/projects.py → ee/api/chalicelib/core/projects.py
View file

@ -1,6 +1,6 @@
import json
from chalicelib.ee import users
from chalicelib.core import users
from chalicelib.utils import pg_client, helper
from chalicelib.utils.TimeUTC import TimeUTC

2
ee/api/chalicelib/core/reset_password.py
View file

@ -3,7 +3,7 @@ from chalicelib.utils import email_helper, captcha, helper
import secrets
from chalicelib.utils import pg_client
from chalicelib.ee import users
from chalicelib.core import users
def step1(data):

2
ee/api/chalicelib/ee/resources.py → ee/api/chalicelib/core/resources.py
View file

@ -1,5 +1,5 @@
from chalicelib.utils import helper
from chalicelib.ee.utils import ch_client
from chalicelib.utils import ch_client
from chalicelib.utils.TimeUTC import TimeUTC

4
ee/api/chalicelib/core/sessions.py
View file

@ -2,9 +2,9 @@ from chalicelib.utils import pg_client, helper
from chalicelib.utils import dev
from chalicelib.core import events, sessions_metas, socket_ios, metadata, events_ios, sessions_mobs, issues
from chalicelib.ee import projects, errors
from chalicelib.core import projects, errors
from chalicelib.ee import resources
from chalicelib.core import resources
SESSION_PROJECTION_COLS = """s.project_id,
s.session_id::text AS session_id,

7
ee/api/chalicelib/ee/signup.py → ee/api/chalicelib/core/signup.py
View file

@ -1,6 +1,6 @@
from chalicelib.utils import helper
from chalicelib.utils import pg_client
from chalicelib.ee import users, telemetry
from chalicelib.core import users, telemetry
from chalicelib.utils import captcha
import json
from chalicelib.utils.TimeUTC import TimeUTC
@ -57,7 +57,8 @@ def create_step1(data):
signed_ups = get_signed_ups()
if len(signed_ups) == 0 and data.get("tenantId") is not None \
or len(signed_ups) > 0 and data.get("tenantId") not in [t['tenantId'] for t in signed_ups]:
or len(signed_ups) > 0 and data.get("tenantId") is not None\
and data.get("tenantId") not in [t['tenantId'] for t in signed_ups]:
errors.append("Tenant does not exist")
if len(errors) > 0:
print("==> error")
@ -156,4 +157,4 @@ def create_step1(data):
"user": r,
"client": c,
}
}
}

2
ee/api/chalicelib/ee/telemetry.py → ee/api/chalicelib/core/telemetry.py
View file

@ -65,4 +65,4 @@ def new_client(tenant_id):
FROM public.tenants
WHERE tenant_id=%(tenant_id)s;""", {"tenant_id": tenant_id}))
data = cur.fetchone()
requests.post('https://parrot.asayer.io/os/signup', json=process_data(data, edition='ee'))
requests.post('https://parrot.asayer.io/os/signup', json=process_data(data, edition='ee'))

25
ee/api/chalicelib/ee/tenants.py → ee/api/chalicelib/core/tenants.py
View file

@ -1,6 +1,26 @@
from chalicelib.utils import pg_client
from chalicelib.utils import helper
from chalicelib.ee import users
from chalicelib.core import users
def get_by_tenant_key(tenant_key):
with pg_client.PostgresClient() as cur:
cur.execute(
cur.mogrify(
f"""SELECT
t.tenant_id,
t.name,
t.api_key,
t.created_at,
t.edition,
t.version_number,
t.opt_out
FROM public.tenants AS t
WHERE t.user_id = %(user_id)s AND t.deleted_at ISNULL
LIMIT 1;""",
{"user_id": tenant_key})
)
return helper.dict_to_camel_case(cur.fetchone())
def get_by_tenant_id(tenant_id):
@ -14,7 +34,8 @@ def get_by_tenant_id(tenant_id):
t.created_at,
t.edition,
t.version_number,
t.opt_out
t.opt_out,
t.user_id AS tenant_key
FROM public.tenants AS t
WHERE t.tenant_id = %(tenantId)s AND t.deleted_at ISNULL
LIMIT 1;""",

0
ee/api/chalicelib/ee/unlock.py → ee/api/chalicelib/core/unlock.py
View file

105
ee/api/chalicelib/ee/users.py → ee/api/chalicelib/core/users.py
View file

@ -1,15 +1,12 @@
import json
import time
from chalicelib.core import authorizers
from chalicelib.core import tenants
from chalicelib.utils import helper
from chalicelib.utils import pg_client
from chalicelib.utils.TimeUTC import TimeUTC
from chalicelib.utils.helper import environ
from chalicelib.ee import tenants
def create_new_member(tenant_id, email, password, admin, name, owner=False):
with pg_client.PostgresClient() as cur:
@ -203,7 +200,8 @@ def get(user_id, tenant_id):
(CASE WHEN role = 'admin' THEN TRUE ELSE FALSE END) AS admin,
(CASE WHEN role = 'member' THEN TRUE ELSE FALSE END) AS member,
appearance,
api_key
api_key,
origin
FROM public.users LEFT JOIN public.basic_authentication ON users.user_id=basic_authentication.user_id
WHERE
users.user_id = %(userId)s
@ -274,7 +272,8 @@ def get_by_email_only(email):
basic_authentication.generated_password,
(CASE WHEN users.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin,
(CASE WHEN users.role = 'admin' THEN TRUE ELSE FALSE END) AS admin,
(CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member
(CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member,
origin
FROM public.users LEFT JOIN public.basic_authentication ON users.user_id=basic_authentication.user_id
WHERE
users.email = %(email)s
@ -363,6 +362,8 @@ def change_password(tenant_id, user_id, email, old_password, new_password):
item = get(tenant_id=tenant_id, user_id=user_id)
if item is None:
return {"errors": ["access denied"]}
if item["origin"] is not None:
return {"errors": ["cannot change your password because you are logged-in form an SSO service"]}
if old_password == new_password:
return {"errors": ["old and new password are the same"]}
auth = authenticate(email, old_password, for_change_password=True)
@ -437,9 +438,19 @@ def auth_exists(user_id, tenant_id, jwt_iat, jwt_aud):
)
def change_jwt_iat(user_id):
with pg_client.PostgresClient() as cur:
query = cur.mogrify(
f"""UPDATE public.users
SET jwt_iat = timezone('utc'::text, now())
WHERE user_id = %(user_id)s
RETURNING jwt_iat;""",
{"user_id": user_id})
cur.execute(query)
return cur.fetchone().get("jwt_iat")
def authenticate(email, password, for_change_password=False, for_plugin=False):
if helper.TRACK_TIME:
now = int(time.time() * 1000)
with pg_client.PostgresClient() as cur:
query = cur.mogrify(
f"""SELECT
@ -451,7 +462,8 @@ def authenticate(email, password, for_change_password=False, for_plugin=False):
(CASE WHEN users.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin,
(CASE WHEN users.role = 'admin' THEN TRUE ELSE FALSE END) AS admin,
(CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member,
users.appearance
users.appearance,
users.origin
FROM public.users AS users INNER JOIN public.basic_authentication USING(user_id)
WHERE users.email = %(email)s
AND basic_authentication.password = crypt(%(password)s, basic_authentication.password)
@ -461,13 +473,45 @@ def authenticate(email, password, for_change_password=False, for_plugin=False):
cur.execute(query)
r = cur.fetchone()
if helper.TRACK_TIME:
now2 = int(time.time() * 1000)
print(f"=====> authentication query&fetch in: {now2 - now} ms")
now = now2
if r is not None:
if r["origin"] is not None:
return {"errors": ["must sign-in with SSO"]}
if for_change_password:
return True
r = helper.dict_to_camel_case(r, ignore_keys=["appearance"])
jwt_iat = change_jwt_iat(r['id'])
return {
"jwt": authorizers.generate_jwt(r['id'], r['tenantId'],
TimeUTC.datetime_to_timestamp(jwt_iat),
aud=f"plugin:{helper.get_stage_name()}" if for_plugin else f"front:{helper.get_stage_name()}"),
"email": email,
**r
}
return None
def authenticate_sso(email, internal_id, exp=None):
with pg_client.PostgresClient() as cur:
query = cur.mogrify(
f"""SELECT
users.user_id AS id,
users.tenant_id,
users.role,
users.name,
False AS change_password,
(CASE WHEN users.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin,
(CASE WHEN users.role = 'admin' THEN TRUE ELSE FALSE END) AS admin,
(CASE WHEN users.role = 'member' THEN TRUE ELSE FALSE END) AS member,
users.appearance,
origin
FROM public.users AS users
WHERE users.email = %(email)s AND internal_id = %(internal_id)s;""",
{"email": email, "internal_id": internal_id})
cur.execute(query)
r = cur.fetchone()
if r is not None:
if for_change_password:
return True
r = helper.dict_to_camel_case(r, ignore_keys=["appearance"])
query = cur.mogrify(
f"""UPDATE public.users
@ -479,8 +523,37 @@ def authenticate(email, password, for_change_password=False, for_plugin=False):
return {
"jwt": authorizers.generate_jwt(r['id'], r['tenantId'],
TimeUTC.datetime_to_timestamp(cur.fetchone()["jwt_iat"]),
aud=f"plugin:{helper.get_stage_name()}" if for_plugin else f"front:{helper.get_stage_name()}"),
aud=f"front:{helper.get_stage_name()}",
exp=exp),
"email": email,
**r
}
return None
def create_sso_user(tenant_id, email, admin, name, origin, internal_id=None):
with pg_client.PostgresClient() as cur:
query = cur.mogrify(f"""\
WITH u AS (
INSERT INTO public.users (tenant_id, email, role, name, data, origin, internal_id)
VALUES (%(tenantId)s, %(email)s, %(role)s, %(name)s, %(data)s, %(origin)s, %(internal_id)s)
RETURNING *
)
SELECT u.user_id AS id,
u.email,
u.role,
u.name,
TRUE AS change_password,
(CASE WHEN u.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin,
(CASE WHEN u.role = 'admin' THEN TRUE ELSE FALSE END) AS admin,
(CASE WHEN u.role = 'member' THEN TRUE ELSE FALSE END) AS member,
u.appearance,
origin
FROM u;""",
{"tenantId": tenant_id, "email": email, "internal_id": internal_id,
"role": "admin" if admin else "member", "name": name, "origin": origin,
"data": json.dumps({"lastAnnouncementView": TimeUTC.now()})})
cur.execute(
query
)
return helper.dict_to_camel_case(cur.fetchone())

0
ee/api/chalicelib/ee/webhook.py → ee/api/chalicelib/core/webhook.py
View file

104
ee/api/chalicelib/utils/SAML2_helper.py Normal file
View file

@ -0,0 +1,104 @@
from http import cookies
from urllib.parse import urlparse, parse_qsl
from onelogin.saml2.auth import OneLogin_Saml2_Auth
from chalicelib.utils.helper import environ
SAML2 = {
"strict": True,
"debug": True,
"sp": {
"entityId": environ["SITE_URL"] + "/api/saml2/metadata/",
"assertionConsumerService": {
"url": environ["SITE_URL"] + "/api/saml2/acs",
"binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST"
},
"singleLogoutService": {
"url": environ["SITE_URL"] + "/api/saml2/sls",
"binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect"
},
"NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress",
"x509cert": "",
"privateKey": ""
},
"idp": None
}
idp = None
# SAML2 config handler
if len(environ.get("SAML2_MD_URL")) > 0:
print("SAML2_MD_URL provided, getting IdP metadata config")
from onelogin.saml2.idp_metadata_parser import OneLogin_Saml2_IdPMetadataParser
idp_data = OneLogin_Saml2_IdPMetadataParser.parse_remote(environ.get("SAML2_MD_URL"))
idp = idp_data.get("idp")
if SAML2["idp"] is None:
if len(environ.get("idp_entityId", "")) > 0 \
and len(environ.get("idp_sso_url", "")) > 0 \
and len(environ.get("idp_x509cert", "")) > 0:
idp = {
"entityId": environ["idp_entityId"],
"singleSignOnService": {
"url": environ["idp_sso_url"],
"binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect"
},
"x509cert": environ["idp_x509cert"]
}
if len(environ.get("idp_sls_url", "")) > 0:
idp["singleLogoutService"] = {
"url": environ["idp_sls_url"],
"binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect"
}
if idp is None:
print("No SAML2 IdP configuration found")
else:
SAML2["idp"] = idp
def init_saml_auth(req):
# auth = OneLogin_Saml2_Auth(req, custom_base_path=environ['SAML_PATH'])
if idp is None:
raise Exception("No SAML2 config provided")
auth = OneLogin_Saml2_Auth(req, old_settings=SAML2)
return auth
def prepare_request(request):
request.args = dict(request.query_params).copy() if request.query_params else {}
request.form = dict(request.json_body).copy() if request.json_body else dict(
parse_qsl(request.raw_body.decode())) if request.raw_body else {}
cookie_str = request.headers.get("cookie", "")
if "session" in cookie_str:
cookie = cookies.SimpleCookie()
cookie.load(cookie_str)
# Even though SimpleCookie is dictionary-like, it internally uses a Morsel object
# which is incompatible with requests. Manually construct a dictionary instead.
extracted_cookies = {}
for key, morsel in cookie.items():
extracted_cookies[key] = morsel.value
session = extracted_cookies["session"]
else:
session = {}
# If server is behind proxys or balancers use the HTTP_X_FORWARDED fields
headers = request.headers
url_data = urlparse('%s://%s' % (headers.get('x-forwarded-proto', 'http'), headers['host']))
return {
'https': 'on' if request.headers.get('x-forwarded-proto', 'http') == 'https' else 'off',
'http_host': request.headers['host'],
'server_port': url_data.port,
'script_name': request.path,
'get_data': request.args.copy(),
# Uncomment if using ADFS as IdP, https://github.com/onelogin/python-saml/pull/144
# 'lowercase_urlencoding': True,
'post_data': request.form.copy(),
'cookie': {"session": session},
'request': request
}
def is_saml2_available():
return idp is not None

0
ee/api/chalicelib/ee/utils/ch_client.py → ee/api/chalicelib/utils/ch_client.py
View file

3
ee/api/requirements.txt
View file

@ -9,4 +9,5 @@ elasticsearch==7.9.1
jira==2.0.0
schedule==1.1.0
croniter==1.0.12
clickhouse-driver==0.1.5
clickhouse-driver==0.1.5
python3-saml==1.10.1

1109
ee/connectors/data_analysis_cookbook/buying_clients.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

6
ee/connectors/data_analysis_cookbook/credentials.yml.example Normal file
View file

@ -0,0 +1,6 @@
pg:
user: user
password: ******
database: db_name
host: '127.0.0.1'
port: 8080

560
ee/connectors/data_analysis_cookbook/explorational_analysis.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

2
ee/connectors/deploy/requirements_clickhouse.txt
View file

@ -9,6 +9,6 @@ pytz==2021.1
requests==2.25.1
SQLAlchemy==1.3.23
tzlocal==2.1
urllib3==1.26.3
urllib3==1.26.5
PyYAML==5.4.1

2
ee/connectors/deploy/requirements_pg.txt
View file

@ -8,5 +8,5 @@ pytz==2021.1
requests==2.25.1
SQLAlchemy==1.3.23
tzlocal==2.1
urllib3==1.26.3
urllib3==1.26.5
PyYAML==5.4.1

2
ee/connectors/deploy/requirements_redshift.txt
View file

@ -9,7 +9,7 @@ pytz==2021.1
requests==2.25.1
SQLAlchemy==1.3.23
tzlocal==2.1
urllib3==1.26.3
urllib3==1.26.5
pandas-redshift
PyYAML
awswrangler

4
ee/connectors/deploy/requirements_snowflake.txt
View file

@ -13,7 +13,7 @@ botocore==1.18.18
certifi==2020.6.20
cffi==1.14.3
chardet==3.0.4
cryptography==2.9.2
cryptography==3.4.7
idna==2.10
isodate==0.6.0
jmespath==0.10.0
@ -30,5 +30,5 @@ requests==2.23.0
requests-oauthlib==1.3.0
s3transfer==0.3.3
six==1.15.0
urllib3==1.25.11
urllib3==1.26.5

6
ee/scripts/helm/db/init_dbs/postgresql/1.2.0/1.2.0.sql Normal file
View file

@ -0,0 +1,6 @@
BEGIN;
CREATE TYPE user_origin AS ENUM ('saml');
ALTER TABLE public.users
ADD COLUMN origin user_origin NULL DEFAULT NULL,
ADD COLUMN internal_id text NULL DEFAULT NULL;
COMMIT;

5
ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql
View file

@ -46,6 +46,7 @@ CREATE TABLE tenants
);
CREATE TYPE user_role AS ENUM ('owner', 'admin', 'member');
CREATE TYPE user_origin AS ENUM ('saml');
CREATE TABLE users
(
user_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY,
@ -119,7 +120,9 @@ CREATE TABLE users
api_key text UNIQUE default generate_api_key(20) not null,
jwt_iat timestamp without time zone NULL DEFAULT NULL,
data jsonb NOT NULL DEFAULT '{}'::jsonb,
weekly_report boolean NOT NULL DEFAULT TRUE
weekly_report boolean NOT NULL DEFAULT TRUE,
origin user_origin NULL DEFAULT NULL,
);

3
frontend/.storybook/webpack.config.js
View file

@ -3,7 +3,8 @@ const mainConfig = require('../webpack.config.js');
module.exports = async ({ config }) => {
var conf = mainConfig();
config.resolve.alias = Object.assign(pathAlias, config.resolve.alias); // Path Alias
config.resolve.alias = Object.assign(conf.resolve.alias, config.resolve.alias); // Path Alias
config.resolve.extensions = conf.resolve.extensions
config.module.rules = conf.module.rules;
config.module.rules[0].use[0] = 'style-loader'; // instead of separated css
config.module.rules[1].use[0] = 'style-loader';

3
frontend/app/api_client.js
View file

@ -20,7 +20,8 @@ const siteIdRequiredPaths = [
'/rehydrations',
'/sourcemaps',
'/errors',
'/funnels'
'/funnels',
'/assist'
];
const noStoringFetchPathStarts = [

11
frontend/app/components/Assist/Assist.tsx Normal file
View file

@ -0,0 +1,11 @@
import React from 'react';
import ChatWindow from './ChatWindow';
export default function Assist() {
return (
<div className="absolute">
{/* <ChatWindow /> */}
</div>
)
}

29
frontend/app/components/Assist/ChatControls/ChatControls.css Normal file
View file

@ -0,0 +1,29 @@
.controls {
height: 38px;
/* margin-top: 5px; */
/* background-color: white; */
/* border-top: solid thin #CCC; */
}
.btnWrapper {
display: flex;
align-items: center;
height: 24px;
font-size: 12px;
color: $gray-medium;
&.disabled {
/* background-color: red; */
& svg {
fill: red;
}
}
}
.endButton {
background-color: $red;
border-radius: 3px;
padding: 2px 8px;
color: white;
font-size: 12px;
}

54
frontend/app/components/Assist/ChatControls/ChatControls.tsx Normal file
View file

@ -0,0 +1,54 @@
import React, { useState } from 'react'
import stl from './ChatControls.css'
import cn from 'classnames'
import { Button, Icon } from 'UI'
interface Props {
stream: MediaStream | null,
endCall: () => void
}
function ChatControls({ stream, endCall } : Props) {
const [audioEnabled, setAudioEnabled] = useState(true)
const [videoEnabled, setVideoEnabled] = useState(true)
const toggleAudio = () => {
if (!stream) { return; }
const aEn = !audioEnabled
stream.getAudioTracks().forEach(track => track.enabled = aEn);
setAudioEnabled(aEn);
}
const toggleVideo = () => {
if (!stream) { return; }
const vEn = !videoEnabled;
stream.getVideoTracks().forEach(track => track.enabled = vEn);
setVideoEnabled(vEn)
}
return (
<div className={cn(stl.controls, "flex items-center w-full justify-start bottom-0 px-2")}>
<div className="flex items-center">
<div className={cn(stl.btnWrapper, { [stl.disabled]: !audioEnabled})}>
<Button plain size="small" onClick={toggleAudio} noPadding className="flex items-center">
<Icon name={audioEnabled ? 'mic' : 'mic-mute'} size="16" />
<span className="ml-2 color-gray-medium text-sm">{audioEnabled ? 'Mute' : 'Unmute'}</span>
</Button>
</div>
<div className={cn(stl.btnWrapper, { [stl.disabled]: !videoEnabled})}>
<Button plain size="small" onClick={toggleVideo} noPadding className="flex items-center">
<Icon name={ videoEnabled ? 'camera-video' : 'camera-video-off' } size="16" />
<span className="ml-2 color-gray-medium text-sm">{videoEnabled ? 'Stop Video' : 'Start Video'}</span>
</Button>
</div>
</div>
<div className="ml-auto">
<button className={stl.endButton} onClick={endCall}>
END
</button>
</div>
</div>
)
}
export default ChatControls

1
frontend/app/components/Assist/ChatControls/index.js Normal file
View file

@ -0,0 +1 @@
export { default } from './ChatControls'

42
frontend/app/components/Assist/ChatWindow/ChatWindow.tsx Normal file
View file

@ -0,0 +1,42 @@
import React, { useState, FC } from 'react'
import VideoContainer from '../components/VideoContainer'
import { Icon, Popup, Button } from 'UI'
import cn from 'classnames'
import Counter from 'App/components/shared/SessionItem/Counter'
import stl from './chatWindow.css'
import ChatControls from '../ChatControls/ChatControls'
import Draggable from 'react-draggable';
export interface Props {
incomeStream: MediaStream | null,
localStream: MediaStream | null,
userId: String,
endCall: () => void
}
const ChatWindow: FC<Props> = function ChatWindow({ userId, incomeStream, localStream, endCall }) {
const [minimize, setMinimize] = useState(false)
return (
<Draggable handle=".handle" bounds="body">
<div
className={cn(stl.wrapper, "fixed radius bg-white shadow-xl mt-16")}
style={{ width: '280px' }}
>
<div className="handle flex items-center p-2 cursor-move select-none">
<div className={stl.headerTitle}><b>Meeting</b> {userId}</div>
<Counter startTime={new Date().getTime() } className="text-sm ml-auto" />
</div>
<div className={cn(stl.videoWrapper, {'hidden' : minimize}, 'relative')}>
<VideoContainer stream={ incomeStream } />
<div className="absolute bottom-0 right-0 z-50">
<VideoContainer stream={ localStream } muted width={50} />
</div>
</div>
<ChatControls stream={localStream} endCall={endCall} />
</div>
</Draggable>
)
}
export default ChatWindow

21
frontend/app/components/Assist/ChatWindow/chatWindow.css Normal file
View file

@ -0,0 +1,21 @@
.wrapper {
background-color: white;
border: solid thin #000;
border-radius: 3px;
position: fixed;
width: 300px;
}
.headerTitle {
font-size: 12px;
max-width: 180px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.videoWrapper {
height: 180px;
overflow: hidden;
background-color: #000;
}

1
frontend/app/components/Assist/ChatWindow/index.ts Normal file
View file

@ -0,0 +1 @@
export { default } from './ChatWindow'

45
frontend/app/components/Assist/ScreenSharing/ScreenSharing.tsx Normal file
View file

@ -0,0 +1,45 @@
import React from 'react'
import { Button } from 'UI'
function ScreenSharing() {
const videoRef = React.createRef<HTMLVideoElement>()
function handleSuccess(stream) {
// startButton.disabled = true;
//videoRef.current?.srcObject = stream;
// @ts-ignore
window.stream = stream; // make variable available to browser console
stream.getVideoTracks()[0].addEventListener('ended', () => {
console.log('The user has ended sharing the screen');
});
}
function handleError(error) {
console.log(`getDisplayMedia error: ${error.name}`, error);
}
const startScreenSharing = () => {
// @ts-ignore
navigator.mediaDevices.getDisplayMedia({video: true})
.then(handleSuccess, handleError);
}
const stopScreenSharing = () => {
// @ts-ignore
window.stream.stop()
console.log('Stop screen sharing')
}
return (
<div className="fixed inset-0 z-50 bg-red">
<video ref={ videoRef } id="screen-share" autoPlay loop muted></video>
<div className="absolute left-0 right-0 bottom-0">
<Button onClick={startScreenSharing}>Start</Button>
<Button onClick={stopScreenSharing}>Stop</Button>
</div>
</div>
)
}
export default ScreenSharing

1
frontend/app/components/Assist/ScreenSharing/index.js Normal file
View file

@ -0,0 +1 @@
export { default } from './ScreenSharing'

8
frontend/app/components/Assist/assist.stories.js Normal file
View file

@ -0,0 +1,8 @@
import { storiesOf } from '@storybook/react';
import ChatWindow from './ChatWindow';
storiesOf('Assist', module)
.add('ChatWindow', () => (
<ChatWindow userId="test@test.com" />
))

11
frontend/app/components/Assist/components/AssistActions/AassistActions.css Normal file
View file

@ -0,0 +1,11 @@
.inCall {
& svg {
fill: $red
}
color: $red;
}
.disabled {
opacity: 0.5;
pointer-events: none;
}

103
frontend/app/components/Assist/components/AssistActions/AssistActions.tsx Normal file
View file

@ -0,0 +1,103 @@
import React, { useState, useEffect } from 'react'
import { Popup, Icon } from 'UI'
import { connect } from 'react-redux'
import cn from 'classnames'
import { toggleChatWindow } from 'Duck/sessions';
import { connectPlayer } from 'Player/store';
import ChatWindow from '../../ChatWindow';
import { callPeer } from 'Player'
import { CallingState, ConnectionStatus } from 'Player/MessageDistributor/managers/AssistManager';
import { toast } from 'react-toastify';
import stl from './AassistActions.css'
interface Props {
userId: String,
toggleChatWindow: (state) => void,
calling: CallingState,
peerConnectionStatus: ConnectionStatus
}
function AssistActions({ toggleChatWindow, userId, calling, peerConnectionStatus }: Props) {
const [ incomeStream, setIncomeStream ] = useState<MediaStream | null>(null);
const [ localStream, setLocalStream ] = useState<MediaStream | null>(null);
const [ endCall, setEndCall ] = useState<()=>void>(()=>{});
useEffect(() => {
return endCall
}, [])
useEffect(() => {
if (peerConnectionStatus == ConnectionStatus.Disconnected) {
toast.info(`Live session was closed.`);
}
}, [peerConnectionStatus])
function onClose(stream) {
stream.getTracks().forEach(t=>t.stop());
}
function onReject() {
toast.info(`Call was rejected.`);
}
function onError() {
toast.error(`Something went wrong!`);
}
function call() {
navigator.mediaDevices.getUserMedia({video:true, audio:true})
.then(lStream => {
setLocalStream(lStream);
setEndCall(() => callPeer(
lStream,
setIncomeStream,
onClose.bind(null, lStream),
onReject,
onError
));
}).catch(onError);
}
const inCall = calling !== CallingState.False;
return (
<div className="flex items-center">
<Popup
trigger={
<div
className={
cn(
'cursor-pointer p-2 mr-2 flex items-center',
{[stl.inCall] : inCall },
{[stl.disabled]: peerConnectionStatus !== ConnectionStatus.Connected}
)
}
onClick={ inCall ? endCall : call}
role="button"
>
<Icon
name="headset"
size="20"
color={ inCall ? "red" : "gray-darkest" }
/>
<span className={cn("ml-2", { 'text-red' : inCall })}>{ inCall ? 'End Call' : 'Call' }</span>
</div>
}
content={ `Call ${userId}` }
size="tiny"
inverted
position="top right"
/>
<div className="fixed ml-3 left-0 top-0" style={{ zIndex: 999 }}>
{ inCall && <ChatWindow endCall={endCall} userId={userId} incomeStream={incomeStream} localStream={localStream} /> }
</div>
</div>
)
}
const con = connect(null, { toggleChatWindow })
export default con(connectPlayer(state => ({
calling: state.calling,
peerConnectionStatus: state.peerConnectionStatus,
}))(AssistActions))

1
frontend/app/components/Assist/components/AssistActions/index.ts Normal file
View file

@ -0,0 +1 @@
export { default } from './AssistActions'

0
ee/api/chalicelib/ee/__init__.py → frontend/app/components/Assist/components/VideoContainer/VideoContainer.css
View file

25
frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx Normal file
View file

@ -0,0 +1,25 @@
import React, { useEffect, useRef } from 'react'
interface Props {
stream: MediaStream | null
muted?: boolean,
width?: number
}
function VideoContainer({ stream, muted = false, width = 280 }: Props) {
const ref = useRef<HTMLVideoElement>(null);
useEffect(() => {
if (ref.current) {
ref.current.srcObject = stream;
}
}, [ ref.current, stream ])
return (
<div>
<video autoPlay ref={ ref } muted={ muted } style={{ width: width }} />
</div>
)
}
export default VideoContainer

1
frontend/app/components/Assist/components/VideoContainer/index.ts Normal file
View file

@ -0,0 +1 @@
export { default } from './VideoContainer'

1
frontend/app/components/Assist/index.ts Normal file
View file

@ -0,0 +1 @@
export { default } from './Assist'

12
frontend/app/components/BugFinder/BugFinder.js
View file

@ -25,6 +25,7 @@ import { LAST_7_DAYS } from 'Types/app/period';
import { resetFunnel } from 'Duck/funnels';
import { resetFunnelFilters } from 'Duck/funnelFilters'
import NoSessionsMessage from '../shared/NoSessionsMessage';
import LiveSessionList from './LiveSessionList'
const AUTOREFRESH_INTERVAL = 10 * 60 * 1000;
@ -134,7 +135,6 @@ export default class BugFinder extends React.PureComponent {
setActiveTab = tab => {
this.props.setActiveTab(tab);
}
render() {
@ -157,12 +157,10 @@ export default class BugFinder extends React.PureComponent {
className="mb-5"
>
<EventFilter />
</div>
{activeFlow && activeFlow.type === 'flows' ?
<FunnelList />
:
<SessionList onMenuItemClick={this.setActiveTab} />
}
</div>
{ activeFlow && activeFlow.type === 'flows' && <FunnelList /> }
{ activeTab.type !== 'live' && <SessionList onMenuItemClick={this.setActiveTab} /> }
{ activeTab.type === 'live' && <LiveSessionList /> }
</div>
</div>
<RehydrateSlidePanel

45
frontend/app/components/BugFinder/CustomFilters/FilterModal.js
View file

@ -21,6 +21,7 @@ const customFilterAutoCompleteKeys = ['METADATA', KEYS.CLICK, KEYS.USER_BROWSER,
customFilters: state.getIn([ 'filters', 'customFilters' ]),
variables: state.getIn([ 'customFields', 'list' ]),
sources: state.getIn([ 'customFields', 'sources' ]),
activeTab: state.getIn([ 'sessions', 'activeTab', 'type' ]),
}), {
applyFilter,
setActiveKey,
@ -81,10 +82,11 @@ export default class FilterModal extends React.PureComponent {
};
renderList(type, list) {
const { activeTab } = this.props;
const blocks = [];
for (let j = 0; j < list.length; j++) {
blocks.push(
<div key={`${ j }-block`} className="mr-5" >
<div key={`${ j }-block`} className={cn("mr-5", { [stl.disabled]: activeTab === 'live' && list[j].key !== 'USERID' })} >
{ list[ j ] && this.renderFilterItem(type, list[ j ]) }
</div>
);
@ -136,6 +138,7 @@ export default class FilterModal extends React.PureComponent {
loading = false,
searchedEvents,
searchQuery = '',
activeTab,
} = this.props;
const { query } = this.state;
const reg = getRE(query, 'i');
@ -158,6 +161,8 @@ export default class FilterModal extends React.PureComponent {
const staticFilters = preloadedFilters
.filter(({ value, actualValue }) => !this.props.loading && this.test(actualValue || value))
// console.log('filteredList', filteredList);
return (!displayed ? null :
<div className={ stl.modal }>
{ loading &&
@ -173,22 +178,26 @@ export default class FilterModal extends React.PureComponent {
{ searchQuery &&
<React.Fragment>
{this.renderEventDropdownPart(TYPES.USERID, 'User Id')}
{this.renderEventDropdownPart(TYPES.METADATA, 'Metadata')}
{this.renderEventDropdownPart(TYPES.CONSOLE, 'Errors')}
{this.renderEventDropdownPart(TYPES.CUSTOM, 'Custom Events')}
{this.renderEventDropdownPart(KEYS.USER_COUNTRY, 'Country', _appliedFilterKeys)}
{this.renderEventDropdownPart(KEYS.USER_BROWSER, 'Browser', _appliedFilterKeys)}
{this.renderEventDropdownPart(KEYS.USER_DEVICE, 'Device', _appliedFilterKeys)}
{this.renderEventDropdownPart(TYPES.LOCATION, 'Page')}
{this.renderEventDropdownPart(TYPES.CLICK, 'Click')}
{this.renderEventDropdownPart(TYPES.FETCH, 'Fetch')}
{this.renderEventDropdownPart(TYPES.INPUT, 'Input')}
{this.renderEventDropdownPart(KEYS.USER_OS, 'Operating System', _appliedFilterKeys)}
{this.renderEventDropdownPart(KEYS.REFERRER, 'Referrer', _appliedFilterKeys)}
{this.renderEventDropdownPart(TYPES.GRAPHQL, 'GraphQL')}
{this.renderEventDropdownPart(TYPES.STATEACTION, 'Store Action')}
{this.renderEventDropdownPart(TYPES.REVID, 'Rev ID')}
{activeTab !== 'live' && (
<>
{this.renderEventDropdownPart(TYPES.METADATA, 'Metadata')}
{this.renderEventDropdownPart(TYPES.CONSOLE, 'Errors')}
{this.renderEventDropdownPart(TYPES.CUSTOM, 'Custom Events')}
{this.renderEventDropdownPart(KEYS.USER_COUNTRY, 'Country', _appliedFilterKeys)}
{this.renderEventDropdownPart(KEYS.USER_BROWSER, 'Browser', _appliedFilterKeys)}
{this.renderEventDropdownPart(KEYS.USER_DEVICE, 'Device', _appliedFilterKeys)}
{this.renderEventDropdownPart(TYPES.LOCATION, 'Page')}
{this.renderEventDropdownPart(TYPES.CLICK, 'Click')}
{this.renderEventDropdownPart(TYPES.FETCH, 'Fetch')}
{this.renderEventDropdownPart(TYPES.INPUT, 'Input')}
{this.renderEventDropdownPart(KEYS.USER_OS, 'Operating System', _appliedFilterKeys)}
{this.renderEventDropdownPart(KEYS.REFERRER, 'Referrer', _appliedFilterKeys)}
{this.renderEventDropdownPart(TYPES.GRAPHQL, 'GraphQL')}
{this.renderEventDropdownPart(TYPES.STATEACTION, 'Store Action')}
{this.renderEventDropdownPart(TYPES.REVID, 'Rev ID')}
</>
)}
</React.Fragment>
}
</div>
@ -201,7 +210,7 @@ export default class FilterModal extends React.PureComponent {
<div className={ stl.list }>
{ this.renderList(category.type, category.keys) }
</div>
</div>
</div>
))
}
</div>

5
frontend/app/components/BugFinder/CustomFilters/filterModal.css
View file

@ -88,4 +88,9 @@ h5.title {
& .filterGroup {
width: 205px;
}
}
.disabled {
opacity: 0.5;
pointer-events: none;
}

61
frontend/app/components/BugFinder/LiveSessionList/LiveSessionList.tsx Normal file
View file

@ -0,0 +1,61 @@
import React, { useEffect, useState } from 'react';
import { fetchLiveList } from 'Duck/sessions';
import { connect } from 'react-redux';
import { NoContent, Loader } from 'UI';
import { List, Map } from 'immutable';
import SessionItem from 'Shared/SessionItem';
interface Props {
loading: Boolean,
list?: List<any>,
fetchLiveList: () => void,
filters: List<any>
}
function LiveSessionList(props: Props) {
const { loading, list, filters } = props;
const [userId, setUserId] = useState(undefined)
useEffect(() => {
props.fetchLiveList();
}, [])
useEffect(() => {
if (filters) {
const userIdFilter = filters.filter(i => i.key === 'USERID').first()
if (userIdFilter)
setUserId(userIdFilter.value[0])
else
setUserId(undefined)
}
}, [filters])
return (
<div>
<NoContent
title={"No live sessions!"}
subtext="Please try changing your search parameters."
icon="exclamation-circle"
show={ !loading && list && list.size === 0}
>
<Loader loading={ loading }>
{list && (userId ? list.filter(i => i.userId === userId) : list).map(session => (
<SessionItem
key={ session.sessionId }
session={ session }
live
// hasUserFilter={hasUserFilter}
/>
))}
</Loader>
</NoContent>
</div>
)
}
export default connect(state => ({
list: state.getIn(['sessions', 'liveSessions']),
loading: state.getIn([ 'sessions', 'loading' ]),
filters: state.getIn([ 'filters', 'appliedFilter', 'filters' ]),
}), { fetchLiveList })(LiveSessionList)

1
frontend/app/components/BugFinder/LiveSessionList/index.js Normal file
View file

@ -0,0 +1 @@
export { default } from './LiveSessionList'

4
frontend/app/components/BugFinder/SessionList/SessionList.js
View file

@ -48,7 +48,7 @@ export default class SessionList extends React.PureComponent {
}
getNoContentMessage = activeTab => {
let str = "No Sessions Found";
let str = "No recordings found";
if (activeTab.type !== 'all') {
str += ' with ' + activeTab.name;
return str;
@ -123,7 +123,7 @@ export default class SessionList extends React.PureComponent {
const { activeTab, allList, total } = this.props;
var filteredList;
if (activeTab.type !== ALL && activeTab.type !== 'bookmark') { // Watchdog sessions
if (activeTab.type !== ALL && activeTab.type !== 'bookmark' && activeTab.type !== 'live') { // Watchdog sessions
filteredList = allList.filter(session => activeTab.fits(session))
} else {
filteredList = allList

13
frontend/app/components/BugFinder/SessionsMenu/SessionsMenu.js
View file

@ -72,6 +72,16 @@ function SessionsMenu(props) {
/>
))}
<div className={stl.divider} />
<div className="my-3">
<SideMenuitem
title="Assist"
iconName="person"
active={activeTab.type === 'live'}
onClick={() => onMenuItemClick({ name: 'Assist', type: 'live' })}
/>
</div>
<div className={stl.divider} />
<div className="my-3">
<SideMenuitem
@ -80,7 +90,8 @@ function SessionsMenu(props) {
active={activeTab.type === 'bookmark'}
onClick={() => onMenuItemClick({ name: 'Bookmarks', type: 'bookmark' })}
/>
</div>
</div>
<div className={cn(stl.divider, 'mb-4')} />
<SavedSearchList />
</div>

31
frontend/app/components/Client/ProfileSettings/ProfileSettings.js
View file

@ -3,12 +3,18 @@ import Settings from './Settings';
import ChangePassword from './ChangePassword';
import styles from './profileSettings.css';
import Api from './Api';
import TenantKey from './TenantKey';
import OptOut from './OptOut';
import Licenses from './Licenses';
import { connect } from 'react-redux';
@withPageTitle('Account - OpenReplay Preferences')
@connect(state => ({
account: state.getIn([ 'user', 'account' ]),
}))
export default class ProfileSettings extends React.PureComponent {
render() {
render() {
const { account } = this.props;
return (
<React.Fragment>
<div className="flex items-center">
@ -43,20 +49,33 @@ export default class ProfileSettings extends React.PureComponent {
<div className="flex items-center">
<div className={ styles.left }>
<h4 className="text-lg mb-4">{ 'Data Collection' }</h4>
<div className={ styles.info }>{ 'Enables you to control how OpenReplay captures data on your organization’s usage to improve our product.' }</div>
<h4 className="text-lg mb-4">{ 'Tenant Key' }</h4>
<div className={ styles.info }>{ 'For SSO (SAML) authentication.' }</div>
</div>
<div><OptOut /></div>
<div><TenantKey /></div>
</div>
<div className="divider" />
<div className="flex items-center">
<div className={ styles.left }>
<h4 className="text-lg mb-4">{ 'License' }</h4>
<h4 className="text-lg mb-4">{ 'Data Collection' }</h4>
<div className={ styles.info }>{ 'Enables you to control how OpenReplay captures data on your organization’s usage to improve our product.' }</div>
</div>
<div><Licenses /></div>
<div><OptOut /></div>
</div>
{ account.license && (
<>
<div className="divider" />
<div className="flex items-center">
<div className={ styles.left }>
<h4 className="text-lg mb-4">{ 'License' }</h4>
</div>
<div><Licenses /></div>
</div>
</>
)}
</React.Fragment>
);
}

51
frontend/app/components/Client/ProfileSettings/TenantKey.js Normal file
View file

@ -0,0 +1,51 @@
// TODO this can be deleted
import copy from 'copy-to-clipboard';
import { connect } from 'react-redux';
import styles from './profileSettings.css';
@connect(state => ({
key: state.getIn([ 'user', 'client', 'tenantKey' ]),
loading: state.getIn([ 'user', 'updateAccountRequest', 'loading' ]) ||
state.getIn([ 'user', 'putClientRequest', 'loading' ]),
}))
export default class TenantKey extends React.PureComponent {
state = { copied: false }
copyHandler = () => {
const { key } = this.props;
this.setState({ copied: true });
copy(key);
setTimeout(() => {
this.setState({ copied: false });
}, 1000);
};
render() {
const { key } = this.props;
const { copied } = this.state;
return (
<form onSubmit={ this.handleSubmit } className={ styles.form }>
<div className={ styles.formGroup }>
<label htmlFor="key">{ 'Tenant Key' }</label>
<div className="ui action input">
<input
name="key"
id="key"
type="text"
readOnly={ true }
value={ key }
/>
<div
className="ui button copy-button"
role="button"
onClick={ this.copyHandler }
>
{ copied ? 'copied' : 'copy' }
</div>
</div>
</div>
</form>
);
}
}

13
frontend/app/components/Client/Sites/NewSiteForm.js
View file

@ -1,7 +1,7 @@
import { connect } from 'react-redux';
import { Input, Button, Label } from 'UI';
import { save, edit, update , fetchList } from 'Duck/site';
import { pushNewSite } from 'Duck/user';
import { pushNewSite, setSiteId } from 'Duck/user';
import styles from './siteForm.css';
@connect(state => ({
@ -14,7 +14,8 @@ import styles from './siteForm.css';
edit,
update,
pushNewSite,
fetchList
fetchList,
setSiteId
})
export default class NewSiteForm extends React.PureComponent {
state = {
@ -34,8 +35,12 @@ export default class NewSiteForm extends React.PureComponent {
})
} else {
this.props.save(this.props.site).then(() => {
const { sites } = this.props;
this.props.onClose(null, sites.last())
const { sites } = this.props;
const site = sites.last();
this.props.pushNewSite(site)
this.props.setSiteId(site.id)
this.props.onClose(null, site)
});
}
}

2
frontend/app/components/Funnels/FunnelSessionList/FunnelSessionList.js
View file

@ -20,7 +20,7 @@ function FunnelSessionList(props) {
<FunnelSessionsHeader sessionsCount={inDetails ? sessionsTotal : list.size} inDetails={inDetails} />
<div className="mb-4" />
<NoContent
title="No Sessions Found!"
title="No recordings found!"
subtext="Please try changing your search parameters."
icon="exclamation-circle"
show={ list.size === 0}

Some files were not shown because too many files have changed in this diff Show more