jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #5 from openreplay/dev

Browse source
This commit is contained in:
ShiKhu 2021-05-16 18:38:24 +02:00 • committed by GitHub
commit c58bc38b37
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 95 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
api/.chalice/config.json
View file

@ -6,7 +6,7 @@
},
"stages": {
"default-foss": {
"api_gateway_stage": "default-foss",
"api_gateway_stage": "default-fos",
"manage_iam_role": false,
"iam_role_arn": "",
"autogen_policy": true,
@ -28,14 +28,14 @@
"assign_link": "http://127.0.0.1:8000/async/email_assignment",
"captcha_server": "",
"captcha_key": "",
"sessions_bucket": "",
"sessions_region": "",
"sessions_bucket": "asayer-mobs",
"sessions_region": "us-east-1",
"put_S3_TTL": "20",
"sourcemaps_bucket": "",
"sourcemaps_bucket": "asayer-sourcemaps",
"sourcemaps_bucket_key": "",
"sourcemaps_bucket_secret": "",
"sourcemaps_bucket_region": "",
"js_cache_bucket": "",
"sourcemaps_bucket_region": "us-east-1",
"js_cache_bucket": "asayer-sessions-assets",
"async_Token": "",
"EMAIL_HOST": "",
"EMAIL_PORT": "587",

3
api/build.sh
View file

@ -25,6 +25,9 @@ function build_api(){
envarg="default-ee"
}
docker build -f ./Dockerfile --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/chalice:${git_sha1} .
[[ $PUSH_IMAGE -eq 1 ]] && {
docker push ${DOCKER_REPO:-'local'}/chalice:${git_sha1}
}
}
check_prereq

29
api/chalicelib/core/dashboard.py
View file

@ -82,7 +82,6 @@ def __get_meta_constraint(project_id, data):
constraints = []
meta_keys = metadata.get(project_id=project_id)
meta_keys = {m["key"]: m["index"] for m in meta_keys}
print(meta_keys)
for i, f in enumerate(data.get("filters", [])):
if f["key"] in meta_keys.keys():
@ -93,20 +92,27 @@ def __get_meta_constraint(project_id, data):
constraints.append(f"{key} = %({f['key']}_{i})s")
else:
filter_type = f["key"].upper()
if filter_type == sessions_metas.meta_type.USERBROWSER:
filter_type = [filter_type, "USER" + filter_type, filter_type[4:]]
if any(item in [sessions_metas.meta_type.USERBROWSER] \
for item in filter_type):
constraints.append(f"sessions.user_browser = %({f['key']}_{i})s")
elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]:
elif any(item in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS] \
for item in filter_type):
constraints.append(f"sessions.user_os = %({f['key']}_{i})s")
elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]:
elif any(item in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS] \
for item in filter_type):
constraints.append(f"sessions.user_device = %({f['key']}_{i})s")
elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]:
elif any(item in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS] \
for item in filter_type):
constraints.append(f"sessions.user_country = %({f['key']}_{i})s")
elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]:
elif any(item in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS] \
for item in filter_type):
constraints.append(f"sessions.user_id = %({f['key']}_{i})s")
elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID,
sessions_metas.meta_type.USERANONYMOUSID_IOS]:
elif any(item in [sessions_metas.meta_type.USERANONYMOUSID, sessions_metas.meta_type.USERANONYMOUSID_IOS] \
for item in filter_type):
constraints.append(f"sessions.user_anonymous_id = %({f['key']}_{i})s")
elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]:
elif any(item in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS] \
for item in filter_type):
constraints.append(f"sessions.rev_id = %({f['key']}_{i})s")
return constraints
@ -140,6 +146,7 @@ def get_processed_sessions(project_id, startTimestamp=TimeUTC.now(delta_days=-1)
ORDER BY generated_timestamp;"""
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)}
print(cur.mogrify(pg_query, params))
cur.execute(cur.mogrify(pg_query, params))
rows = cur.fetchall()
results = {
@ -634,8 +641,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False,
WHERE {" AND ".join(pg_sub_query)}
LIMIT 10;"""
print(cur.mogrify(pg_query, {"project_id": project_id,
"value": helper.string_to_sql_like(text),
"platform_0": platform}))
"value": helper.string_to_sql_like(text),
"platform_0": platform}))
cur.execute(cur.mogrify(pg_query, {"project_id": project_id,
"value": helper.string_to_sql_like(text),
"platform_0": platform}))

10
api/chalicelib/core/sessions_mobs.py
View file

@ -4,11 +4,11 @@ import boto3
def get_web(sessionId):
return boto3.client('s3',
endpoint_url=environ["S3_HOST"],
aws_access_key_id=environ["S3_KEY"],
aws_secret_access_key=environ["S3_SECRET"],
region_name=environ["sessions_region"]).generate_presigned_url(
return boto3.client('s3',
endpoint_url=environ["S3_HOST"],
aws_access_key_id=environ["S3_KEY"],
aws_secret_access_key=environ["S3_SECRET"],
region_name=environ["sessions_region"]).generate_presigned_url(
'get_object',
Params={
'Bucket': environ["sessions_bucket"],

6
backend/build.sh
View file

@ -27,11 +27,17 @@ function build_api(){
[[ $2 != "" ]] && {
image="$2"
docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image .
[[ $PUSH_IMAGE -eq 1 ]] && {
docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1}
}
return
}
for image in $(ls services);
do
docker build -t ${DOCKER_REPO:-'local'}/$image:${git_sha1} --build-arg SERVICE_NAME=$image .
[[ $PUSH_IMAGE -eq 1 ]] && {
docker push ${DOCKER_REPO:-'local'}/$image:${git_sha1}
}
echo "::set-output name=image::${DOCKER_REPO:-'local'}/$image:${git_sha1}"
done
}

6
backend/services/storage/main.go
View file

@ -23,7 +23,7 @@ func main() {
storageWeb := storage.NewS3(env.String("AWS_REGION_WEB"), env.String("S3_BUCKET_WEB"))
storageIos := storage.NewS3(env.String("AWS_REGION_IOS"), env.String("S3_BUCKET_IOS"))
//storageIos := storage.NewS3(env.String("AWS_REGION_IOS"), env.String("S3_BUCKET_IOS"))
FS_DIR := env.String("FS_DIR") + "/"
var uploadKey func(string, int, *storage.S3)
@ -54,8 +54,8 @@ func main() {
switch msg.(type) {
case *messages.SessionEnd:
uploadKey(strconv.FormatUint(sessionID, 10), 5, storageWeb)
case *messages.IOSSessionEnd:
uploadKey(strconv.FormatUint(sessionID, 10), 5, storageIos)
//case *messages.IOSSessionEnd:
// uploadKey(strconv.FormatUint(sessionID, 10), 5, storageIos)
}
},
)

16
ee/api/.chalice/config.json
View file

@ -31,14 +31,14 @@
"assign_link": "http://127.0.0.1:8000/async/email_assignment",
"captcha_server": "",
"captcha_key": "",
"sessions_bucket": "asayer-mobs-staging",
"sessions_region": "eu-central-1",
"sessions_bucket": "asayer-mobs",
"sessions_region": "us-east-1",
"put_S3_TTL": "20",
"sourcemaps_bucket": "",
"sourcemaps_bucket": "asayer-sourcemaps",
"sourcemaps_bucket_key": "",
"sourcemaps_bucket_secret": "",
"sourcemaps_bucket_region": "",
"js_cache_bucket": "",
"sourcemaps_bucket_region": "us-east-1",
"js_cache_bucket": "asayer-sessions-assets",
"async_Token": "",
"EMAIL_HOST": "",
"EMAIL_PORT": "587",
@ -51,9 +51,9 @@
"EMAIL_FROM": "OpenReplay<do-not-reply@openreplay.com>",
"SITE_URL": "",
"announcement_url": "",
"jwt_secret": "",
"jwt_algorithm": "",
"jwt_exp_delta_seconds": "",
"jwt_secret": "SET A RANDOM STRING HERE",
"jwt_algorithm": "HS512",
"jwt_exp_delta_seconds": "2592000",
"S3_HOST": "",
"S3_KEY": "",
"S3_SECRET": ""

16
ee/api/chalicelib/utils/s3.py
View file

@ -1,19 +1,15 @@
from botocore.exceptions import ClientError
from chalicelib.utils.helper import environ
from chalicelib.utils import helper
import boto3
if helper.is_free_open_source_edition() or helper.is_enterprise_edition():
from botocore.client import Config
from botocore.client import Config
client = boto3.client('s3', endpoint_url=environ["S3_HOST"],
aws_access_key_id=environ["S3_KEY"],
aws_secret_access_key=environ["S3_SECRET"],
config=Config(signature_version='s3v4'),
region_name='us-east-1')
else:
client = boto3.client('s3')
client = boto3.client('s3', endpoint_url=environ["S3_HOST"],
aws_access_key_id=environ["S3_KEY"],
aws_secret_access_key=environ["S3_SECRET"],
config=Config(signature_version='s3v4'),
region_name='us-east-1')
def exists(bucket, key):

10
frontend/app/player/MessageDistributor/MessageDistributor.js
View file

@ -164,6 +164,16 @@ export default class MessageDistributor extends StatedScreen {
let mCount = 0;
const msgs = [];
while (mGen.hasNext()) {
mCount++;
const next = mGen.next();
if (next != null) {
this.#lastMessageTime = next[0].time;
this.#distributeMessage(next[0], next[1]);
msgs.push(next[0]);
}
}
// Hack for upet (TODO: fix ordering in one mutation (removes first))
const headChildrenIds = msgs.filter(m => m.parentID === 1).map(m => m.id);
//const createNodeTypes = ["create_text_node", "create_element_node"];

1
scripts/helm/app/assets.yaml
View file

@ -24,6 +24,7 @@ resources:
env:
ASSETS_ORIGIN: /asayer-sessions-assets # TODO: full path (with the minio prefix)
S3_BUCKET_ASSETS: asayer-sessions-assets
AWS_ENDPOINT: http://minio.db.svc.cluster.local:9000
AWS_ACCESS_KEY_ID: "minios3AccessKeyS3cr3t"
AWS_SECRET_ACCESS_KEY: "m1n10s3CretK3yPassw0rd"
AWS_REGION: us-east-1

14
scripts/helm/app/chalice.yaml
View file

@ -37,12 +37,12 @@ env:
captcha_server: ''
captcha_key: ''
sessions_bucket: asayer-mobs
sessions_region: eu-central-1
sessions_region: us-east-1
put_S3_TTL: '20'
sourcemaps_bucket: asayer-sourcemaps
sourcemaps_bucket_key: minios3AccessKeyS3cr3t
sourcemaps_bucket_secret: m1n10s3CretK3yPassw0rd
sourcemaps_bucket_region: eu-central-1
sourcemaps_bucket_secret: m1n10s3CretK3yPassw0rd
sourcemaps_bucket_region: us-east-1
js_cache_bucket: asayer-sessions-assets
async_Token: ''
EMAIL_HOST: ''
@ -56,10 +56,12 @@ env:
EMAIL_FROM: OpenReplay<do-not-reply@openreplay.com>
SITE_URL: ''
announcement_url: ''
jwt_secret: ''
jwt_algorithm: 'HS512'
jwt_secret: SET A RANDOM STRING HERE
jwt_algorithm: HS512
jwt_exp_delta_seconds: '2592000'
S3_HOST: 's3://minio.db.svc.cluster.local:9000'
# Override with your https://domain_name
# eg: https://openreplay.mycompany.com
S3_HOST: ''
S3_KEY: minios3AccessKeyS3cr3t
S3_SECRET: m1n10s3CretK3yPassw0rd
# Enable logging for python app

1
scripts/helm/app/storage.yaml
View file

@ -31,6 +31,7 @@ pvc:
storageSize: 5Gi
env:
AWS_ENDPOINT: http://minio.db.svc.cluster.local:9000
AWS_ACCESS_KEY_ID: "minios3AccessKeyS3cr3t"
AWS_SECRET_ACCESS_KEY: "m1n10s3CretK3yPassw0rd"
AWS_REGION_WEB: eu-central-1

18
scripts/helm/build_deploy.sh Normal file
View file

@ -0,0 +1,18 @@
#!/bin/bash
set -e
# This script will build and push docker image to registry
# Usage: IMAGE_TAG=latest DOCKER_REPO=rg.fr-par.scw.cloud/foss bash build_deploy.sh
echo $DOCKER_REPO
[[ -z DOCKER_REPO ]] && {
echo Set DOCKER_REPO="your docker registry"
exit 1
} || {
docker login $DOCKER_REPO
cd ../../api
PUSH_IMAGE=1 bash build.sh
cd ../backend
PUSH_IMAGE=1 bash build.sh
}

2
scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml
View file

@ -8,7 +8,7 @@ data:
location /healthz {
return 200 'OK';
}
location ~ ^/(asayer-mobs|asayer-sessions-assets|frontend|static)/ {
location ~ ^/(asayer-mobs|asayer-sessions-assets|frontend|static|asayer-sourcemaps)/ {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;

1
scripts/helm/roles/openreplay/templates/chalice.yaml
View file

@ -12,3 +12,4 @@ env:
S3_SECRET: "{{ minio_secret_key }}"
sourcemaps_bucket_key: "{{ minio_access_key }}"
sourcemaps_bucket_secret: "{{ minio_secret_key }}"
S3_HOST: "https://{{ domain_name }}"