diff --git a/api/.chalice/config.bundle.json b/api/.chalice/config.bundle.json new file mode 100644 index 000000000..32370e574 --- /dev/null +++ b/api/.chalice/config.bundle.json @@ -0,0 +1,67 @@ +{ + "version": "2.0", + "app_name": "parrot", + "environment_variables": { + }, + "stages": { + "default-foss": { + "api_gateway_stage": "default-fos", + "manage_iam_role": false, + "iam_role_arn": "", + "autogen_policy": true, + "environment_variables": { + "isFOS": "true", + "isEE": "false", + "stage": "default-foss", + "jwt_issuer": "openreplay-default-foss", + "sentryURL": "", + "pg_host": "postgresql.db.svc.cluster.local", + "pg_port": "5432", + "pg_dbname": "postgres", + "pg_user": "postgres", + "pg_password": "asayerPostgres", + "alert_ntf": "http://127.0.0.1:8000/async/alerts/notifications/%s", + "email_signup": "http://127.0.0.1:8000/async/email_signup/%s", + "email_funnel": "http://127.0.0.1:8000/async/funnel/%s", + "email_basic": "http://127.0.0.1:8000/async/basic/%s", + "assign_link": "http://127.0.0.1:8000/async/email_assignment", + "captcha_server": "", + "captcha_key": "", + "sessions_bucket": "mobs", + "sessions_region": "us-east-1", + "put_S3_TTL": "20", + "sourcemaps_reader": "http://0.0.0.0:9000/sourcemaps", + "sourcemaps_bucket": "sourcemaps", + "js_cache_bucket": "sessions-assets", + "peers": "http://0.0.0.0:9000/assist/peers", + "async_Token": "", + "EMAIL_HOST": "", + "EMAIL_PORT": "587", + "EMAIL_USER": "", + "EMAIL_PASSWORD": "", + "EMAIL_USE_TLS": "true", + "EMAIL_USE_SSL": "false", + "EMAIL_SSL_KEY": "", + "EMAIL_SSL_CERT": "", + "EMAIL_FROM": "OpenReplay", + "SITE_URL": "", + "announcement_url": "", + "jwt_secret": "", + "jwt_algorithm": "HS512", + "jwt_exp_delta_seconds": "2592000", + "S3_HOST": "", + "S3_KEY": "", + "S3_SECRET": "", + "invitation_link": "/api/users/invitation?token=%s", + "change_password_link": "/reset-password?invitation=%s&&pass=%s", + "version_number": "1.2.0" + }, + "lambda_timeout": 150, + "lambda_memory_size": 400, + "subnet_ids": [ + ], + "security_group_ids": [ + ] + } + } +} diff --git a/api/.chalice/config.json b/api/.chalice/config.json index 5cc01aa47..c3bd72cb6 100644 --- a/api/.chalice/config.json +++ b/api/.chalice/config.json @@ -64,4 +64,4 @@ ] } } -} +} \ No newline at end of file diff --git a/api/Dockerfile b/api/Dockerfile index 4dbfe4f7f..c9c0eaf2c 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -13,4 +13,4 @@ ENV ENTERPRISE_BUILD ${envarg} ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini RUN chmod +x /tini ENTRYPOINT ["/tini", "--"] -CMD ./entrypoint.sh +CMD ./entrypoint.sh \ No newline at end of file diff --git a/api/Dockerfile.bundle b/api/Dockerfile.bundle new file mode 100644 index 000000000..54709db89 --- /dev/null +++ b/api/Dockerfile.bundle @@ -0,0 +1,27 @@ +FROM python:3.6-slim +LABEL Maintainer="Rajesh Rajendran" +WORKDIR /work +COPY . . +COPY ../utilities ./utilities +RUN rm entrypoint.sh && rm .chalice/config.json +RUN mv entrypoint.bundle.sh entrypoint.sh && mv .chalice/config.bundle.json .chalice/config.json +RUN pip install -r requirements.txt -t ./vendor --upgrade +RUN pip install chalice==1.22.2 +# Installing Nodejs +RUN apt update && apt install -y curl && \ + curl -fsSL https://deb.nodesource.com/setup_12.x | bash - && \ + apt install -y nodejs && \ + apt remove --purge -y curl && \ + rm -rf /var/lib/apt/lists/* && \ + cd utilities && \ + npm install + +# Add Tini +# Startup daemon +ENV TINI_VERSION v0.19.0 +ARG envarg +ENV ENTERPRISE_BUILD ${envarg} +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] +CMD ./entrypoint.sh diff --git a/api/chalicelib/blueprints/bp_core.py b/api/chalicelib/blueprints/bp_core.py index 14110f383..e2ef487ed 100644 --- a/api/chalicelib/blueprints/bp_core.py +++ b/api/chalicelib/blueprints/bp_core.py @@ -884,6 +884,15 @@ def sessions_live(projectId, context): return {'data': data} +@app.route('/{projectId}/assist/sessions', methods=['POST']) +def sessions_live_search(projectId, context): + data = app.current_request.json_body + if data is None: + data = {} + data = assist.get_live_sessions(projectId, filters=data.get("filters")) + return {'data': data} + + @app.route('/{projectId}/heatmaps/url', methods=['POST']) def get_heatmaps_by_url(projectId, context): data = app.current_request.json_body diff --git a/api/chalicelib/core/assist.py b/api/chalicelib/core/assist.py index 2e17bd516..18978e159 100644 --- a/api/chalicelib/core/assist.py +++ b/api/chalicelib/core/assist.py @@ -1,5 +1,5 @@ from chalicelib.utils import pg_client, helper -from chalicelib.core import projects +from chalicelib.core import projects, sessions, sessions_metas import requests from chalicelib.utils.helper import environ @@ -19,7 +19,7 @@ SESSION_PROJECTION_COLS = """s.project_id, """ -def get_live_sessions(project_id): +def get_live_sessions(project_id, filters=None): project_key = projects.get_project_key(project_id) connected_peers = requests.get(environ["peers"] + f"/{project_key}") if connected_peers.status_code != 200: @@ -31,13 +31,31 @@ def get_live_sessions(project_id): if len(connected_peers) == 0: return [] connected_peers = tuple(connected_peers) + extra_constraints = ["project_id = %(project_id)s", "session_id IN %(connected_peers)s"] + extra_params = {} + if filters is not None: + for i, f in enumerate(filters): + if not isinstance(f.get("value"), list): + f["value"] = [f.get("value")] + if len(f["value"]) == 0 or f["value"][0] is None: + continue + filter_type = f["type"].upper() + f["value"] = sessions.__get_sql_value_multiple(f["value"]) + if filter_type == sessions_metas.meta_type.USERID: + op = sessions.__get_sql_operator(f["operator"]) + extra_constraints.append(f"user_id {op} %(value_{i})s") + extra_params[f"value_{i}"] = helper.string_to_sql_like_with_op(f["value"][0], op) + with pg_client.PostgresClient() as cur: query = cur.mogrify(f"""\ SELECT {SESSION_PROJECTION_COLS}, %(project_key)s||'-'|| session_id AS peer_id FROM public.sessions AS s - WHERE s.project_id = %(project_id)s - AND session_id IN %(connected_peers)s;""", - {"project_id": project_id, "connected_peers": connected_peers, "project_key": project_key}) + WHERE {" AND ".join(extra_constraints)} + LIMIT 500;""", + {"project_id": project_id, + "connected_peers": connected_peers, + "project_key": project_key, + **extra_params}) cur.execute(query) results = cur.fetchall() return helper.list_to_camel_case(results) @@ -52,4 +70,4 @@ def is_live(project_id, session_id, project_key=None): print(connected_peers.text) return False connected_peers = connected_peers.json().get("data", []) - return session_id in connected_peers + return str(session_id) in connected_peers diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 8663aa89a..340733d30 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -99,7 +99,8 @@ def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_ data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['issues'] = issues.get_by_session_id(session_id=session_id) - data['live'] = assist.is_live(project_id=project_id, session_id=session_id, + data['live'] = assist.is_live(project_id=project_id, + session_id=session_id, project_key=data["projectKey"]) return data diff --git a/api/entrypoint.bundle.sh b/api/entrypoint.bundle.sh new file mode 100755 index 000000000..8478d9d14 --- /dev/null +++ b/api/entrypoint.bundle.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd utilities +nohup npm start &> /tmp/utilities.log & +cd .. +python env_handler.py +chalice local --no-autoreload --host 0.0.0.0 --stage ${ENTERPRISE_BUILD} diff --git a/backend/Dockerfile b/backend/Dockerfile index 43ea47d22..d2d94da64 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -25,6 +25,7 @@ ENV TZ=UTC \ MAXMINDDB_FILE=/root/geoip.mmdb \ UAPARSER_FILE=/root/regexes.yaml \ HTTP_PORT=80 \ + BEACON_SIZE_LIMIT=1000000 \ KAFKA_USE_SSL=true \ REDIS_STREAMS_MAX_LEN=3000 \ TOPIC_RAW=raw \ @@ -42,7 +43,7 @@ ENV TZ=UTC \ AWS_REGION_ASSETS=eu-central-1 \ CACHE_ASSETS=false \ ASSETS_SIZE_LIMIT=6291456 \ - FS_CLEAN_HRS=240 + FS_CLEAN_HRS=72 ARG SERVICE_NAME diff --git a/backend/Dockerfile.bundle b/backend/Dockerfile.bundle index 3105742cf..904bb45f4 100644 --- a/backend/Dockerfile.bundle +++ b/backend/Dockerfile.bundle @@ -26,6 +26,7 @@ ENV TZ=UTC \ MAXMINDDB_FILE=/root/geoip.mmdb \ UAPARSER_FILE=/root/regexes.yaml \ HTTP_PORT=80 \ + BEACON_SIZE_LIMIT=1000000 \ KAFKA_USE_SSL=true \ REDIS_STREAMS_MAX_LEN=3000 \ TOPIC_RAW=raw \ diff --git a/backend/services/ender/builder/deadClickDetector.go b/backend/services/ender/builder/deadClickDetector.go index 4f79f78bf..725b025cb 100644 --- a/backend/services/ender/builder/deadClickDetector.go +++ b/backend/services/ender/builder/deadClickDetector.go @@ -43,6 +43,9 @@ func (d *deadClickDetector) HandleMessage(msg Message, messageID uint64, timesta case *CreateDocument: d.inputIDSet = nil case *MouseClick: + if m.Label == "" { + return nil + } i = d.HandleReaction(timestamp) if d.inputIDSet[m.ID] { // ignore if input return i diff --git a/backend/services/http/assets.go b/backend/services/http/assets.go index 8ed625859..dd3dd4703 100644 --- a/backend/services/http/assets.go +++ b/backend/services/http/assets.go @@ -7,7 +7,7 @@ import ( func sendAssetForCache(sessionID uint64, baseURL string, relativeURL string) { if fullURL, cacheable := assets.GetFullCachableURL(baseURL, relativeURL); cacheable { - producer.Produce(topicTrigger, sessionID, messages.Encode(&messages.AssetCache{ + producer.Produce(TOPIC_TRIGGER, sessionID, messages.Encode(&messages.AssetCache{ URL: fullURL, })) } @@ -20,7 +20,7 @@ func sendAssetsForCacheFromCSS(sessionID uint64, baseURL string, css string) { } func handleURL(sessionID uint64, baseURL string, url string) string { - if cacheAssets { + if CACHE_ASSESTS { rewrittenURL, isCachable := rewriter.RewriteURL(sessionID, baseURL, url) if isCachable { sendAssetForCache(sessionID, baseURL, url) @@ -31,7 +31,7 @@ func handleURL(sessionID uint64, baseURL string, url string) string { } func handleCSS(sessionID uint64, baseURL string, css string) string { - if cacheAssets { + if CACHE_ASSESTS { sendAssetsForCacheFromCSS(sessionID, baseURL, css) return rewriter.RewriteCSS(sessionID, baseURL, css) } diff --git a/backend/services/http/handlers.go b/backend/services/http/handlers.go index 761aab764..e22a97abe 100644 --- a/backend/services/http/handlers.go +++ b/backend/services/http/handlers.go @@ -19,7 +19,6 @@ import ( ) const JSON_SIZE_LIMIT int64 = 1e3 // 1Kb -const BATCH_SIZE_LIMIT int64 = 1e6 // 1Mb func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { type request struct { @@ -90,7 +89,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { tokenData = &token.TokenData{sessionID, expTime.UnixNano() / 1e6} country := geoIP.ExtractISOCodeFromHTTPRequest(r) - producer.Produce(topicRaw, tokenData.ID, Encode(&SessionStart{ + producer.Produce(TOPIC_RAW, tokenData.ID, Encode(&SessionStart{ Timestamp: req.Timestamp, ProjectID: uint64(p.ProjectID), TrackerVersion: req.TrackerVersion, @@ -120,7 +119,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { } func pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64) { - body := http.MaxBytesReader(w, r.Body, BATCH_SIZE_LIMIT) + body := http.MaxBytesReader(w, r.Body, BEACON_SIZE_LIMIT) //defer body.Close() var reader io.ReadCloser switch r.Header.Get("Content-Encoding") { @@ -139,7 +138,7 @@ func pushMessages(w http.ResponseWriter, r *http.Request, sessionID uint64) { responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging return } - producer.Produce(topicRaw, sessionID, buf) // What if not able to send? + producer.Produce(TOPIC_RAW, sessionID, buf) // What if not able to send? w.WriteHeader(http.StatusOK) } @@ -158,7 +157,7 @@ func pushMessagesSeparatelyHandler(w http.ResponseWriter, r *http.Request) { responseWithError(w, http.StatusUnauthorized, err) return } - body := http.MaxBytesReader(w, r.Body, BATCH_SIZE_LIMIT) + body := http.MaxBytesReader(w, r.Body, BEACON_SIZE_LIMIT) //defer body.Close() buf, err := ioutil.ReadAll(body) if err != nil { @@ -234,8 +233,8 @@ func pushMessagesSeparatelyHandler(w http.ResponseWriter, r *http.Request) { responseWithError(w, http.StatusForbidden, err) return } - producer.Produce(topicRaw, sessionData.ID, rewritenBuf) - //producer.Produce(topicAnalytics, sessionData.ID, WriteBatch(analyticsMessages)) + producer.Produce(TOPIC_RAW, sessionData.ID, rewritenBuf) + //producer.Produce(TOPIC_ANALYTICS, sessionData.ID, WriteBatch(analyticsMessages)) //duration := time.Now().Sub(startTime) //log.Printf("Sended batch within %v nsec; %v nsek/byte", duration.Nanoseconds(), duration.Nanoseconds()/int64(len(buf))) w.WriteHeader(http.StatusOK) diff --git a/backend/services/http/main.go b/backend/services/http/main.go index 27a83548b..647dc42f5 100644 --- a/backend/services/http/main.go +++ b/backend/services/http/main.go @@ -31,20 +31,22 @@ var uaParser *uaparser.UAParser var geoIP *geoip.GeoIP var tokenizer *token.Tokenizer var s3 *storage.S3 -var topicRaw string -var topicTrigger string -var topicAnalytics string + +var TOPIC_RAW string +var TOPIC_TRIGGER string +var TOPIC_ANALYTICS string // var kafkaTopicEvents string -var cacheAssets bool +var CACHE_ASSESTS bool +var BEACON_SIZE_LIMIT int64 func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) producer = queue.NewProducer() defer producer.Close(15000) - topicRaw = env.String("TOPIC_RAW") - topicTrigger = env.String("TOPIC_TRIGGER") - topicAnalytics = env.String("TOPIC_ANALYTICS") + TOPIC_RAW = env.String("TOPIC_RAW") + TOPIC_TRIGGER = env.String("TOPIC_TRIGGER") + TOPIC_ANALYTICS = env.String("TOPIC_ANALYTICS") rewriter = assets.NewRewriter(env.String("ASSETS_ORIGIN")) pgconn = cache.NewPGCache(postgres.NewConn(env.String("POSTGRES_STRING")), 1000 * 60 * 20) defer pgconn.Close() @@ -53,7 +55,8 @@ func main() { uaParser = uaparser.NewUAParser(env.String("UAPARSER_FILE")) geoIP = geoip.NewGeoIP(env.String("MAXMINDDB_FILE")) flaker = flakeid.NewFlaker(env.WorkerID()) - cacheAssets = env.Bool("CACHE_ASSETS") + CACHE_ASSESTS = env.Bool("CACHE_ASSETS") + BEACON_SIZE_LIMIT = int64(env.Uint64("BEACON_SIZE_LIMIT")) HTTP_PORT := env.String("HTTP_PORT") diff --git a/frontend/app/components/BugFinder/LiveSessionList/LiveSessionList.tsx b/frontend/app/components/BugFinder/LiveSessionList/LiveSessionList.tsx index 7c60216c8..8b5934c23 100644 --- a/frontend/app/components/BugFinder/LiveSessionList/LiveSessionList.tsx +++ b/frontend/app/components/BugFinder/LiveSessionList/LiveSessionList.tsx @@ -1,5 +1,5 @@ -import React, { useEffect, useState } from 'react'; -import { fetchLiveList } from 'Duck/sessions'; +import React, { useEffect } from 'react'; +import { fetchList } from 'Duck/sessions'; import { connect } from 'react-redux'; import { NoContent, Loader } from 'UI'; import { List, Map } from 'immutable'; @@ -7,45 +7,31 @@ import SessionItem from 'Shared/SessionItem'; interface Props { loading: Boolean, - list?: List, - fetchLiveList: () => void, + list?: List, + fetchList: (params) => void, filters: List } function LiveSessionList(props: Props) { - const { loading, list, filters } = props; - const [userId, setUserId] = useState(undefined) + const { loading, list, filters } = props; - useEffect(() => { - props.fetchLiveList(); + useEffect(() => { + props.fetchList(filters.toJS()); }, []) - useEffect(() => { - if (filters) { - const userIdFilter = filters.filter(i => i.key === 'USERID').first() - if (userIdFilter) - setUserId(userIdFilter.value[0]) - else - setUserId(undefined) - } - }, [filters]) - - return (
} show={ !loading && list && list.size === 0} > - {list && (userId ? list.filter(i => i.userId === userId) : list).map(session => ( + {list && list.map(session => ( ))} @@ -57,5 +43,5 @@ function LiveSessionList(props: Props) { export default connect(state => ({ list: state.getIn(['sessions', 'liveSessions']), loading: state.getIn([ 'sessions', 'loading' ]), - filters: state.getIn([ 'filters', 'appliedFilter', 'filters' ]), -}), { fetchLiveList })(LiveSessionList) + filters: state.getIn([ 'filters', 'appliedFilter' ]), +}), { fetchList })(LiveSessionList) diff --git a/frontend/app/components/Client/Sites/Sites.js b/frontend/app/components/Client/Sites/Sites.js index 087cca66f..0f54b0b02 100644 --- a/frontend/app/components/Client/Sites/Sites.js +++ b/frontend/app/components/Client/Sites/Sites.js @@ -118,7 +118,7 @@ class Sites extends React.PureComponent { const { modalContent, showTrackingCode } = this.state; const isAdmin = user.admin || user.superAdmin; const canAddSites = isAdmin && account.limits.projects && account.limits.projects.remaining !== 0; - const canDeleteSites = sites.size > 1; + const canDeleteSites = sites.size > 1 && isAdmin; return ( @@ -189,15 +189,15 @@ class Sites extends React.PureComponent {