From f1db3b3a3f64a90ae037bcd7302d9429df103c47 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 11 Mar 2022 20:17:12 +0100 Subject: [PATCH 01/22] feat(ui) - assist wip --- tracker/tracker-assist/layout/index.html | 260 ++++++++++++++++++++--- 1 file changed, 236 insertions(+), 24 deletions(-) diff --git a/tracker/tracker-assist/layout/index.html b/tracker/tracker-assist/layout/index.html index bc323b16c..90ab1542c 100644 --- a/tracker/tracker-assist/layout/index.html +++ b/tracker/tracker-assist/layout/index.html @@ -10,7 +10,7 @@ - +
Connecting...
-
+
-

Starting video...

+
-

Starting video...

+
-
+
- + - +
- End +
+ + +
+
+
+ + + + Chat +
+
+ + + +
+
+
+
+
Hey, did you get the key?
+
+ Username + 00:00 +
+
+
+
+ Oui, merci! +
+
+ Username + 00:00 +
+
+
+
+ +
+ + + +
+
+
From 39cd7ac1901aedf8dad25bd342caeafeeb1a81da Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 14 Mar 2022 17:38:51 +0100 Subject: [PATCH 02/22] feat(ui) - assist chat wip --- tracker/tracker-assist/layout/index.html | 190 ++++++++++++++++++----- 1 file changed, 150 insertions(+), 40 deletions(-) diff --git a/tracker/tracker-assist/layout/index.html b/tracker/tracker-assist/layout/index.html index 90ab1542c..e541fc1e3 100644 --- a/tracker/tracker-assist/layout/index.html +++ b/tracker/tracker-assist/layout/index.html @@ -9,6 +9,13 @@ @@ -273,6 +355,26 @@ +
+
The agent is requesting remote control
+
+ + +
+
+ +
+
Answer the call so the agent can assist.
+
+ + +
+
Connecting...
@@ -301,39 +403,47 @@
+ - - + + + - -
-
- + + +
+ +
- -
+ +
@@ -342,12 +452,12 @@ Chat
- +
-
+
Hey, did you get the key?
@@ -365,7 +475,7 @@
-
+
From 895b5db1e6faf7c9eabb2dcef7c36d27486f5c2b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 22 Mar 2022 19:19:43 +0100 Subject: [PATCH 03/22] feat(api): funnels changed flatten call --- api/chalicelib/core/funnels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index e63ea6efc..e6cdc4864 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -299,7 +299,7 @@ def get(funnel_id, project_id, user_id, flatten=True, fix_stages=True): f["filter"]["events"] = __fix_stages(f["filter"]["events"]) f["filter"]["events"] = [e.dict() for e in f["filter"]["events"]] if flatten: - f["filter"] = helper.old_search_payload_to_flat(f["filter"]) + f["filter"] = helper.old_search_payload_to_flat(f) return f From 1ba5c002a90e031bd94f36dd074c5c8b54e8a66b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 22 Mar 2022 19:43:57 +0100 Subject: [PATCH 04/22] feat(api): funnels un-flatten insights --- api/chalicelib/core/funnels.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index e6cdc4864..a81782832 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -201,7 +201,7 @@ def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.Funnel data.events = filter_stages(data.events) data.events = __fix_stages(data.events) if len(data.events) == 0: - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=data.range_value, @@ -226,7 +226,7 @@ def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_dat def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data: schemas.FunnelInsightsPayloadSchema): data.events = filter_stages(__parse_events(data.events)) if len(data.events) == 0: - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=data.rangeValue, @@ -256,7 +256,7 @@ def get_issues_on_the_fly(funnel_id, user_id, project_id, data: schemas.FunnelSe data.events = filter_stages(data.events) data.events = __fix_stages(data.events) if len(data.events) == 0: - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=data.rangeValue, @@ -307,7 +307,7 @@ def get(funnel_id, project_id, user_id, flatten=True, fix_stages=True): def search_by_issue(user_id, project_id, funnel_id, issue_id, data: schemas.FunnelSearchPayloadSchema, range_value=None, start_date=None, end_date=None): if len(data.events) == 0: - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} data.startDate = data.startDate if data.startDate is not None else start_date From 4a71556f92d3c3d3943a314ee329ea28a4a4eae7 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 22 Mar 2022 19:52:55 +0100 Subject: [PATCH 05/22] feat(api): funnels fixed get --- api/chalicelib/core/funnels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index a81782832..1a56f0272 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -299,7 +299,7 @@ def get(funnel_id, project_id, user_id, flatten=True, fix_stages=True): f["filter"]["events"] = __fix_stages(f["filter"]["events"]) f["filter"]["events"] = [e.dict() for e in f["filter"]["events"]] if flatten: - f["filter"] = helper.old_search_payload_to_flat(f) + f["filter"] = helper.old_search_payload_to_flat(f["filter"]) return f From eee68d53d5578333f4bf4a3607ea9d6f85427659 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 22 Mar 2022 20:45:44 +0100 Subject: [PATCH 06/22] feat(api): sessions-search fixed events timestamp --- api/chalicelib/core/sessions.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index c6e1a7efd..1903cc08b 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -385,6 +385,19 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d return sessions +def __is_valid_event(is_any: bool, event: schemas._SessionSearchEventSchema): + return not (not is_any and len(event.value) == 0 and event.type not in [schemas.EventType.request_details, + schemas.EventType.graphql_details] \ + or event.type in [schemas.PerformanceEventType.location_dom_complete, + schemas.PerformanceEventType.location_largest_contentful_paint_time, + schemas.PerformanceEventType.location_ttfb, + schemas.PerformanceEventType.location_avg_cpu_load, + schemas.PerformanceEventType.location_avg_memory_usage + ] and (event.source is None or len(event.source) == 0) \ + or event.type in [schemas.EventType.request_details, schemas.EventType.graphql_details] and ( + event.filters is None or len(event.filters) == 0)) + + def search_query_parts(data, error_status, errors_only, favorite_only, issue, project_id, user_id, extra_event=None): ss_constraints = [] full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, @@ -600,6 +613,13 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr value_key=f_k)) # --------------------------------------------------------------------------- if len(data.events) > 0: + valid_events_count = 0 + for event in data.events: + is_any = _isAny_opreator(event.operator) + if not isinstance(event.value, list): + event.value = [event.value] + if __is_valid_event(is_any=is_any, event=event): + valid_events_count += 1 events_query_from = [] event_index = 0 or_events = data.events_order == schemas.SearchEventOrder._or @@ -610,16 +630,7 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr is_any = _isAny_opreator(event.operator) if not isinstance(event.value, list): event.value = [event.value] - if not is_any and len(event.value) == 0 and event_type not in [schemas.EventType.request_details, - schemas.EventType.graphql_details] \ - or event_type in [schemas.PerformanceEventType.location_dom_complete, - schemas.PerformanceEventType.location_largest_contentful_paint_time, - schemas.PerformanceEventType.location_ttfb, - schemas.PerformanceEventType.location_avg_cpu_load, - schemas.PerformanceEventType.location_avg_memory_usage - ] and (event.source is None or len(event.source) == 0) \ - or event_type in [schemas.EventType.request_details, schemas.EventType.graphql_details] and ( - event.filters is None or len(event.filters) == 0): + if not __is_valid_event(is_any=is_any, event=event): continue op = __get_sql_operator(event.operator) is_not = False @@ -938,7 +949,7 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr """) else: events_query_from.append(f"""\ - (SELECT main.session_id, MIN(main.timestamp) AS timestamp + (SELECT main.session_id, {"MIN" if event_index < (valid_events_count - 1) else "MAX"}(main.timestamp) AS timestamp FROM {event_from} WHERE {" AND ".join(event_where)} GROUP BY 1 From 979e76321b1faf32baf6c2218438fd217193b187 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 23 Mar 2022 12:17:08 +0100 Subject: [PATCH 07/22] feat(api): sessions-search-value-helper fixed value-operator for non-string-values --- api/chalicelib/utils/helper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py index 8e1f5788c..8cfab8a3f 100644 --- a/api/chalicelib/utils/helper.py +++ b/api/chalicelib/utils/helper.py @@ -213,11 +213,11 @@ def values_for_operator(value: Union[str, list], op: schemas.SearchEventOperator if value is None: return value if op == schemas.SearchEventOperator._starts_with: - return value + '%' + return f"{value}%" elif op == schemas.SearchEventOperator._ends_with: - return '%' + value + return f"%{value}" elif op == schemas.SearchEventOperator._contains or op == schemas.SearchEventOperator._not_contains: - return '%' + value + '%' + return f"%{value}%" return value From 0de151c133e8918f287438dc3306596bf3443ae9 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 23 Mar 2022 15:05:41 +0100 Subject: [PATCH 08/22] confirm window --- tracker/tracker-assist/src/ConfirmWindow.ts | 163 ++++++++++++-------- tracker/tracker-assist/src/icons.ts | 4 +- 2 files changed, 104 insertions(+), 63 deletions(-) diff --git a/tracker/tracker-assist/src/ConfirmWindow.ts b/tracker/tracker-assist/src/ConfirmWindow.ts index 7a1238508..1b426426d 100644 --- a/tracker/tracker-assist/src/ConfirmWindow.ts +++ b/tracker/tracker-assist/src/ConfirmWindow.ts @@ -2,75 +2,94 @@ import type { Properties } from 'csstype'; import { declineCall, acceptCall, cross, remoteControl } from './icons.js' -type ButtonOptions = HTMLButtonElement | string | { - innerHTML: string, - style?: Properties, -} +const TEXT_GRANT_REMORTE_ACCESS = "Grant Remote Access"; +const TEXT_REJECT = "Reject"; +const TEXT_ANSWER_CALL = `${acceptCall}   Answer`; +type ButtonOptions = + | HTMLButtonElement + | string + | { + innerHTML: string; + style?: Properties; + }; // TODO: common strategy for InputOptions/defaultOptions merging interface ConfirmWindowOptions { - text: string, - style?: Properties, - confirmBtn: ButtonOptions, - declineBtn: ButtonOptions, + text: string; + style?: Properties; + confirmBtn: ButtonOptions; + declineBtn: ButtonOptions; } -export type Options = string | Partial +export type Options = string | Partial; function confirmDefault( opts: Options, confirmBtn: ButtonOptions, declineBtn: ButtonOptions, - text: string, + text: string ): ConfirmWindowOptions { - const isStr = typeof opts === "string" - return Object.assign({ - text: isStr ? opts : text, - confirmBtn, - declineBtn, - }, isStr ? undefined : opts) + const isStr = typeof opts === "string"; + return Object.assign( + { + text: isStr ? opts : text, + confirmBtn, + declineBtn + }, + isStr ? undefined : opts + ); } -export const callConfirmDefault = (opts: Options) => - confirmDefault(opts, acceptCall, declineCall, "You have an incoming call. Do you want to answer?") -export const controlConfirmDefault = (opts: Options) => - confirmDefault(opts, remoteControl, cross, "Allow remote control?") +export const callConfirmDefault = (opts: Options) => + confirmDefault( + opts, + TEXT_ANSWER_CALL, + TEXT_REJECT, + "You have an incoming call. Do you want to answer?" + ); +export const controlConfirmDefault = (opts: Options) => + confirmDefault( + opts, + TEXT_GRANT_REMORTE_ACCESS, + TEXT_REJECT, + "Allow remote control?" + ); function makeButton(options: ButtonOptions): HTMLButtonElement { if (options instanceof HTMLButtonElement) { - return options + return options; } - const btn = document.createElement('button') + const btn = document.createElement("button"); Object.assign(btn.style, { - background: "transparent", - padding: 0, - margin: 0, - border: 0, + padding: "10px 14px", + fontSize: "14px", + borderRadius: "3px", + border: "none", cursor: "pointer", - borderRadius: "50%", - width: "22px", - height: "22px", - color: "white", // TODO: nice text button in case when only text is passed - }) + display: "flex", + alignItems: "center", + textTransform: "uppercase", + marginRight: "10px" + }); if (typeof options === "string") { - btn.innerHTML = options + btn.innerHTML = options; } else { - btn.innerHTML = options.innerHTML - Object.assign(btn.style, options.style) + btn.innerHTML = options.innerHTML; + Object.assign(btn.style, options.style); } - return btn + return btn; } export default class ConfirmWindow { private wrapper: HTMLDivElement; constructor(options: ConfirmWindowOptions) { - const wrapper = document.createElement('div'); - const popup = document.createElement('div'); - const p = document.createElement('p'); + const wrapper = document.createElement("div"); + const popup = document.createElement("div"); + const p = document.createElement("p"); p.innerText = options.text; - const buttons = document.createElement('div'); + const buttons = document.createElement("div"); const confirmBtn = makeButton(options.confirmBtn); const declineBtn = makeButton(options.declineBtn); buttons.appendChild(confirmBtn); @@ -78,37 +97,54 @@ export default class ConfirmWindow { popup.appendChild(p); popup.appendChild(buttons); + Object.assign(confirmBtn.style, { + background: "rgba(0, 167, 47, 1)", + color: "white" + }); + + Object.assign(declineBtn.style, { + background: "#FFE9E9", + color: "#CC0000" + }); + Object.assign(buttons.style, { marginTop: "10px", display: "flex", alignItems: "center", - justifyContent: "space-evenly", + // justifyContent: "space-evenly", + backgroundColor: "white", + padding: "10px", + boxShadow: "0px 0px 3.99778px 1.99889px rgba(0, 0, 0, 0.1)", + borderRadius: "6px" }); - Object.assign(popup.style, { - position: "relative", - pointerEvents: "auto", - margin: "4em auto", - width: "90%", - maxWidth: "400px", - padding: "25px 30px", - background: "black", - opacity: ".75", - color: "white", - textAlign: "center", - borderRadius: ".25em .25em .4em .4em", - boxShadow: "0 0 20px rgb(0 0 0 / 20%)", - }, options.style); + Object.assign( + popup.style, + { + font: "14px 'Roboto', sans-serif", + position: "relative", + pointerEvents: "auto", + margin: "4em auto", + width: "90%", + maxWidth: "fit-content", + padding: "20px", + background: "#F3F3F3", + opacity: ".75", + color: "black", + borderRadius: "3px", + boxShadow: "0px 0px 3.99778px 1.99889px rgba(0, 0, 0, 0.1)" + }, + options.style + ); Object.assign(wrapper.style, { - position: "fixed", left: 0, top: 0, height: "100%", width: "100%", pointerEvents: "none", - zIndex: 2147483647 - 1, - }) + zIndex: 2147483647 - 1 + }); wrapper.appendChild(popup); this.wrapper = wrapper; @@ -116,18 +152,19 @@ export default class ConfirmWindow { confirmBtn.onclick = () => { this._remove(); this.resolve(true); - } + }; declineBtn.onclick = () => { this._remove(); this.resolve(false); - } + }; } - private resolve: (result: boolean) => void = ()=>{}; - private reject: ()=>void = ()=>{}; + private resolve: (result: boolean) => void = () => {}; + private reject: () => void = () => {}; mount(): Promise { document.body.appendChild(this.wrapper); + return new Promise((resolve, reject) => { this.resolve = resolve; this.reject = reject; @@ -135,7 +172,9 @@ export default class ConfirmWindow { } private _remove() { - if (!this.wrapper.parentElement) { return; } + if (!this.wrapper.parentElement) { + return; + } document.body.removeChild(this.wrapper); } remove() { diff --git a/tracker/tracker-assist/src/icons.ts b/tracker/tracker-assist/src/icons.ts index 724d94248..763b015b9 100644 --- a/tracker/tracker-assist/src/icons.ts +++ b/tracker/tracker-assist/src/icons.ts @@ -2,7 +2,9 @@ // TODO: something with these big strings in bundle? -export const declineCall = ``; +export const declineCall = ` + +`; export const acceptCall = declineCall.replace('fill="#ef5261"', 'fill="green"') From 6188b385554e8a25f5af796f56ac33d4b3cda9a6 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 23 Mar 2022 16:48:02 +0100 Subject: [PATCH 09/22] feat(frontend-assist): annotations & iremote typing --- frontend/.gitignore | 1 + .../StatedScreen/Screen/BaseScreen.ts | 23 +++-- .../StatedScreen/Screen/screen.css | 1 + .../managers/AnnotationCanvas.ts | 84 +++++++++++++++++++ .../managers/AssistManager.ts | 66 ++++++++++++--- 5 files changed, 155 insertions(+), 20 deletions(-) create mode 100644 frontend/app/player/MessageDistributor/managers/AnnotationCanvas.ts diff --git a/frontend/.gitignore b/frontend/.gitignore index 92150a232..dfcb0fd79 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -8,3 +8,4 @@ app/components/ui/SVG.js *.DS_Store .env *css.d.ts +*.cache diff --git a/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts b/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts index e2cd635fd..fc4acdf03 100644 --- a/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts +++ b/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts @@ -70,10 +70,10 @@ export default abstract class BaseScreen { private boundingRect: DOMRect | null = null; private getBoundingClientRect(): DOMRect { - //if (this.boundingRect === null) { - return this.boundingRect = this.overlay.getBoundingClientRect(); // expensive operation? - //} - //return this.boundingRect; + if (this.boundingRect === null) { + return this.boundingRect = this.overlay.getBoundingClientRect() // expensive operation? + } + return this.boundingRect } getInternalViewportCoordinates({ x, y }: Point): Point { @@ -85,17 +85,22 @@ export default abstract class BaseScreen { const screenX = (x - overlayX) * scale; const screenY = (y - overlayY) * scale; - return { x: screenX, y: screenY }; + return { x: Math.round(screenX), y: Math.round(screenY) }; + } + + getCurrentScroll(): Point { + const docEl = this.document?.documentElement + const x = docEl ? docEl.scrollLeft : 0 + const y = docEl ? docEl.scrollTop : 0 + return { x, y } } getInternalCoordinates(p: Point): Point { const { x, y } = this.getInternalViewportCoordinates(p); - const docEl = this.document?.documentElement - const scrollX = docEl ? docEl.scrollLeft : 0 - const scrollY = docEl ? docEl.scrollTop : 0 + const sc = this.getCurrentScroll() - return { x: x+scrollX, y: y+scrollY }; + return { x: x+sc.x, y: y+sc.y }; } getElementFromInternalPoint({ x, y }: Point): Element | null { diff --git a/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css b/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css index b715986d2..39808f1db 100644 --- a/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css +++ b/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css @@ -12,6 +12,7 @@ background: white; } .overlay { + user-select: none; position: absolute; top: 0; left: 0; diff --git a/frontend/app/player/MessageDistributor/managers/AnnotationCanvas.ts b/frontend/app/player/MessageDistributor/managers/AnnotationCanvas.ts new file mode 100644 index 000000000..ad110c2c3 --- /dev/null +++ b/frontend/app/player/MessageDistributor/managers/AnnotationCanvas.ts @@ -0,0 +1,84 @@ +export default class AnnotationCanvas { + readonly canvas: HTMLCanvasElement + private ctx: CanvasRenderingContext2D | null = null + private painting: boolean = false + constructor() { + this.canvas = document.createElement('canvas') + Object.assign(this.canvas.style, { + position: "fixed", + cursor: "url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABMAAAATCAYAAAByUDbMAAAAAXNSR0IArs4c6QAAAWNJREFUOE+l1D1Lw1AUBuD35Catg5NzaCMRMilINnGok7sguLg4OlRcBTd/hqBVB0ed7KDgIPgXhJoaG10Kgk4a83EkhcYYktimd703z31zzuESSqwGIDs1bRvAIiRcWrZ9ETFUwhJ6XTsDsPH7Le1bz08H42JkGMa09+W2CVhKBmHC7jhYlOgUTPdUEa3Q86+SIDN/j4olf43BtJMFjoJl1AgMUJMUcRInZHT+w7KgYakGoDxVafmue0hBsJeLmaapvPffziFhraDjDMKWZdvHRaNRlCi2mUNHYl55dBwrDysFZWGloTQ2EZTEJoZiTFXVmaos34Ixn9e5qNgCaHR6vW7emcFozNVmN1ERbfb9myww3bVCTK9rPsDrpCh37HnXAC3Ek5lqf9ErM0im1zUG8BmGtCqq4mEIjppoeEESA5g/JIkaLMuv7AVHEgfNohqlU/7Fol3mPodiufvS7Yz7cP4ARjbPWyYPZSMAAAAASUVORK5CYII=') 0 20, crosshair", + left: 0, + top: 0, + //zIndex: 2147483647 - 2, + }) + } + + isPainting() { + return this.painting + } + + private resizeCanvas = () => { + if (!this.canvas.parentElement) { return } + this.canvas.width = this.canvas.parentElement.offsetWidth + this.canvas.height = this.canvas.parentElement.offsetHeight + } + + private lastPosition: [number, number] = [0,0] + start = (p: [number, number]) => { + this.painting = true + this.clrTmID && clearTimeout(this.clrTmID) + this.lastPosition = p + } + + stop = () => { + this.painting = false + this.fadeOut() + } + + move = (p: [number, number]) =>{ + if (!this.ctx || !this.painting) { return } + this.ctx.globalAlpha = 1.0 + this.ctx.beginPath() + this.ctx.moveTo(this.lastPosition[0], this.lastPosition[1]) + this.ctx.lineTo(p[0], p[1]) + this.ctx.lineWidth = 8 + this.ctx.lineCap = "round" + this.ctx.lineJoin = "round" + this.ctx.strokeStyle = "red" + this.ctx.stroke() + this.lastPosition = p + } + + clrTmID: ReturnType | null = null + private fadeOut() { + let timeoutID: ReturnType + const fadeStep = () => { + if (!this.ctx || this.painting ) { return } + this.ctx.globalCompositeOperation = 'destination-out' + this.ctx.fillStyle = "rgba(255, 255, 255, 0.1)" + this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height) + this.ctx.globalCompositeOperation = 'source-over' + timeoutID = setTimeout(fadeStep,100) + } + this.clrTmID = setTimeout(() => { + clearTimeout(timeoutID) + this.ctx && + this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height) + }, 3700) + fadeStep() + } + + mount(parent: HTMLElement) { + parent.appendChild(this.canvas) + this.ctx = this.canvas.getContext("2d") + window.addEventListener("resize", this.resizeCanvas) + this.resizeCanvas() + } + + remove() { + if (this.canvas.parentNode){ + this.canvas.parentNode.removeChild(this.canvas) + } + window.removeEventListener("resize", this.resizeCanvas) + } +} \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager.ts b/frontend/app/player/MessageDistributor/managers/AssistManager.ts index 0b570fd87..92756567d 100644 --- a/frontend/app/player/MessageDistributor/managers/AssistManager.ts +++ b/frontend/app/player/MessageDistributor/managers/AssistManager.ts @@ -7,8 +7,8 @@ import store from 'App/store'; import type { LocalStream } from './LocalStream'; import { update, getState } from '../../store'; import { iceServerConfigFromString } from 'App/utils' - -import MStreamReader from '../messages/MStreamReader';; +import AnnotationCanvas from './AnnotationCanvas'; +import MStreamReader from '../messages/MStreamReader'; import JSONRawMessageReader from '../messages/JSONRawMessageReader' export enum CallingState { @@ -136,12 +136,13 @@ export default class AssistManager { //socket.onAny((...args) => console.log(...args)) socket.on("connect", () => { waitingForMessages = true - this.setStatus(ConnectionStatus.WaitingMessages) + this.setStatus(ConnectionStatus.WaitingMessages) // TODO: happens frequently on bad network }) socket.on("disconnect", () => { this.toggleRemoteControl(false) }) socket.on('messages', messages => { + //console.log(messages.filter(m => m._id === 41 || m._id === 44)) showDisconnectTimeout && clearTimeout(showDisconnectTimeout); jmr.append(messages) // as RawMessage[] @@ -173,9 +174,8 @@ export default class AssistManager { this.setStatus(ConnectionStatus.Disconnected) }, 30000) - if (getState().remoteControl === RemoteControlStatus.Requesting || - getState().remoteControl === RemoteControlStatus.Enabled) { - this.toggleRemoteControl(false) + if (getState().remoteControl === RemoteControlStatus.Requesting) { + this.toggleRemoteControl(false) // else its remaining } // Call State @@ -200,7 +200,7 @@ export default class AssistManager { private onMouseMove = (e: MouseEvent): void => { if (!this.socket) { return } const data = this.md.getInternalCoordinates(e) - this.socket.emit("move", [ Math.round(data.x), Math.round(data.y) ]) + this.socket.emit("move", [ data.x, data.y ]) } private onWheel = (e: WheelEvent): void => { @@ -213,15 +213,23 @@ export default class AssistManager { private onMouseClick = (e: MouseEvent): void => { if (!this.socket) { return; } - const data = this.md.getInternalViewportCoordinates(e); + const data = this.md.getInternalViewportCoordinates(e) // const el = this.md.getElementFromPoint(e); // requires requestiong node_id from domManager const el = this.md.getElementFromInternalPoint(data) if (el instanceof HTMLElement) { el.focus() - el.oninput = e => e.preventDefault(); - el.onkeydown = e => e.preventDefault(); + el.oninput = e => { + if (el instanceof HTMLTextAreaElement + || el instanceof HTMLInputElement + ) { + this.socket && this.socket.emit("input", el.value) + } else if (el.isContentEditable) { + this.socket && this.socket.emit("input", el.innerText) + } + } + //el.onkeydown = e => e.preventDefault() } - this.socket.emit("click", [ Math.round(data.x), Math.round(data.y) ]); + this.socket.emit("click", [ data.x, data.y ]); } private toggleRemoteControl(newState: boolean){ @@ -310,6 +318,8 @@ export default class AssistManager { this.callConnection && this.callConnection.close() update({ calling: CallingState.NoCall }) this.callArgs = null + this.annot?.remove() + this.annot = null } private initiateCallEnd = () => { @@ -355,6 +365,8 @@ export default class AssistManager { } } + private annot: AnnotationCanvas | null = null + private _call() { if (![CallingState.NoCall, CallingState.Reconnecting].includes(getState().calling)) { return } update({ calling: CallingState.Connecting }) @@ -379,6 +391,34 @@ export default class AssistManager { call.on('stream', stream => { update({ calling: CallingState.OnCall }) this.callArgs && this.callArgs.onStream(stream) + + if (!this.annot) { + const annot = this.annot = new AnnotationCanvas() + annot.mount(this.md.overlay) + annot.canvas.addEventListener("mousedown", e => { + if (!this.socket) { return } + const data = this.md.getInternalViewportCoordinates(e) + annot.start([ data.x, data.y ]) + this.socket.emit("startAnnotation", [ data.x, data.y ]) + }) + annot.canvas.addEventListener("mouseleave", () => { + if (!this.socket) { return } + annot.stop() + this.socket.emit("stopAnnotation") + }) + annot.canvas.addEventListener("mouseup", () => { + if (!this.socket) { return } + annot.stop() + this.socket.emit("stopAnnotation") + }) + annot.canvas.addEventListener("mousemove", e => { + if (!this.socket || !annot.isPainting()) { return } + + const data = this.md.getInternalViewportCoordinates(e) + annot.move([ data.x, data.y ]) + this.socket.emit("moveAnnotation", [ data.x, data.y ]) + }) + } }); //call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) @@ -409,6 +449,10 @@ export default class AssistManager { this.socket.close() document.removeEventListener('visibilitychange', this.onVisChange) } + if (this.annot) { + this.annot.remove() + this.annot = null + } } } From e221615c4c21037c61f203814ab25746853dc102 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 23 Mar 2022 17:14:31 +0100 Subject: [PATCH 10/22] fix(frontend-player): fix overlay restoration after the elements inspector --- .../StatedScreen/Screen/BaseScreen.ts | 24 +++++++++++-------- .../StatedScreen/Screen/screen.css | 2 +- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts b/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts index fc4acdf03..fa66d5eb4 100644 --- a/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts +++ b/frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts @@ -31,16 +31,6 @@ export default abstract class BaseScreen { const screen = document.createElement('div'); - setTimeout(function() { - iframe.contentDocument?.addEventListener('mousemove', function() { - overlay.style.display = 'block'; - }) - - overlay.addEventListener('contextmenu', function() { - overlay.style.display = 'none'; - }) - }, 10) - screen.className = styles.screen; screen.appendChild(iframe); screen.appendChild(overlay); @@ -58,6 +48,20 @@ export default abstract class BaseScreen { // parentElement.onresize = this.scale; window.addEventListener('resize', this.scale); this.scale(); + + /* == For the Inspecting Document content == */ + this.overlay.addEventListener('contextmenu', () => { + this.overlay.style.display = 'none' + const doc = this.document + if (!doc) { return } + const returnOverlay = () => { + this.overlay.style.display = 'block' + doc.removeEventListener('mousemove', returnOverlay) + doc.removeEventListener('mouseclick', returnOverlay) // TODO: prevent default in case of input selection + } + doc.addEventListener('mousemove', returnOverlay) + doc.addEventListener('mouseclick', returnOverlay) + }) } get window(): WindowProxy | null { diff --git a/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css b/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css index 39808f1db..696b38e7a 100644 --- a/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css +++ b/frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css @@ -1,4 +1,5 @@ .screen { + user-select: none; overflow: hidden; position: absolute; transform-origin: left top; @@ -12,7 +13,6 @@ background: white; } .overlay { - user-select: none; position: absolute; top: 0; left: 0; From b7c66e32143ccf6108317cf3df95817f457aba11 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 23 Mar 2022 17:19:45 +0100 Subject: [PATCH 11/22] feat(tracker-assist):3.5.6:annotation on call; remote typing; RemoteControl logic taken out --- tracker/tracker-assist/package.json | 2 +- .../tracker-assist/src/AnnotationCanvas.ts | 80 +++++++++++ tracker/tracker-assist/src/Assist.ts | 128 +++++++++--------- tracker/tracker-assist/src/ConfirmWindow.ts | 3 +- tracker/tracker-assist/src/Mouse.ts | 2 + tracker/tracker-assist/src/RemoteControl.ts | 88 ++++++++++++ 6 files changed, 239 insertions(+), 64 deletions(-) create mode 100644 tracker/tracker-assist/src/AnnotationCanvas.ts create mode 100644 tracker/tracker-assist/src/RemoteControl.ts diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 30f85875b..0a9fda457 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "3.5.5", + "version": "3.5.6", "keywords": [ "WebRTC", "assistance", diff --git a/tracker/tracker-assist/src/AnnotationCanvas.ts b/tracker/tracker-assist/src/AnnotationCanvas.ts new file mode 100644 index 000000000..afda8e2a5 --- /dev/null +++ b/tracker/tracker-assist/src/AnnotationCanvas.ts @@ -0,0 +1,80 @@ +export default class AnnotationCanvas { + private canvas: HTMLCanvasElement + private ctx: CanvasRenderingContext2D | null = null + private painting: boolean = false + constructor() { + this.canvas = document.createElement('canvas') + Object.assign(this.canvas.style, { + position: "fixed", + left: 0, + top: 0, + pointerEvents: "none", + zIndex: 2147483647 - 2, + }) + } + + private resizeCanvas = () => { + this.canvas.width = window.innerWidth + this.canvas.height = window.innerHeight + } + + private lastPosition: [number, number] = [0,0] + start = (p: [number, number]) => { + this.painting = true + this.clrTmID && clearTimeout(this.clrTmID) + this.lastPosition = p + } + + stop = () => { + this.painting = false + this.fadeOut() + } + + move = (p: [number, number]) =>{ + if (!this.ctx || !this.painting) { return } + this.ctx.globalAlpha = 1.0 + this.ctx.beginPath() + this.ctx.moveTo(this.lastPosition[0], this.lastPosition[1]) + this.ctx.lineTo(p[0], p[1]) + this.ctx.lineWidth = 8 + this.ctx.lineCap = "round" + this.ctx.lineJoin = "round" + this.ctx.strokeStyle = "red" + this.ctx.stroke() + this.lastPosition = p + } + + clrTmID: ReturnType | null = null + private fadeOut() { + let timeoutID: ReturnType + const fadeStep = () => { + if (!this.ctx || this.painting ) { return } + this.ctx.globalCompositeOperation = 'destination-out' + this.ctx.fillStyle = "rgba(255, 255, 255, 0.1)" + this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height) + this.ctx.globalCompositeOperation = 'source-over' + timeoutID = setTimeout(fadeStep,100) + } + this.clrTmID = setTimeout(() => { + clearTimeout(timeoutID) + this.ctx && + this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height) + }, 4000) + fadeStep() + } + + + mount() { + document.body.appendChild(this.canvas) + this.ctx = this.canvas.getContext("2d") + window.addEventListener("resize", this.resizeCanvas) + this.resizeCanvas() + } + + remove() { + if (this.canvas.parentNode){ + this.canvas.parentNode.removeChild(this.canvas) + } + window.removeEventListener("resize", this.resizeCanvas) + } +} \ No newline at end of file diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index b5272510b..f7569bf26 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -5,8 +5,9 @@ import type { Properties } from 'csstype'; import { App } from '@openreplay/tracker'; import RequestLocalStream from './LocalStream.js'; -import Mouse from './Mouse.js'; +import RemoteControl from './RemoteControl.js'; import CallWindow from './CallWindow.js'; +import AnnotationCanvas from './AnnotationCanvas.js'; import ConfirmWindow, { callConfirmDefault, controlConfirmDefault } from './ConfirmWindow.js'; import type { Options as ConfirmOptions } from './ConfirmWindow.js'; @@ -14,12 +15,12 @@ import type { Options as ConfirmOptions } from './ConfirmWindow.js'; //@ts-ignore peerjs hack for webpack5 (?!) TODO: ES/node modules; Peer = Peer.default || Peer; -type BehinEndCallback = () => ((()=>{}) | void) +type StartEndCallback = () => ((()=>{}) | void) export interface Options { - onAgentConnect: BehinEndCallback, - onCallStart: BehinEndCallback, - onRemoteControlStart: BehinEndCallback, + onAgentConnect: StartEndCallback, + onCallStart: StartEndCallback, + onRemoteControlStart: StartEndCallback, session_calling_peer_key: string, session_control_peer_key: string, callConfirm: ConfirmOptions, @@ -39,8 +40,11 @@ enum CallingState { }; +// TODO typing???? +type OptionalCallback = (()=>{}) | void type Agent = { - onDisconnect: ((()=>{}) | void), // TODO: better types here + onDisconnect?: OptionalCallback, + onControlReleased?: OptionalCallback, name?: string // } @@ -139,6 +143,34 @@ export default class Assist { }) socket.onAny((...args) => app.debug.log("Socket:", ...args)) + + const remoteControl = new RemoteControl( + this.options, + id => { + this.agents[id].onControlReleased = this.options.onRemoteControlStart() + this.emit("control_granted", id) + }, + id => { + const cb = this.agents[id].onControlReleased + delete this.agents[id].onControlReleased + typeof cb === "function" && cb() + this.emit("control_rejected", id) + }, + ) + + // TODO: check incoming args + socket.on("request_control", remoteControl.requestControl) + socket.on("release_control", remoteControl.releaseControl) + socket.on("scroll", remoteControl.scroll) + socket.on("click", remoteControl.click) + socket.on("move", remoteControl.move) + socket.on("input", remoteControl.input) + + let annot: AnnotationCanvas | null = null + socket.on("moveAnnotation", (_, p) => annot && annot.move(p)) // TODO: restrict by id + socket.on("startAnnotation", (_, p) => annot && annot.start(p)) + socket.on("stopAnnotation", () => annot && annot.stop()) + socket.on("NEW_AGENT", (id: string, info) => { this.agents[id] = { onDisconnect: this.options.onAgentConnect && this.options.onAgentConnect(), @@ -148,7 +180,7 @@ export default class Assist { this.app.stop(); this.app.start().then(() => { this.assistDemandedRestart = false }) }) - socket.on("AGENTS_CONNECTED", (ids) => { + socket.on("AGENTS_CONNECTED", (ids: string[]) => { ids.forEach(id =>{ this.agents[id] = { onDisconnect: this.options.onAgentConnect && this.options.onAgentConnect(), @@ -157,61 +189,10 @@ export default class Assist { this.assistDemandedRestart = true this.app.stop(); this.app.start().then(() => { this.assistDemandedRestart = false }) - const storedControllingAgent = sessionStorage.getItem(this.options.session_control_peer_key) - if (storedControllingAgent !== null && ids.includes(storedControllingAgent)) { - grantControl(storedControllingAgent) - socket.emit("control_granted", storedControllingAgent) - } else { - sessionStorage.removeItem(this.options.session_control_peer_key) - } + + remoteControl.reconnect(ids) }) - let confirmRC: ConfirmWindow | null = null - const mouse = new Mouse() // TODO: lazy init - let controllingAgent: string | null = null - const requestControl = (id: string) => { - if (controllingAgent !== null) { - socket.emit("control_rejected", id) - return - } - controllingAgent = id // TODO: more explicit pending state - confirmRC = new ConfirmWindow(controlConfirmDefault(this.options.controlConfirm)) - confirmRC.mount().then(allowed => { - if (allowed) { - grantControl(id) - socket.emit("control_granted", id) - } else { - releaseControl() - socket.emit("control_rejected", id) - } - }).catch() - } - let onRemoteControlStop: (()=>void) | null = null - const grantControl = (id: string) => { - controllingAgent = id - mouse.mount() - onRemoteControlStop = this.options.onRemoteControlStart() || null - sessionStorage.setItem(this.options.session_control_peer_key, id) - } - const releaseControl = () => { - typeof onRemoteControlStop === 'function' && onRemoteControlStop() - onRemoteControlStop = null - confirmRC?.remove() - mouse.remove() - controllingAgent = null - sessionStorage.removeItem(this.options.session_control_peer_key) - } - socket.on("request_control", requestControl) - socket.on("release_control", (id: string) => { - if (controllingAgent !== id) { return } - releaseControl() - }) - - - socket.on("scroll", (id, d) => { id === controllingAgent && mouse.scroll(d) }) - socket.on("click", (id, xy) => { id === controllingAgent && mouse.click(xy) }) - socket.on("move", (id, xy) => { id === controllingAgent && mouse.move(xy) }) - let confirmCall:ConfirmWindow | null = null socket.on("AGENT_DISCONNECTED", (id) => { @@ -219,7 +200,7 @@ export default class Assist { this.agents[id] && this.agents[id].onDisconnect != null && this.agents[id].onDisconnect() delete this.agents[id] - controllingAgent === id && releaseControl() + remoteControl.releaseControl(id) // close the call also if (callingAgent === id) { @@ -281,11 +262,20 @@ export default class Assist { style: this.options.confirmStyle, })) confirmAnswer = confirmCall.mount() + this.playNotificationSound() this.onRemoteCallEnd = () => { // if call cancelled by a caller before confirmation app.debug.log("Received call_end during confirm window opened") confirmCall?.remove() setCallingState(CallingState.False) + call.close() } + setTimeout(() => { + if (this.callingState !== CallingState.Requesting) { return } + call.close() + confirmCall?.remove() + this.notifyCallEnd() + setCallingState(CallingState.False) + }, 30000) } confirmAnswer.then(agreed => { @@ -296,13 +286,17 @@ export default class Assist { return } - let callUI = new CallWindow() + const callUI = new CallWindow() + annot = new AnnotationCanvas() + annot.mount() callUI.setAssistentName(agentName) const onCallEnd = this.options.onCallStart() const handleCallEnd = () => { call.close() callUI.remove() + annot && annot.remove() + annot = null setCallingState(CallingState.False) onCallEnd && onCallEnd() } @@ -350,6 +344,16 @@ export default class Assist { }); } + private playNotificationSound() { + if ('Audio' in window) { + new Audio("https://static.openreplay.com/tracker-assist/notification.mp3") + .play() + .catch(e => { + this.app.debug.warn(e) + }) + } + } + private clean() { if (this.peer) { this.peer.destroy() diff --git a/tracker/tracker-assist/src/ConfirmWindow.ts b/tracker/tracker-assist/src/ConfirmWindow.ts index 1b426426d..6cfabbca9 100644 --- a/tracker/tracker-assist/src/ConfirmWindow.ts +++ b/tracker/tracker-assist/src/ConfirmWindow.ts @@ -129,7 +129,7 @@ export default class ConfirmWindow { maxWidth: "fit-content", padding: "20px", background: "#F3F3F3", - opacity: ".75", + //opacity: ".75", color: "black", borderRadius: "3px", boxShadow: "0px 0px 3.99778px 1.99889px rgba(0, 0, 0, 0.1)" @@ -138,6 +138,7 @@ export default class ConfirmWindow { ); Object.assign(wrapper.style, { + position: "fixed", left: 0, top: 0, height: "100%", diff --git a/tracker/tracker-assist/src/Mouse.ts b/tracker/tracker-assist/src/Mouse.ts index b02337c7f..911c29236 100644 --- a/tracker/tracker-assist/src/Mouse.ts +++ b/tracker/tracker-assist/src/Mouse.ts @@ -45,7 +45,9 @@ export default class Mouse { if (el instanceof HTMLElement) { el.click() el.focus() + return el } + return null } private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct diff --git a/tracker/tracker-assist/src/RemoteControl.ts b/tracker/tracker-assist/src/RemoteControl.ts new file mode 100644 index 000000000..a32f81035 --- /dev/null +++ b/tracker/tracker-assist/src/RemoteControl.ts @@ -0,0 +1,88 @@ +import Mouse from './Mouse.js'; +import ConfirmWindow, { controlConfirmDefault } from './ConfirmWindow.js'; +import type { Options as AssistOptions } from './Assist' + +enum RCStatus { + Disabled, + Requesting, + Enabled, +} + +export default class RemoteControl { + private mouse: Mouse | null + private status: RCStatus = RCStatus.Disabled + private agentID: string | null = null + + constructor( + private options: AssistOptions, + private onGrand: (sting?) => void, + private onRelease: (sting?) => void) {} + + reconnect(ids: string[]) { + const storedID = sessionStorage.getItem(this.options.session_control_peer_key) + if (storedID !== null && ids.includes(storedID)) { + this.grantControl(storedID) + } else { + sessionStorage.removeItem(this.options.session_control_peer_key) + } + } + + private confirm: ConfirmWindow | null = null + requestControl = (id: string) => { + if (this.agentID !== null) { + this.releaseControl(id) + return + } + setTimeout(() =>{ + if (this.status === RCStatus.Requesting) { + this.releaseControl(id) + } + }, 30000) + this.agentID = id + this.status = RCStatus.Requesting + this.confirm = new ConfirmWindow(controlConfirmDefault(this.options.controlConfirm)) + this.confirm.mount().then(allowed => { + if (allowed) { + this.grantControl(id) + } else { + this.releaseControl(id) + } + }).catch() + } + grantControl = (id: string) => { + this.agentID = id + this.status = RCStatus.Enabled + this.mouse = new Mouse() + this.mouse.mount() + sessionStorage.setItem(this.options.session_control_peer_key, id) + this.onGrand(id) + } + + releaseControl = (id: string) => { + if (this.agentID !== id) { return } + this.confirm?.remove() + this.mouse?.remove() + this.mouse = null + this.status = RCStatus.Disabled + this.agentID = null + sessionStorage.removeItem(this.options.session_control_peer_key) + this.onRelease(id) + } + + scroll = (id, d) => { id === this.agentID && this.mouse?.scroll(d) } + move = (id, xy) => { id === this.agentID && this.mouse?.move(xy) } + private focused: HTMLElement | null = null + click = (id, xy) => { + if (id !== this.agentID || !this.mouse) { return } + this.focused = this.mouse.click(xy) + } + input = (id, value) => { + if (id !== this.agentID || !this.mouse || !this.focused) { return } + if (this.focused instanceof HTMLTextAreaElement + || this.focused instanceof HTMLInputElement) { + this.focused.value = value + } else if (this.focused.isContentEditable) { + this.focused.innerText = value + } + } +} \ No newline at end of file From ba4b322fc5b2a92acf5730271178c0ec626d0f0d Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 23 Mar 2022 17:50:03 +0100 Subject: [PATCH 12/22] change(ui) - swap snippet tabs --- .../Onboarding/components/OnboardingTabs/OnboardingTabs.js | 4 ++-- .../components/shared/TrackingCodeModal/TrackingCodeModal.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/app/components/Onboarding/components/OnboardingTabs/OnboardingTabs.js b/frontend/app/components/Onboarding/components/OnboardingTabs/OnboardingTabs.js index fd05da5b6..398b7d240 100644 --- a/frontend/app/components/Onboarding/components/OnboardingTabs/OnboardingTabs.js +++ b/frontend/app/components/Onboarding/components/OnboardingTabs/OnboardingTabs.js @@ -8,14 +8,14 @@ const DOCUMENTATION = 'NPM'; // const SEGMENT = 'SEGMENT'; // const GOOGLE_TAG = 'GOOGLE TAG'; const TABS = [ - { key: PROJECT, text: PROJECT }, { key: DOCUMENTATION, text: DOCUMENTATION }, + { key: PROJECT, text: PROJECT }, // { key: SEGMENT, text: SEGMENT }, // { key: GOOGLE_TAG, text: GOOGLE_TAG } ]; class TrackingCodeModal extends React.PureComponent { - state = { copied: false, changed: false, activeTab: PROJECT }; + state = { copied: false, changed: false, activeTab: DOCUMENTATION }; setActiveTab = (tab) => { this.setState({ activeTab: tab }); diff --git a/frontend/app/components/shared/TrackingCodeModal/TrackingCodeModal.js b/frontend/app/components/shared/TrackingCodeModal/TrackingCodeModal.js index 1a8eb17bf..a94eb5f1a 100644 --- a/frontend/app/components/shared/TrackingCodeModal/TrackingCodeModal.js +++ b/frontend/app/components/shared/TrackingCodeModal/TrackingCodeModal.js @@ -10,12 +10,12 @@ import cn from 'classnames'; const PROJECT = 'Using Script'; const DOCUMENTATION = 'Using NPM'; const TABS = [ + { key: DOCUMENTATION, text: DOCUMENTATION }, { key: PROJECT, text: PROJECT }, - { key: DOCUMENTATION, text: DOCUMENTATION } ]; class TrackingCodeModal extends React.PureComponent { - state = { copied: false, changed: false, activeTab: PROJECT }; + state = { copied: false, changed: false, activeTab: DOCUMENTATION }; setActiveTab = (tab) => { this.setState({ activeTab: tab }); From b0d42d26344370a4dcd0aceabe26109c2744b83c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 23 Mar 2022 18:54:16 +0100 Subject: [PATCH 13/22] feat(api): changed funnels drop percentage formula --- api/chalicelib/core/funnels.py | 7 ++++++- api/chalicelib/core/significance.py | 26 +------------------------- 2 files changed, 7 insertions(+), 26 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 1a56f0272..0b49037e6 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -217,9 +217,14 @@ def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_dat return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=f["filter"], project_id=project_id) + insights = helper.list_to_camel_case(insights) if len(insights) > 0: + # fix: this fix for huge drop count + if total_drop_due_to_issues > insights[0]["sessionsCount"]: + total_drop_due_to_issues = insights[0]["sessionsCount"] + # end fix insights[-1]["dropDueToIssues"] = total_drop_due_to_issues - return {"data": {"stages": helper.list_to_camel_case(insights), + return {"data": {"stages": insights, "totalDropDueToIssues": total_drop_due_to_issues}} diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index 035890e2f..2e698dcfd 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -528,7 +528,7 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False) split = issue.split('__^__') issues_dict['significant' if is_sign else 'insignificant'].append({ "type": split[0], - "title": get_issue_title(split[0]), + "title": helper.get_issue_title(split[0]), "affected_sessions": affected_sessions[issue], "unaffected_sessions": session_counts[1] - affected_sessions[issue], "lost_conversions": lost_conversions, @@ -641,27 +641,3 @@ def get_overview(filter_d, project_id, first_stage=None, last_stage=None): output['stages'] = stages_list output['criticalIssuesCount'] = n_critical_issues return output - - -def get_issue_title(issue_type): - return {'click_rage': "Click Rage", - 'dead_click': "Dead Click", - 'excessive_scrolling': "Excessive Scrolling", - 'bad_request': "Bad Request", - 'missing_resource': "Missing Image", - 'memory': "High Memory Usage", - 'cpu': "High CPU", - 'slow_resource': "Slow Resource", - 'slow_page_load': "Slow Page Performance", - 'crash': "Crash", - 'ml_cpu': "High CPU", - 'ml_memory': "High Memory Usage", - 'ml_dead_click': "Dead Click", - 'ml_click_rage': "Click Rage", - 'ml_mouse_thrashing': "Mouse Thrashing", - 'ml_excessive_scrolling': "Excessive Scrolling", - 'ml_slow_resources': "Slow Resource", - 'custom': "Custom Event", - 'js_exception': "Error", - 'custom_event_error': "Custom Error", - 'js_error': "Error"}.get(issue_type, issue_type) From 62ec33c6b5fd6579550153d9a7adcf4e95e344a4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 23 Mar 2022 19:08:52 +0100 Subject: [PATCH 14/22] feat(api): changed funnels drop percentage formula --- api/chalicelib/core/funnels.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 0b49037e6..3ae31f5e4 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -241,6 +241,10 @@ def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data: schemas.Fu data.events = __fix_stages(data.events) insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data.dict(), project_id=project_id) if len(insights) > 0: + # fix: this fix for huge drop count + if total_drop_due_to_issues > insights[0]["sessionsCount"]: + total_drop_due_to_issues = insights[0]["sessionsCount"] + # end fix insights[-1]["dropDueToIssues"] = total_drop_due_to_issues return {"data": {"stages": helper.list_to_camel_case(insights), "totalDropDueToIssues": total_drop_due_to_issues}} From 926e91e6cb3c7719c9e6b28b880daa0b377f88ab Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 23 Mar 2022 19:18:31 +0100 Subject: [PATCH 15/22] feat(api): changed funnels drop percentage --- api/chalicelib/core/funnels.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 3ae31f5e4..1dc9e3347 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -240,13 +240,14 @@ def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data: schemas.Fu data = schemas.FunnelInsightsPayloadSchema.parse_obj(f["filter"]) data.events = __fix_stages(data.events) insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data.dict(), project_id=project_id) + insights = helper.list_to_camel_case(insights) if len(insights) > 0: # fix: this fix for huge drop count if total_drop_due_to_issues > insights[0]["sessionsCount"]: total_drop_due_to_issues = insights[0]["sessionsCount"] # end fix insights[-1]["dropDueToIssues"] = total_drop_due_to_issues - return {"data": {"stages": helper.list_to_camel_case(insights), + return {"data": {"stages": insights, "totalDropDueToIssues": total_drop_due_to_issues}} From 1e3bc978279ddc7eb1a1604370e3181774dcc574 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Wed, 23 Mar 2022 23:25:36 +0400 Subject: [PATCH 16/22] Updated version to v1.5.4 --- frontend/env.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/env.js b/frontend/env.js index a76295cf1..7c026cd31 100644 --- a/frontend/env.js +++ b/frontend/env.js @@ -13,7 +13,7 @@ const oss = { ORIGIN: () => 'window.location.origin', API_EDP: () => 'window.location.origin + "/api"', ASSETS_HOST: () => 'window.location.origin + "/assets"', - VERSION: '1.5.3', + VERSION: '1.5.4', SOURCEMAP: true, MINIO_ENDPOINT: process.env.MINIO_ENDPOINT, MINIO_PORT: process.env.MINIO_PORT, From cdf8a9cbdd51a023d031818b6021bb27df3fd114 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 24 Mar 2022 12:27:28 +0100 Subject: [PATCH 17/22] feat(frontend): updated tracker version --- frontend/env.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/env.js b/frontend/env.js index 7c026cd31..8c5294fd8 100644 --- a/frontend/env.js +++ b/frontend/env.js @@ -21,7 +21,7 @@ const oss = { MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, ICE_SERVERS: process.env.ICE_SERVERS, - TRACKER_VERSION: '3.5.3' // trackerInfo.version, + TRACKER_VERSION: '3.5.4' // trackerInfo.version, } module.exports = { From 77bf304fcd4965fc59509666381f1f8bf5054307 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 24 Mar 2022 14:46:37 +0100 Subject: [PATCH 18/22] feat(api): fixed autocomplete redundant values --- api/chalicelib/core/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/core/events.py b/api/chalicelib/core/events.py index db515d995..933e3f800 100644 --- a/api/chalicelib/core/events.py +++ b/api/chalicelib/core/events.py @@ -432,7 +432,7 @@ def __get_autocomplete_table(value, project_id): AND value ILIKE %(value)s LIMIT 5)""") with pg_client.PostgresClient() as cur: - query = cur.mogrify("UNION ALL".join(sub_queries) + ";", + query = cur.mogrify(" UNION ".join(sub_queries) + ";", {"project_id": project_id, "value": helper.string_to_sql_like(value), "svalue": helper.string_to_sql_like("^" + value)}) cur.execute(query) From 1a6c85a5f45a970cbdec6894176c9c1b90ce6cc2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 24 Mar 2022 15:22:38 +0100 Subject: [PATCH 19/22] feat(db): fixed upgrade delta queries --- ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql | 2 +- scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql index 46f04521c..a32610f4a 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -22,7 +22,7 @@ $$ LIMIT 1) ORDER BY LOWER(email)) THEN raise notice 'duplicate users detected'; - FOR duplicate IN SELECT user_id, email, deleted_at, verified_email, jwt_iat + FOR duplicate IN SELECT user_id, email, deleted_at, jwt_iat FROM users WHERE lower(email) = (SELECT LOWER(email) diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql index 40fd1c88e..744f96563 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -22,7 +22,7 @@ $$ LIMIT 1) ORDER BY LOWER(email)) THEN raise notice 'duplicate users detected'; - FOR duplicate IN SELECT user_id, email, deleted_at, verified_email, jwt_iat + FOR duplicate IN SELECT user_id, email, deleted_at, jwt_iat FROM users WHERE lower(email) = (SELECT LOWER(email) From abfb6620fcd7103ac99e75e1e7df636470c3cf62 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 24 Mar 2022 15:24:12 +0100 Subject: [PATCH 20/22] feat(db): allow delta fail --- ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql | 1 + scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql index a32610f4a..c8469f77c 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -1,3 +1,4 @@ +\set ON_ERROR_STOP true BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql index 744f96563..dabed7d76 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -1,3 +1,4 @@ +\set ON_ERROR_STOP true BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS From 6ecc7ddbbb09270f3b7ee3978a1c2c15bfcabbb6 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Thu, 24 Mar 2022 14:38:36 +0000 Subject: [PATCH 21/22] stdout stderr ouputs --- scripts/helmcharts/openreplay/files/postgresql.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/openreplay/files/postgresql.sh b/scripts/helmcharts/openreplay/files/postgresql.sh index cea47a73e..a641e4650 100644 --- a/scripts/helmcharts/openreplay/files/postgresql.sh +++ b/scripts/helmcharts/openreplay/files/postgresql.sh @@ -17,13 +17,13 @@ function migrate() { IFS=',' read -r -a migration_versions <<< "$1" for version in ${migration_versions[*]}; do echo "Migrating postgresql version $version" - psql -f ${pgdir}/${version}/${version}.sql + psql -f ${pgdir}/${version}/${version}.sql 2>&1 done } function init() { echo "Initializing postgresql" - psql -f ${pgdir}/init_schema.sql + psql -f ${pgdir}/init_schema.sql 2>&1 } # /bin/bash postgresql.sh migrate $migration_versions From 88be23dd120b37de4d36b6293733e9e1ae42c561 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 24 Mar 2022 15:55:18 +0100 Subject: [PATCH 22/22] feat(db): show notice messages --- ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql | 1 + scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql index c8469f77c..1a640b4be 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -1,4 +1,5 @@ \set ON_ERROR_STOP true +SET client_min_messages TO NOTICE; BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql index dabed7d76..e7be94997 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -1,4 +1,5 @@ \set ON_ERROR_STOP true +SET client_min_messages TO NOTICE; BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS